bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

[Paul Eggert]

[-- Attachment #1: Type: text/plain, Size: 564 bytes --]

Paul Eggert wrote:
> Eli Zaretskii wrote:
>> once
>> gc-cons-threshold is set to a large value, it effectively disables GC
>> for the rest of the session, even if after that gc-cons-threshold is
>> reset back
> 
> Yes, that's next on my list of things to look at, after untangling this XDG 
> configuration mess.

Although I haven't finished untangling the XDG configuration mess, I did get 
some time free to attack the gc-cons-threshold issue and installed the attached. 
As this should fix the problem reported in Bug#37006 I'm boldly closing that bug 
report.

[-- Attachment #2: 0001-Sync-consing_until_gc-with-gc-cons-threshold.patch --]
[-- Type: text/x-patch, Size: 4951 bytes --]

From 97ffa339b6d67cebcbefbdfaa2880214adab639c Mon Sep 17 00:00:00 2001
From: Paul Eggert <eggert@cs.ucla.edu>
Date: Tue, 3 Sep 2019 13:03:34 -0700
Subject: [PATCH] Sync consing_until_gc with gc-cons-threshold

Add watchers for gc-cons-threshold and gc-cons-percentage
that update consing_until_gc accordingly.
Suggested by Eli Zaretskii (Bug#37006#52).
* src/alloc.c (consing_threshold, bump_consing_until_gc)
(watch_gc_cons_threshold, watch_gc_cons_percentage):
New functions.
(garbage_collect_1): Use consing_threshold.
(syms_of_alloc): Arrange to watch gc-cons-threshold and
gc-cons-percentage.
---
 src/alloc.c | 100 ++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 81 insertions(+), 19 deletions(-)

diff --git a/src/alloc.c b/src/alloc.c
index 39964c4b29..5f8ef0a5dd 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -5781,6 +5781,68 @@ mark_and_sweep_weak_table_contents (void)
     }
 }
 
+/* Return the number of bytes to cons between GCs, assuming
+   gc-cons-threshold is THRESHOLD and gc-cons-percentage is
+   GC_CONS_PERCENTAGE.  */
+static intmax_t
+consing_threshold (intmax_t threshold, Lisp_Object gc_cons_percentage)
+{
+  if (!NILP (Vmemory_full))
+    return memory_full_cons_threshold;
+  else
+    {
+      threshold = max (threshold, GC_DEFAULT_THRESHOLD / 10);
+      if (FLOATP (gc_cons_percentage))
+	{
+	  double tot = (XFLOAT_DATA (gc_cons_percentage)
+			* total_bytes_of_live_objects ());
+	  if (threshold < tot)
+	    {
+	      if (tot < INTMAX_MAX)
+		threshold = tot;
+	      else
+		threshold = INTMAX_MAX;
+	    }
+	}
+      return threshold;
+    }
+}
+
+/* Increment consing_until_gc by DIFF, avoiding overflow.  */
+static Lisp_Object
+bump_consing_until_gc (intmax_t diff)
+{
+  /* If consing_until_gc is negative leave it alone, since this prevents
+     negative integer overflow and a GC would have been done soon anyway.  */
+  if (0 <= consing_until_gc
+      && INT_ADD_WRAPV (consing_until_gc, diff, &consing_until_gc))
+    consing_until_gc = INTMAX_MAX;
+  return Qnil;
+}
+
+/* Watch changes to gc-cons-threshold.  */
+static Lisp_Object
+watch_gc_cons_threshold (Lisp_Object symbol, Lisp_Object newval,
+			 Lisp_Object operation, Lisp_Object where)
+{
+  intmax_t new_threshold;
+  int diff = (INTEGERP (newval) && integer_to_intmax (newval, &new_threshold)
+	      ? (consing_threshold (new_threshold, Vgc_cons_percentage)
+		 - consing_threshold (gc_cons_threshold, Vgc_cons_percentage))
+	      : 0);
+  return bump_consing_until_gc (diff);
+}
+
+/* Watch changes to gc-cons-percentage.  */
+static Lisp_Object
+watch_gc_cons_percentage (Lisp_Object symbol, Lisp_Object newval,
+			  Lisp_Object operation, Lisp_Object where)
+{
+  int diff = (consing_threshold (consing_until_gc, newval)
+	      - consing_threshold (consing_until_gc, Vgc_cons_percentage));
+  return bump_consing_until_gc (diff);
+}
+
 /* Subroutine of Fgarbage_collect that does most of the work.  */
 static bool
 garbage_collect_1 (struct gcstat *gcst)
@@ -5923,25 +5985,8 @@ garbage_collect_1 (struct gcstat *gcst)
 
   unblock_input ();
 
-  if (!NILP (Vmemory_full))
-    consing_until_gc = memory_full_cons_threshold;
-  else
-    {
-      intmax_t threshold = max (gc_cons_threshold, GC_DEFAULT_THRESHOLD / 10);
-      if (FLOATP (Vgc_cons_percentage))
-	{
-	  double tot = (XFLOAT_DATA (Vgc_cons_percentage)
-			* total_bytes_of_live_objects ());
-	  if (threshold < tot)
-	    {
-	      if (tot < INTMAX_MAX)
-		threshold = tot;
-	      else
-		threshold = INTMAX_MAX;
-	    }
-	}
-      consing_until_gc = threshold;
-    }
+  consing_until_gc = consing_threshold (gc_cons_threshold,
+					Vgc_cons_percentage);
 
   if (garbage_collection_messages && NILP (Vmemory_full))
     {
@@ -7362,6 +7407,7 @@ do hash-consing of the objects allocated to pure space.  */);
   DEFSYM (Qheap, "heap");
   DEFSYM (QAutomatic_GC, "Automatic GC");
 
+  DEFSYM (Qgc_cons_percentage, "gc-cons-percentage");
   DEFSYM (Qgc_cons_threshold, "gc-cons-threshold");
   DEFSYM (Qchar_table_extra_slots, "char-table-extra-slots");
 
@@ -7395,6 +7441,22 @@ N should be nonnegative.  */);
   defsubr (&Smemory_info);
   defsubr (&Smemory_use_counts);
   defsubr (&Ssuspicious_object);
+
+  Lisp_Object watcher;
+
+  static union Aligned_Lisp_Subr Swatch_gc_cons_threshold =
+     {{{ PSEUDOVECTOR_FLAG | (PVEC_SUBR << PSEUDOVECTOR_AREA_BITS) },
+       { .a4 = watch_gc_cons_threshold },
+       4, 4, "watch_gc_cons_threshold", 0, 0}};
+  XSETSUBR (watcher, &Swatch_gc_cons_threshold.s);
+  Fadd_variable_watcher (Qgc_cons_threshold, watcher);
+
+  static union Aligned_Lisp_Subr Swatch_gc_cons_percentage =
+     {{{ PSEUDOVECTOR_FLAG | (PVEC_SUBR << PSEUDOVECTOR_AREA_BITS) },
+       { .a4 = watch_gc_cons_percentage },
+       4, 4, "watch_gc_cons_percentage", 0, 0}};
+  XSETSUBR (watcher, &Swatch_gc_cons_percentage.s);
+  Fadd_variable_watcher (Qgc_cons_percentage, watcher);
 }
 
 #ifdef HAVE_X_WINDOWS
-- 
2.17.1

bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

[Joseph Mingrone]

[-- Attachment #1: Type: text/plain, Size: 1484 bytes --]

Paul Eggert <eggert@cs.ucla.edu> writes:

> Paul Eggert wrote:
>> Eli Zaretskii wrote:
>>> once
>>> gc-cons-threshold is set to a large value, it effectively disables GC
>>> for the rest of the session, even if after that gc-cons-threshold is
>>> reset back
>> Yes, that's next on my list of things to look at, after untangling this XDG 
>> configuration mess.

> Although I haven't finished untangling the XDG configuration mess, I did get some time free to attack the gc-cons-threshold issue and installed the attached. 
> As this should fix the problem reported in Bug#37006 I'm boldly closing that bug report.

This issue still exists for me.  After rebuilding on 5f089ac, I re-added
the (setq gc-cons-threshold most-positive-fixnum) / (setq
gc-cons-threshold 800000) surrounding init.el (assuming the issue was
fixed) then when Emacs' memory usage was over 2 GB, I turned on
`garbage-collection-messages', opened some large files, and could see
that garbage collection was not happening.

I just removed the surrounding (setq gc-cons-threshold
most-positive-fixnum) / (setq gc-cons-threshold 800000) and I see
garbage collection is happening for now.  I will report back if this
does not remain so.  In the last week or two running on a commit that
was some time after a recent fix (sorry, don't have the exact commit)
garbage collection was happening and the memory usage was not exploding.
The `gc-cons-threshold' hack in init.el was commented out.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 979 bytes --]

bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

[Paul Eggert]

On 9/5/19 9:12 AM, Joseph Mingrone wrote:
>> As this should fix the problem reported in Bug#37006 I'm boldly closing that bug report.
> This issue still exists for me.  After rebuilding on 5f089ac, I re-added
> the (setq gc-cons-threshold most-positive-fixnum) / (setq
> gc-cons-threshold 800000) surrounding init.el (assuming the issue was
> fixed) then when Emacs' memory usage was over 2 GB, I turned on
> `garbage-collection-messages', opened some large files, and could see
> that garbage collection was not happening.

OK, reopening the bug report. I take it that all I need to do is to 
modify init.el as mentioned above, and then edit some large files. I'll 
take a look at that.

bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

[Joseph Mingrone]

[-- Attachment #1: Type: text/plain, Size: 898 bytes --]

Paul Eggert <eggert@cs.ucla.edu> writes:

> On 9/5/19 9:12 AM, Joseph Mingrone wrote:
>>> As this should fix the problem reported in Bug#37006 I'm boldly closing that bug report.
>> This issue still exists for me.  After rebuilding on 5f089ac, I re-added
>> the (setq gc-cons-threshold most-positive-fixnum) / (setq
>> gc-cons-threshold 800000) surrounding init.el (assuming the issue was
>> fixed) then when Emacs' memory usage was over 2 GB, I turned on
>> `garbage-collection-messages', opened some large files, and could see
>> that garbage collection was not happening.

> OK, reopening the bug report. I take it that all I need to do is to modify init.el as mentioned above, and then edit some large files. I'll 
> take a look at that.

That's all I had to do (on a FreeBSD 12.0 system).  I can give you an
account a FreeBSD system if that is helpful and saves you some time.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 979 bytes --]

bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

[Paul Eggert]

[-- Attachment #1: Type: text/plain, Size: 100 bytes --]

I reproduced the problem on Fedora 30 x86-64 and installed the attached 
fix. Please give it a try.

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Fix-bugs-when-recalculating-consing_until_gc.patch --]
[-- Type: text/x-patch; name="0001-Fix-bugs-when-recalculating-consing_until_gc.patch", Size: 2948 bytes --]

From 2c246fde5ebe76bf48322c09fd685e48c0737f2a Mon Sep 17 00:00:00 2001
From: Paul Eggert <eggert@cs.ucla.edu>
Date: Thu, 5 Sep 2019 13:25:43 -0700
Subject: [PATCH] Fix bugs when recalculating consing_until_gc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Problem reported by Joseph Mingrone (Bug#37006#72).
* src/alloc.c (watch_gc_cons_threshold)
(watch_gc_cons_percentage):
Don’t try to store an intmax_t into an int.
Redo to make the code clearer.
(watch_gc_cons_percentage):
Use gc_cons_threshold, not consing_until_gc.
---
 src/alloc.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/alloc.c b/src/alloc.c
index 089f61f833..5fc515f33b 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -5783,18 +5783,18 @@ mark_and_sweep_weak_table_contents (void)
 
 /* Return the number of bytes to cons between GCs, assuming
    gc-cons-threshold is THRESHOLD and gc-cons-percentage is
-   GC_CONS_PERCENTAGE.  */
+   PERCENTAGE.  */
 static intmax_t
-consing_threshold (intmax_t threshold, Lisp_Object gc_cons_percentage)
+consing_threshold (intmax_t threshold, Lisp_Object percentage)
 {
   if (!NILP (Vmemory_full))
     return memory_full_cons_threshold;
   else
     {
       threshold = max (threshold, GC_DEFAULT_THRESHOLD / 10);
-      if (FLOATP (gc_cons_percentage))
+      if (FLOATP (percentage))
 	{
-	  double tot = (XFLOAT_DATA (gc_cons_percentage)
+	  double tot = (XFLOAT_DATA (percentage)
 			* total_bytes_of_live_objects ());
 	  if (threshold < tot)
 	    {
@@ -5825,11 +5825,12 @@ bump_consing_until_gc (intmax_t diff)
 watch_gc_cons_threshold (Lisp_Object symbol, Lisp_Object newval,
 			 Lisp_Object operation, Lisp_Object where)
 {
-  intmax_t new_threshold;
-  int diff = (INTEGERP (newval) && integer_to_intmax (newval, &new_threshold)
-	      ? (consing_threshold (new_threshold, Vgc_cons_percentage)
-		 - consing_threshold (gc_cons_threshold, Vgc_cons_percentage))
-	      : 0);
+  Lisp_Object percentage = Vgc_cons_percentage;
+  intmax_t threshold;
+  intmax_t diff = (INTEGERP (newval) && integer_to_intmax (newval, &threshold)
+		   ? (consing_threshold (threshold, percentage)
+		      - consing_threshold (gc_cons_threshold, percentage))
+		   : 0);
   return bump_consing_until_gc (diff);
 }
 
@@ -5838,8 +5839,9 @@ watch_gc_cons_threshold (Lisp_Object symbol, Lisp_Object newval,
 watch_gc_cons_percentage (Lisp_Object symbol, Lisp_Object newval,
 			  Lisp_Object operation, Lisp_Object where)
 {
-  int diff = (consing_threshold (consing_until_gc, newval)
-	      - consing_threshold (consing_until_gc, Vgc_cons_percentage));
+  intmax_t threshold = gc_cons_threshold;
+  intmax_t diff = (consing_threshold (threshold, newval)
+		   - consing_threshold (threshold, Vgc_cons_percentage));
   return bump_consing_until_gc (diff);
 }
 
-- 
2.21.0

bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

[Joseph Mingrone]

[-- Attachment #1: Type: text/plain, Size: 351 bytes --]

Paul Eggert <eggert@cs.ucla.edu> writes:

> I reproduced the problem on Fedora 30 x86-64 and installed the attached fix. Please give it a try.

Hello Paul,

I see lots of garbage collection messages after a few minutes, so it
seems to be working.  If garbage collection stops after Emacs has been
running longer, I'll report back.

Thank you,

Joseph

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 979 bytes --]

bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

[Paul Eggert]

On 9/5/19 2:28 PM, Joseph Mingrone wrote:

> I see lots of garbage collection messages after a few minutes, so it
> seems to be working.  If garbage collection stops after Emacs has been
> running longer, I'll report back.

Thanks for checking. I'll close the bug report (again :-). If the bug 
reappears I can reopen it again.

bug#37006: Emacs 27 master consumes more memory than 26 and freezes regularly

[Eli Zaretskii]

> Cc: Eli Zaretskii <eliz@gnu.org>, Akater <nuclearspace@gmail.com>,
>  37006@debbugs.gnu.org, Mattias Engdegård <mattiase@acm.org>
> From: Paul Eggert <eggert@cs.ucla.edu>
> Date: Thu, 5 Sep 2019 14:34:56 -0700
> 
> On 9/5/19 2:28 PM, Joseph Mingrone wrote:
> 
> > I see lots of garbage collection messages after a few minutes, so it
> > seems to be working.  If garbage collection stops after Emacs has been
> > running longer, I'll report back.
> 
> Thanks for checking. I'll close the bug report (again :-). If the bug 
> reappears I can reopen it again.

Thanks for fixing the bug.  I wonder if it would make sense to have a
simple test for this particular use case, as this seems to be quite a
popular technique these days in users' init files.  The test would do
the sequence of GC settings like the one which triggered this bug, and
then verify that GC happens after it when expected.

WDYT?