bug#27571: #27571; C stack overflow from `prin1' on deeply nested lisp object.

[Keith David Bershatsky <esq@lawlist.com>]

Paul:

I will use the layman's approach of performing git hard resets and going back in time -- building Emacs 25 and trying the test case.  I used vc-region-history on emacs.c for the Emacs 25 branch and came up with just a few hits.  I will start chiseling away with the layman's approach as time permits and report back with the results.

Keith

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

commit 1564080f0b24551765d7068b9fc02f6e5a78fea3
Author: Paul Eggert <eggert@cs.ucla.edu>
Date:   Sun Aug 31 19:37:22 2014 -0700

    Clean up extern decls a bit.
    
    * configure.ac (WERROR_CFLAGS): Don't disable -Wnested-externs.
    While we're at it, don't disable -Wlogical-op either.
    * src/bytecode.c: Include blockinput.h and keyboard.h rather
    than rolling their APIs by hand.
    * src/emacs.c: Include regex.h and rely on its and lisp.h's API
    rather than rolling them by hand.
    * src/lastfile.c: Include lisp.h, to check this file's API.
    * src/lisp.h (lisp_eval_depth, my_edata, my_endbss, my_endbss_static):
    New decls.
    * src/regex.h (re_max_failures): New decl.
    * src/unexcw.c, src/unexmacosx.c, src/unexw32.c:
    Rely on lisp.h's API rather than rolling it by hand.
    * src/vm-limit.c (__after_morecore_hook, __morecore, real_morecore):
    Declare at top level, to pacify GCC -Wnested-externs.

diff --git a/src/emacs.c b/src/emacs.c
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -868,34 +866,33 @@
   if (1
 #ifndef CANNOT_DUMP
       && (!noninteractive || initialized)
 #endif
       && !getrlimit (RLIMIT_STACK, &rlim))
     {
       long newlim;
-      extern size_t re_max_failures;
       /* Approximate the amount regex.c needs per unit of re_max_failures.  */
       int ratio = 20 * sizeof (char *);
       /* Then add 33% to cover the size of the smaller stacks that regex.c
 	 successively allocates and discards, on its way to the maximum.  */
       ratio += ratio / 3;
       /* Add in some extra to cover
 	 what we're likely to use for other reasons.  */
       newlim = re_max_failures * ratio + 200000;
 #ifdef __NetBSD__
       /* NetBSD (at least NetBSD 1.2G and former) has a bug in its
        stack allocation routine for new process that the allocation
        fails if stack limit is not on page boundary.  So, round up the
        new limit to page boundary.  */
       newlim = (newlim + getpagesize () - 1) / getpagesize () * getpagesize ();
 #endif
       if (newlim > rlim.rlim_max)
 	{
 	  newlim = rlim.rlim_max;
 	  /* Don't let regex.c overflow the stack we have.  */
 	  re_max_failures = (newlim - 200000) / ratio;
 	}
       if (rlim.rlim_cur < newlim)
 	rlim.rlim_cur = newlim;
 
       setrlimit (RLIMIT_STACK, &rlim);
     }

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

commit 5e617bc2b62189768814fafd1a875e89a094d3ef
Author: Juanma Barranquero <lekktu@gmail.com>
Date:   Fri Sep 9 03:06:52 2011 +0200

    Whitespace changes.

diff --git a/src/emacs.c b/src/emacs.c
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -830,34 +830,34 @@
   if (1
 #ifndef CANNOT_DUMP
       && (!noninteractive || initialized)
 #endif
       && !getrlimit (RLIMIT_STACK, &rlim))
     {
       long newlim;
       extern size_t re_max_failures;
       /* Approximate the amount regex.c needs per unit of re_max_failures.  */
       int ratio = 20 * sizeof (char *);
       /* Then add 33% to cover the size of the smaller stacks that regex.c
 	 successively allocates and discards, on its way to the maximum.  */
       ratio += ratio / 3;
       /* Add in some extra to cover
 	 what we're likely to use for other reasons.  */
       newlim = re_max_failures * ratio + 200000;
 #ifdef __NetBSD__
       /* NetBSD (at least NetBSD 1.2G and former) has a bug in its
        stack allocation routine for new process that the allocation
        fails if stack limit is not on page boundary.  So, round up the
        new limit to page boundary.  */
-      newlim = (newlim + getpagesize () - 1) / getpagesize () * getpagesize();
+      newlim = (newlim + getpagesize () - 1) / getpagesize () * getpagesize ();
 #endif
       if (newlim > rlim.rlim_max)
 	{
 	  newlim = rlim.rlim_max;
 	  /* Don't let regex.c overflow the stack we have.  */
 	  re_max_failures = (newlim - 200000) / ratio;
 	}
       if (rlim.rlim_cur < newlim)
 	rlim.rlim_cur = newlim;
 
       setrlimit (RLIMIT_STACK, &rlim);
     }

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

commit fa8459a34e076cacde3b7c259af9b5dd84b60802
Author: Dan Nicolaescu <dann@ics.uci.edu>
Date:   Fri Sep 30 22:38:16 2005 +0000

    * image.c (slurp_file, xbm_read_bitmap_data): Cast to the correct
    type.
    * xterm.c (handle_one_xevent, handle_one_xevent): Likewise.
    
    * unexelf.c (fatal): Fix prototype.
    
    * term.c (fatal): Implement using varargs.
    
    * regex.c (re_char): Move typedef ...
    * regex.h (re_char): ... here.
    (re_iswctype, re_wctype, re_set_whitespace_regexp): New
    prototypes.
    
    * emacs.c (malloc_set_state): Fix return type.
    (endif): Fix type.
    
    * lisp.h (fatal): Add argument types.
    
    * dispextern.h (fatal): Delete prototype.
    
    * systime.h: (make_time): Prototype moved from ...
    * editfns.c (make_time): ... here.
    
    * editfns.c: Move systime.h include after lisp.h.
    * dired.c:
    * xsmfns.c:
    * process.c: Likewise.
    
    * alloc.c (old_malloc_hook, old_realloc_hook, old_realloc_hook):
    Add parameter types.
    (__malloc_hook, __realloc_hook, __free_hook): Fix prototypes.
    (emacs_blocked_free): Change definition to match __free_hook.
    (emacs_blocked_malloc): Change definition to match __malloc_hook.
    (emacs_blocked_realloc): Change definition to match
    __realloc_hook.

diff --git a/src/emacs.c b/src/emacs.c
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -997,34 +997,34 @@
   if (1
 #ifndef CANNOT_DUMP
       && (!noninteractive || initialized)
 #endif
       && !getrlimit (RLIMIT_STACK, &rlim))
     {
       long newlim;
-      extern int re_max_failures;
+      extern size_t re_max_failures;
       /* Approximate the amount regex.c needs per unit of re_max_failures.  */
       int ratio = 20 * sizeof (char *);
       /* Then add 33% to cover the size of the smaller stacks that regex.c
 	 successively allocates and discards, on its way to the maximum.  */
       ratio += ratio / 3;
       /* Add in some extra to cover
 	 what we're likely to use for other reasons.  */
       newlim = re_max_failures * ratio + 200000;
 #ifdef __NetBSD__
       /* NetBSD (at least NetBSD 1.2G and former) has a bug in its
        stack allocation routine for new process that the allocation
        fails if stack limit is not on page boundary.  So, round up the
        new limit to page boundary.  */
       newlim = (newlim + getpagesize () - 1) / getpagesize () * getpagesize();
 #endif
       if (newlim > rlim.rlim_max)
 	{
 	  newlim = rlim.rlim_max;
 	  /* Don't let regex.c overflow the stack we have.  */
 	  re_max_failures = (newlim - 200000) / ratio;
 	}
       if (rlim.rlim_cur < newlim)
 	rlim.rlim_cur = newlim;
 
       setrlimit (RLIMIT_STACK, &rlim);
     }

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

commit 03effc232ed9b79aba077d912f17dd844d703e5e
Author: Karl Heuer <kwzh@gnu.org>
Date:   Thu Dec 4 05:53:41 1997 +0000

    (main): Fix the stack-limit code to calculate
    the ratio for re_max_failures accurately and leave some extra slack.

diff --git a/src/emacs.c b/src/emacs.c
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -590,28 +590,34 @@
   if (1
 #ifndef CANNOT_DUMP
       && (!noninteractive || initialized)
 #endif
       && !getrlimit (RLIMIT_STACK, &rlim))
     {
       long newlim;
       extern int re_max_failures;
-      /* Approximate the amount regex.c needs, plus some more.  */
-      newlim = re_max_failures * 2 * 20 * sizeof (char *);
+      /* Approximate the amount regex.c needs per unit of re_max_failures.  */
+      int ratio = 20 * sizeof (char *);
+      /* Then add 33% to cover the size of the smaller stacks that regex.c
+	 successively allocates and discards, on its way to the maximum.  */
+      ratio += ratio / 3;
+      /* Add in some extra to cover
+	 what we're likely to use for other reasons.  */
+      newlim = re_max_failures * ratio + 200000;
 #ifdef __NetBSD__
       /* NetBSD (at least NetBSD 1.2G and former) has a bug in its
        stack allocation routine for new process that the allocation
        fails if stack limit is not on page boundary.  So, round up the
        new limit to page boundary.  */
       newlim = (newlim + getpagesize () - 1) / getpagesize () * getpagesize();
 #endif
       if (newlim > rlim.rlim_max)
 	{
 	  newlim = rlim.rlim_max;
-	  /* Don't let regex.c overflow the stack.  */
-	  re_max_failures = newlim / (2 * 20 * sizeof (char *));
+	  /* Don't let regex.c overflow the stack we have.  */
+	  re_max_failures = (newlim - 200000) / ratio;
 	}
       if (rlim.rlim_cur < newlim)
 	rlim.rlim_cur = newlim;
 
       setrlimit (RLIMIT_STACK, &rlim);
     }

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

commit 6c2935e99fd15a0c10a4a648a09a499076e031c1
Author: Richard M. Stallman <rms@gnu.org>
Date:   Fri Aug 15 05:07:01 1997 +0000

    (main): Update re_max_failures so regex.c won't overflow
    the stack, except when dumping.

diff --git a/src/emacs.c b/src/emacs.c
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -582,20 +585,28 @@
-  /* Extend the stack space available.  */
-  if (!getrlimit (RLIMIT_STACK, &rlim))
+  if (1
+#ifndef CANNOT_DUMP
+      && (!noninteractive || initialized)
+#endif
+      && !getrlimit (RLIMIT_STACK, &rlim))
     {
       long newlim;
+      extern int re_max_failures;
       /* Approximate the amount regex.c needs, plus some more.  */
-      newlim = 800000 * sizeof (char *);
+      newlim = re_max_failures * 2 * 20 * sizeof (char *);
 #ifdef __NetBSD__
       /* NetBSD (at least NetBSD 1.2G and former) has a bug in its
        stack allocation routine for new process that the allocation
        fails if stack limit is not on page boundary.  So, round up the
        new limit to page boundary.  */
       newlim = (newlim + getpagesize () - 1) / getpagesize () * getpagesize();
 #endif
       if (newlim > rlim.rlim_max)
-	newlim = rlim.rlim_max;
+	{
+	  newlim = rlim.rlim_max;
+	  /* Don't let regex.c overflow the stack.  */
+	  re_max_failures = newlim / (2 * 20 * sizeof (char *));
+	}
       if (rlim.rlim_cur < newlim)
 	rlim.rlim_cur = newlim;
 
       setrlimit (RLIMIT_STACK, &rlim);
     }

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

commit d0381a7fa3f50d1042a2372eb23b6f03299aaaa5
Author: Richard M. Stallman <rms@gnu.org>
Date:   Wed Jul 9 00:07:19 1997 +0000

    (main) [__NetBSD__]: Round up new stack limit to page bdry.

diff --git a/src/emacs.c b/src/emacs.c
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -581,13 +581,20 @@
   /* Extend the stack space available.  */
   if (!getrlimit (RLIMIT_STACK, &rlim))
     {
       long newlim;
       /* Approximate the amount regex.c needs, plus some more.  */
       newlim = 800000 * sizeof (char *);
+#ifdef __NetBSD__
+      /* NetBSD (at least NetBSD 1.2G and former) has a bug in its
+       stack allocation routine for new process that the allocation
+       fails if stack limit is not on page boundary.  So, round up the
+       new limit to page boundary.  */
+      newlim = (newlim + getpagesize () - 1) / getpagesize () * getpagesize();
+#endif
       if (newlim > rlim.rlim_max)
 	newlim = rlim.rlim_max;
       if (rlim.rlim_cur < newlim)
 	rlim.rlim_cur = newlim;
 
       setrlimit (RLIMIT_STACK, &rlim);
     }

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

commit 509a8fcde89b144b6638693f1bbeb854e7aa492c
Author: Richard M. Stallman <rms@gnu.org>
Date:   Mon Feb 3 02:51:09 1997 +0000

    (main): Don't extend stack limit too far.

diff --git a/src/emacs.c b/src/emacs.c
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -553,6 +553,13 @@
   /* Extend the stack space available.  */
   if (!getrlimit (RLIMIT_STACK, &rlim))
     {
-      rlim.rlim_cur = rlim.rlim_max;
+      long newlim;
+      /* Approximate the amount regex.c needs, plus some more.  */
+      newlim = 800000 * sizeof (char *);
+      if (newlim > rlim.rlim_max)
+	newlim = rlim.rlim_max;
+      if (rlim.rlim_cur < newlim)
+	rlim.rlim_cur = newlim;
+
       setrlimit (RLIMIT_STACK, &rlim);
     }

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

commit 53c58b5d489f21fdeb5f3d011e34638f8124fb91
Author: Richard M. Stallman <rms@gnu.org>
Date:   Sun Sep 1 20:47:10 1996 +0000

    [HAVE_SETRLIMIT]: Include time.h and resource.h.
    (main) [HAVE_SETRLIMIT]: Call setrlimit to extend the stack limit.
    New local `rlim'.

diff --git a/src/emacs.c b/src/emacs.c
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -504,0 +513,6 @@
+  /* Extend the stack space available.  */
+  if (!getrlimit (RLIMIT_STACK, &rlim))
+    {
+      rlim.rlim_cur = rlim.rlim_max;
+      setrlimit (RLIMIT_STACK, &rlim);
+    }


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

DATE:  [01-08-2018 23:53:05] <8 Jan 2018 23:53:05 -0800>
FROM:  Paul Eggert <eggert@cs.ucla.edu>
> 
> * * *
> 
> That commit was a merge commit, and installed all sorts of changes. The patch 
> you sent reverses just part of the commit. It'd be helpful to know the original 
> commit that caused the problem, as opposed to the later merge.
> 
> Also, the patch undoes some fixes, such as integer overflow checking, that we'd 
> like to keep. This is another reason that it'd be helpful to know the original 
> commit. Alternatively, it'd be helpful to know why the patch fixes the bug, so 
> that we can keep that part of the patch without discarding other fixes from the 
> source.