bug#24358: 25.1.50; re-search-forward errors with "Variable binding depth exceeds max-specpdl-size"

[npostavs@users.sourceforge.net]

[-- Attachment #1: Type: text/plain, Size: 2720 bytes --]

Eli Zaretskii <eliz@gnu.org> writes:

>> From: npostavs@users.sourceforge.net
>> Date: Fri, 07 Oct 2016 20:29:36 -0400
>> Cc: 24358@debbugs.gnu.org
>> 
>> npostavs@users.sourceforge.net writes:
>> >
>> >> (I'm also on GNU/Linux, Arch) I get the same max-specpdl-size error with
>> >> 25.1.50, with 24.5 (and below) I get (error "Stack overflow in regexp
>> >> matcher")
>> 
>> icalendar--read-element has been fixed, but this still reproduces when
>> doing (re-search-forward ".*\\(\n.*\\)*" nil t) on the text file given
>> in the OP.
>
> Isn't that "user error"?

Yes, but it should give "Stack overflow in regexp matcher", not overflow
the lisp stack (or assertion failure).

>
>> And I'm still seeing an assertion failure due to what looks like
>> memory corruption on the emacs-25 branch.
>
> Details of the assertion?

(See also https://debbugs.gnu.org/cgi/bugreport.cgi?bug=24358#8)

I tracked the corruption to a malloc call, but I wasn't able to figure
out what's happening there.

I used the following to debug:

Apply the attached bug-24358-hunting.diff and then run

    gdb --args ./emacs -Q -batch -l ~/src/emacs/bug-24358-regex-max-specpdl.el

Where ~/src/emacs/bug-24358-regex-max-specpdl.el is:

    (with-temp-buffer
      (insert-file-contents "~/src/emacs/bug-24358-regex-max-specpdl.txt") ; adjust path
      (goto-char (point-min))
      (re-search-forward ".*\\(\n.*\\)*" nil t))

I show some more excerpts in the attached bug-24358-debug.log, but my
main finding is that string1 of re_match_2_internal is originally:

    string1=0x1835980 "DESCRIPTION;LANGUAGE=

but then it becomes corrupted during a malloc:

Old value = 68 'D'
New value = 0 '\000'
0x00007ffff0cc01a7 in __memset_sse2_unaligned_erms () from /usr/lib/libc.so.6

(gdb) bt 13
#0  0x00007ffff0cc01a7 in __memset_sse2_unaligned_erms () from /usr/lib/libc.so.6
#1  0x00000000006d27f5 in r_alloc_sbrk (size=290816) at ralloc.c:848
#2  0x00000000006ced96 in get_contiguous_space (size=290816, position=0x1833000) at gmalloc.c:476
#3  0x00000000006cf92a in _malloc_internal_nolock (size=163840) at gmalloc.c:844
#4  0x00000000006cfe9d in _malloc_internal (size=163840) at gmalloc.c:927
#5  0x00000000006cff1a in gmalloc (size=163840) at gmalloc.c:951
#6  0x00000000006d14e4 in malloc (size=163840) at gmalloc.c:1827
#7  0x00000000005f3e6b in lmalloc (size=163840) at alloc.c:1414
#8  0x00000000005f3356 in xmalloc (size=163840) at alloc.c:821
#9  0x00000000005f38e4 in record_xmalloc (size=163840) at alloc.c:1038
#10 0x00000000005ee233 in re_match_2_internal (bufp=0xd6d650 <searchbufs+5072>, string1=0x1835980 "", 
    size1=0, string2=0x1835980 "", size2=40918, pos=0, regs=0xd6deb0 <search_regs>, stop=40918)
    at regex.c:5844



[-- Attachment #2: changes to hunt down bug 24358 --]
[-- Type: text/plain, Size: 3199 bytes --]

diff --git i/src/.gdbinit w/src/.gdbinit
index a4e9f70..d17d1ba 100644
--- i/src/.gdbinit
+++ w/src/.gdbinit
@@ -1280,3 +1280,22 @@ commands
   end
   continue
 end
+
+# bug 24315
+break re_match_2_internal if (size2>2000 && size2==stop)
+commands
+  p debug = 1
+  continue
+end
+break debug_spot
+commands
+  watch -l string1[0]
+  disable 4
+  # cond 4 (string1[0] != 'D')
+  # continue
+end
+# break debug_malloc if ((mem <= 0x1834980) && (0x1834980 < mem + size))
+
+
+
+
diff --git i/src/gmalloc.c w/src/gmalloc.c
index 00b8364..5084609 100644
--- i/src/gmalloc.c
+++ w/src/gmalloc.c
@@ -914,6 +914,10 @@ _malloc_internal_nolock (size_t size)
   return result;
 }
 
+void debug_malloc (void* mem, size_t size)
+{
+}
+
 void *
 _malloc_internal (size_t size)
 {
@@ -923,6 +927,7 @@ _malloc_internal (size_t size)
   result = _malloc_internal_nolock (size);
   UNLOCK ();
 
+  debug_malloc (result, size);
   return result;
 }
 
diff --git i/src/regex.c w/src/regex.c
index 164eb46..861b800 100644
--- i/src/regex.c
+++ w/src/regex.c
@@ -828,6 +828,7 @@ extract_number_and_incr (re_char **source)
    interactively.  And if linked with the main program in `main.c' and
    the other test files, you can run the already-written tests.  */
 
+#define DEBUG
 #ifdef DEBUG
 
 /* We use standard I/O for debugging.  */
@@ -838,6 +839,13 @@ extract_number_and_incr (re_char **source)
 
 static int debug = -100000;
 
+static void debug_spot (int fail_stack_avail, const char*string1, const char*string2)
+{
+  extern void r_alloc_check (void);
+  //r_alloc_check ();
+  fail_stack_avail++;
+}
+
 # define DEBUG_STATEMENT(e) e
 # define DEBUG_PRINT(...) if (debug > 0) printf (__VA_ARGS__)
 # define DEBUG_COMPILES_ARGUMENTS
@@ -1172,16 +1180,31 @@ print_double_string (re_char *where, re_char *string1, ssize_t size1,
     printf ("(null)");
   else
     {
+      int i;
       if (FIRST_STRING_P (where))
 	{
-	  for (this_char = where - string1; this_char < size1; this_char++)
-	    putchar (string1[this_char]);
+	  for (i = 0, this_char = where - string1; this_char < size1; i++, this_char++)
+            {
+              if (i > 20)
+                {
+                  putchar ('.'); putchar ('.'); putchar ('.');
+                  break;
+                }
+              putchar (string1[this_char]);
+            }
 
 	  where = string2;
 	}
 
-      for (this_char = where - string2; this_char < size2; this_char++)
-	putchar (string2[this_char]);
+      for (i = 0, this_char = where - string2; this_char < size2; i++, this_char++)
+        {
+          if (i > 20)
+            {
+              putchar ('.'); putchar ('.'); putchar ('.');
+              break;
+            }
+          putchar (string2[this_char]);
+        }
     }
 }
 
@@ -1533,6 +1556,7 @@ while (REMAINING_AVAIL_SLOTS <= space) {				\
      of 0 + -1 isn't done as unsigned.  */				\
   									\
   DEBUG_STATEMENT (nfailure_points_pushed++);				\
+  if (debug > 0) debug_spot((fail_stack).avail, string1,string2);              \
   DEBUG_PRINT ("\nPUSH_FAILURE_POINT:\n");				\
   DEBUG_PRINT ("  Before push, next avail: %zd\n", (fail_stack).avail);	\
   DEBUG_PRINT ("			size: %zd\n", (fail_stack).size);\

[-- Attachment #3: gdb session excerpts --]
[-- Type: text/plain, Size: 7321 bytes --]

The compiled pattern is: The string to match is: "DESCRIPTION;LANGUAGE=..."

0x144aa80: EXECUTING on_failure_jump_smart 4 (to 0x144aa87).
  smart default => slow loop.

0x144aa80: EXECUTING on_failure_jump 4 (to 0x144aa87):

Thread 1 "emacs" hit Breakpoint 4, debug_spot (fail_stack_avail=0, 
    string1=0x1835980 "DESCRIPTION;LANGUAGE=en-US:Nn Nnnnn\\,\\n\\nNnnnnnnnn nnn nnn nnnnnn nn nnnnnn\n nnnnnnn nnnn nnnnnnnnn. N nnnn nnnnnnnnn nn nnn nnnnnnnn nnnnnnn nnn nn nn\n nn nn-nnnnnnn nn Nnnnnnn nn 99.99 NNNN\\n\\nNnnn "..., 
    string2=0x1835980 "DESCRIPTION;LANGUAGE=en-US:Nn Nnnnn\\,\\n\\nNnnnnnnnn nnn nnn nnnnnn nn nnnnnn\n nnnnnnn nnnn nnnnnnnnn. N nnnn nnnnnnnnn nn nnn nnnnnnnn nnnnnnn nnn nn nn\n nn nn-nnnnnnn nn Nnnnnnn nn 99.99 NNNN\\n\\nNnnn "...) at regex.c:846
846	  fail_stack_avail++;
Hardware watchpoint 5: -location string1[0]
(gdb) bt 5
#0  debug_spot (fail_stack_avail=0, 
    string1=0x1835980 "DESCRIPTION;LANGUAGE=en-US:Nn Nnnnn\\,\\n\\nNnnnnnnnn nnn nnn nnnnnn nn nnnnnn\n nnnnnnn nnnn nnnnnnnnn. N nnnn nnnnnnnnn nn nnn nnnnnnnn nnnnnnn nnn nn nn\n nn nn-nnnnnnn nn Nnnnnnn nn 99.99 NNNN\\n\\nNnnn "..., 
    string2=0x1835980 "DESCRIPTION;LANGUAGE=en-US:Nn Nnnnn\\,\\n\\nNnnnnnnnn nnn nnn nnnnnn nn nnnnnn\n nnnnnnn nnnn nnnnnnnnn. N nnnn nnnnnnnnn nn nnn nnnnnnnn nnnnnnn nnn nn nn\n nn nn-nnnnnnn nn Nnnnnnn nn 99.99 NNNN\\n\\nNnnn "...) at regex.c:846
#1  0x00000000005ee090 in re_match_2_internal (bufp=0xd6d650 <searchbufs+5072>, 
    string1=0x1835980 "DESCRIPTION;LANGUAGE=en-US:Nn Nnnnn\\,\\n\\nNnnnnnnnn nnn nnn nnnnnn nn nnnnnn\n nnnnnnn nnnn nnnnnnnnn. N nnnn nnnnnnnnn nn nnn nnnnnnnn nnnnnnn nnn nn nn\n nn nn-nnnnnnn nn Nnnnnnn nn 99.99 NNNN\\n\\nNnnn "..., size1=0, 
    string2=0x1835980 "DESCRIPTION;LANGUAGE=en-US:Nn Nnnnn\\,\\n\\nNnnnnnnnn nnn nnn nnnnnn nn nnnnnn\n nnnnnnn nnnn nnnnnnnnn. N nnnn nnnnnnnnn nn nnn nnnnnnnn nnnnnnn nnn nn nn\n nn nn-nnnnnnn nn Nnnnnnn nn 99.99 NNNN\\n\\nNnnn "..., size2=40918, pos=0, regs=0xd6deb0 <search_regs>, stop=40918)
    at regex.c:5844
#2  0x00000000005e9022 in re_search_2 (bufp=0xd6d650 <searchbufs+5072>, 
    str1=0x1835980 "DESCRIPTION;LANGUAGE=en-US:Nn Nnnnn\\,\\n\\nNnnnnnnnn nnn nnn nnnnnn nn nnnnnn\n nnnnnnn nnnn nnnnnnnnn. N nnnn nnnnnnnnn nn nnn nnnnnnnn nnnnnnn nnn nn nn\n nn nn-nnnnnnn nn Nnnnnnn nn 99.99 NNNN\\n\\nNnnn "..., size1=0, 
    str2=0x1835980 "DESCRIPTION;LANGUAGE=en-US:Nn Nnnnn\\,\\n\\nNnnnnnnnn nnn nnn nnnnnn nn nnnnnn\n nnnnnnn nnnn nnnnnnnnn. N nnnn nnnnnnnnn nn nnn nnnnnnnn nnnnnnn nnn nn nn\n nn nn-nnnnnnn nn Nnnnnnn nn 99.99 NNNN\\n\\nNnnn "..., size2=40918, startpos=0, range=40918, regs=0xd6deb0 <search_regs>, 
    stop=40918) at regex.c:4470
#3  0x00000000005d6c06 in search_buffer (string=25301860, pos=1, pos_byte=1, lim=40891, 
    lim_byte=40919, n=1, RE=1, trt=20893029, inverse_trt=20483397, posix=false) at search.c:1265
#4  0x00000000005d63a1 in search_command (string=25301860, bound=0, noerror=44544, count=0, 
    direction=1, RE=1, posix=false) at search.c:1058
(More stack frames follow...)


(gdb) cont
[...]

PUSH_FAILURE_POINT:
  Before push, next avail: 5115
			size: 5120

  Push frame index: 5115
  Push string 0x1836013: ".nnn>\;\n> +NNNN <NNN..."
0:	/on_failure_jump to 7
3:	/anychar
4:	/jump to 0
7:	/stop_memory/1
9:	/jump to -8
12:	/succeed
13:	end of pattern.
  Push pattern 0x144aa8f: 
0x144aa92: EXECUTING anychar.
  Matched "46".

0x144aa93: EXECUTING jump -7 (to 0x144aa8f).

0x144aa8f: EXECUTING on_failure_jump 4 (to 0x144aa96):

PUSH_FAILURE_POINT:
  Before push, next avail: 5118
			size: 5120

Thread 1 "emacs" hit Hardware watchpoint 5: -location string1[0]

Old value = 68 'D'
New value = 0 '\000'
0x00007ffff0cc01a7 in __memset_sse2_unaligned_erms () from /usr/lib/libc.so.6

(gdb) bt 13
#0  0x00007ffff0cc01a7 in __memset_sse2_unaligned_erms () from /usr/lib/libc.so.6
#1  0x00000000006d27f5 in r_alloc_sbrk (size=290816) at ralloc.c:848
#2  0x00000000006ced96 in get_contiguous_space (size=290816, position=0x1833000) at gmalloc.c:476
#3  0x00000000006cf92a in _malloc_internal_nolock (size=163840) at gmalloc.c:844
#4  0x00000000006cfe9d in _malloc_internal (size=163840) at gmalloc.c:927
#5  0x00000000006cff1a in gmalloc (size=163840) at gmalloc.c:951
#6  0x00000000006d14e4 in malloc (size=163840) at gmalloc.c:1827
#7  0x00000000005f3e6b in lmalloc (size=163840) at alloc.c:1414
#8  0x00000000005f3356 in xmalloc (size=163840) at alloc.c:821
#9  0x00000000005f38e4 in record_xmalloc (size=163840) at alloc.c:1038
#10 0x00000000005ee233 in re_match_2_internal (bufp=0xd6d650 <searchbufs+5072>, string1=0x1835980 "", 
    size1=0, string2=0x1835980 "", size2=40918, pos=0, regs=0xd6deb0 <search_regs>, stop=40918)
    at regex.c:5844
#11 0x00000000005e9022 in re_search_2 (bufp=0xd6d650 <searchbufs+5072>, str1=0x1835980 "", size1=0, 
    str2=0x1835980 "", size2=40918, startpos=0, range=40918, regs=0xd6deb0 <search_regs>, stop=40918)
    at regex.c:4470
#12 0x00000000005d6c06 in search_buffer (string=25301860, pos=1, pos_byte=1, lim=40891, 
    lim_byte=40919, n=1, RE=1, trt=20893029, inverse_trt=20483397, posix=false) at search.c:1265
(More stack frames follow...)

Continuing.

Thread 1 "emacs" hit Hardware watchpoint 5: -location string1[0]

Old value = 0 '\000'
New value = -34 '\336'
0x00007ffff0d67b64 in __memcpy_ssse3 () from /usr/lib/libc.so.6

(gdb) cont
Continuing.

  Doubled stack; size now: 20480
	 slots available: 15362

  Push frame index: 5118
  Push string 0x1836014: "ª$..."

[...]

PUSH_FAILURE_POINT:
  Before push, next avail: 5130
			size: 20480

  Push frame index: 5130
  Push string 0x1836018: "ª$\..."
0:	/on_failure_jump to 7
3:	/anychar
4:	/jump to 0
7:	/stop_memory/1
9:	/jump to -8
12:	/succeed
13:	end of pattern.
  Push pattern 0x144aa8f: 
0x144aa92: EXECUTING anychar.

character.h:696: Emacs fatal error: assertion failed: CHAR_VALID_P (ch)

Thread 1 "emacs" hit Breakpoint 1, terminate_due_to_signal (sig=6, backtrace_limit=2147483647)
    at emacs.c:354
354	  signal (sig, SIG_DFL);

(gdb) bt 7
#0  terminate_due_to_signal (sig=6, backtrace_limit=2147483647) at emacs.c:354
#1  0x00000000005fdb9b in die (msg=0x725888 "CHAR_VALID_P (ch)", file=0x72587c "character.h", 
    line=696) at alloc.c:7224
#2  0x000000000056c000 in char_table_translate (obj=20893029, ch=4195178) at character.h:696
#3  0x00000000005eb8db in re_match_2_internal (bufp=0xd6d650 <searchbufs+5072>, 
    string1=0x1835980 "\336[\203\001", size1=0, string2=0x1835980 "\336[\203\001", size2=40918, 
    pos=0, regs=0xd6deb0 <search_regs>, stop=40918) at regex.c:5454
#4  0x00000000005e9022 in re_search_2 (bufp=0xd6d650 <searchbufs+5072>, 
    str1=0x1835980 "\336[\203\001", size1=0, str2=0x1835980 "\336[\203\001", size2=40918, startpos=0, 
    range=40918, regs=0xd6deb0 <search_regs>, stop=40918) at regex.c:4470
#5  0x00000000005d6c06 in search_buffer (string=25301860, pos=1, pos_byte=1, lim=40891, 
    lim_byte=40919, n=1, RE=1, trt=20893029, inverse_trt=20483397, posix=false) at search.c:1265
#6  0x00000000005d63a1 in search_command (string=25301860, bound=0, noerror=44544, count=0, 
    direction=1, RE=1, posix=false) at search.c:1058
(More stack frames follow...)