bug#33174: 27.0.50; Dump fails on GNU/Linux ppc64le

[Paul Eggert <eggert@cs.ucla.edu>]

[-- Attachment #1: Type: text/plain, Size: 472 bytes --]

Thomas Fitzsimmons wrote:
> Actually, EMACS_HEAP_EXEC is true!

OK, thanks, that explains things. I installed the first attached patch to the 
emacs-26 branch as a quick hack to work around the problem, and the second 
attached patch to the master branch to come up with a better way to address the 
underlying issue that doesn't involve fiddling with environment variables. 
Please let us know if this doesn't work for your platform, as I've tested this 
only on x86-64.

[-- Attachment #2: 0001-Work-around-dumping-bug-on-GNU-Linux-ppc64le.patch --]
[-- Type: text/x-patch, Size: 1128 bytes --]

From 1d79c2ebd9bd9aa36586e57463502373c0296d11 Mon Sep 17 00:00:00 2001
From: Paul Eggert <eggert@cs.ucla.edu>
Date: Sun, 11 Nov 2018 22:34:46 -0800
Subject: [PATCH] Work around dumping bug on GNU/Linux ppc64le

Problem reported by Thomas Fitzsimmons (Bug#33174).
Do not merge to master, as we have a better fix there.
* src/Makefile.in (emacs$(EXEEXT)):
(bootstrap-emacs$(EXEEXT)):
Unset EMACS_HEAP_EXEC before invoking temacs.
---
 src/Makefile.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Makefile.in b/src/Makefile.in
index 6ed8f3cc91..53c18e7ac0 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -534,6 +534,7 @@ emacs$(EXEEXT):
 ifeq ($(CANNOT_DUMP),yes)
 	ln -f temacs$(EXEEXT) $@
 else
+	unset EMACS_HEAP_EXEC; \
 	LC_ALL=C $(RUN_TEMACS) -batch $(BUILD_DETAILS) -l loadup dump
   ifneq ($(PAXCTL_dumped),)
 	$(PAXCTL_dumped) $@
@@ -739,6 +740,7 @@ bootstrap-emacs$(EXEEXT):
 ifeq ($(CANNOT_DUMP),yes)
 	ln -f temacs$(EXEEXT) $@
 else
+	unset EMACS_HEAP_EXEC; \
 	$(RUN_TEMACS) --batch $(BUILD_DETAILS) --load loadup bootstrap
   ifneq ($(PAXCTL_dumped),)
 	$(PAXCTL_dumped) emacs$(EXEEXT)
-- 
2.17.1


[-- Attachment #3: 0001-Fix-dumping-on-GNU-Linux-ppc64le.patch --]
[-- Type: text/x-patch, Size: 5574 bytes --]

From c14eab222c5208ec0650292c3771a3ee632fdb0d Mon Sep 17 00:00:00 2001
From: Paul Eggert <eggert@cs.ucla.edu>
Date: Sun, 11 Nov 2018 22:18:47 -0800
Subject: [PATCH] Fix dumping on GNU/Linux ppc64le

Problem reported by Thomas Fitzsimmons (Bug#33174).
* src/emacs.c (main): Adjust to sysdep.c changes.
* src/sysdep.c (exec_personality): New static var.
(disable_address_randomization): Remove, replacing with ...
(maybe_disable_address_randomization): ... this new function.
Do not set or use an environment variable; use a command-line
argument instead, and set the new static var.  Migrate the emacs.c
personality-change code to here, where it belongs.
(emacs_exec_file): Simplify by using new static var.
---
 src/emacs.c  | 23 +----------------
 src/lisp.h   |  7 +++--
 src/sysdep.c | 73 +++++++++++++++++++++++++++++++++-------------------
 3 files changed, 53 insertions(+), 50 deletions(-)

diff --git a/src/emacs.c b/src/emacs.c
index 07df191035..512174d562 100644
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -706,28 +706,7 @@ main (int argc, char **argv)
   dumping = false;
 #endif
 
-  /* True if address randomization interferes with memory allocation.  */
-# ifdef __PPC64__
-  bool disable_aslr = true;
-# else
-  bool disable_aslr = dumping;
-# endif
-
-  if (disable_aslr && disable_address_randomization ()
-      && !getenv ("EMACS_HEAP_EXEC"))
-    {
-      /* Set this so the personality will be reverted before execs
-	 after this one, and to work around an re-exec loop on buggy
-	 kernels (Bug#32083).  */
-      xputenv ("EMACS_HEAP_EXEC=true");
-
-      /* Address randomization was enabled, but is now disabled.
-	 Re-execute Emacs to get a clean slate.  */
-      execvp (argv[0], argv);
-
-      /* If the exec fails, warn and then try anyway.  */
-      perror (argv[0]);
-    }
+  argc = maybe_disable_address_randomization (dumping, argc, argv);
 
 #ifndef CANNOT_DUMP
   might_dump = !initialized;
diff --git a/src/lisp.h b/src/lisp.h
index eb6762678c..383d61274c 100644
--- a/src/lisp.h
+++ b/src/lisp.h
@@ -4325,9 +4325,12 @@ struct tty_display_info;
 
 /* Defined in sysdep.c.  */
 #ifdef HAVE_PERSONALITY_ADDR_NO_RANDOMIZE
-extern bool disable_address_randomization (void);
+extern int maybe_disable_address_randomization (bool, int, char **);
 #else
-INLINE bool disable_address_randomization (void) { return false; }
+INLINE void
+maybe_disable_address_randomization (bool dumping, int argc, char **argv)
+{
+}
 #endif
 extern int emacs_exec_file (char const *, char *const *, char *const *);
 extern void init_standard_fds (void);
diff --git a/src/sysdep.c b/src/sysdep.c
index 7a0c8a8ab8..ddcb594f66 100644
--- a/src/sysdep.c
+++ b/src/sysdep.c
@@ -150,22 +150,52 @@ static const int baud_convert[] =
 #ifdef HAVE_PERSONALITY_ADDR_NO_RANDOMIZE
 # include <sys/personality.h>
 
-/* Disable address randomization in the current process.  Return true
-   if addresses were randomized but this has been disabled, false
-   otherwise. */
-bool
-disable_address_randomization (void)
+/* If not -1, the personality that should be restored before exec.  */
+static int exec_personality;
+
+/* Try to disable randomization if the current process needs it and
+   does not appear to have it already.  */
+int
+maybe_disable_address_randomization (bool dumping, int argc, char **argv)
 {
-  int pers = personality (0xffffffff);
-  if (pers < 0)
-    return false;
-  int desired_pers = pers | ADDR_NO_RANDOMIZE;
+  /* Undocumented Emacs option used only by this function.  */
+  static char const aslr_disabled_option[] = "--__aslr-disabled";
 
-  /* Call 'personality' twice, to detect buggy platforms like WSL
-     where 'personality' always returns 0.  */
-  return (pers != desired_pers
-	  && personality (desired_pers) == pers
-	  && personality (0xffffffff) == desired_pers);
+  if (argc < 2 || strcmp (argv[1], aslr_disabled_option) != 0)
+    {
+      bool disable_aslr = dumping;
+# ifdef __PPC64__
+      disable_aslr = true;
+# endif
+      exec_personality = disable_aslr ? personality (0xffffffff) : -1;
+      if (exec_personality & ADDR_NO_RANDOMIZE)
+	exec_personality = -1;
+      if (exec_personality != -1
+	  && personality (exec_personality | ADDR_NO_RANDOMIZE) != -1)
+	{
+	  char **newargv = malloc ((argc + 2) * sizeof *newargv);
+	  if (newargv)
+	    {
+	      /* Invoke self with undocumented option.  */
+	      newargv[0] = argv[0];
+	      newargv[1] = (char *) aslr_disabled_option;
+	      memcpy (&newargv[2], &argv[1], argc * sizeof *newargv);
+	      execvp (newargv[0], newargv);
+	    }
+
+	  /* If malloc or execvp fails, warn and then try anyway.  */
+	  perror (argv[0]);
+	  free (newargv);
+	}
+    }
+  else
+    {
+      /* Our earlier incarnation already disabled ASLR.  */
+      argc--;
+      memmove (&argv[1], &argv[2], argc * sizeof *argv);
+    }
+
+  return argc;
 }
 #endif
 
@@ -177,21 +207,12 @@ int
 emacs_exec_file (char const *file, char *const *argv, char *const *envp)
 {
 #ifdef HAVE_PERSONALITY_ADDR_NO_RANDOMIZE
-  int pers = getenv ("EMACS_HEAP_EXEC") ? personality (0xffffffff) : -1;
-  bool change_personality = 0 <= pers && pers & ADDR_NO_RANDOMIZE;
-  if (change_personality)
-    personality (pers & ~ADDR_NO_RANDOMIZE);
+  if (exec_personality != -1)
+    personality (exec_personality);
 #endif
 
   execve (file, argv, envp);
-  int err = errno;
-
-#ifdef HAVE_PERSONALITY_ADDR_NO_RANDOMIZE
-  if (change_personality)
-    personality (pers);
-#endif
-
-  return err;
+  return errno;
 }
 
 /* If FD is not already open, arrange for it to be open with FLAGS.  */
-- 
2.17.1