bug#41242: Port feature/native-comp to Windows - Determine the emacs root dir...

["Nicolas Bértolo" <nicolasbertolo@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 1188 bytes --]

I have found a few bugs in Windows.
Patches attached (they are in my repo too).

- It is still a good idea to avoid the call to getenv("emacs_dir"), if only for
  performance reasons.

- We can't use "gensym" for register_native_comp_unit() because the dynamic
  library may not have been loaded yet when loading a dump file.

- Commit `f5dceed09a8234548d5b3acb76d443569533cab9` "* lisp/loadup.el: Use new
  'native-comp-available-p'." causes load_gccjit_if_necessary() to be called in
  temacs. This didn't work because because term/w32-win.el had not been loaded
  yet. In particular, we need `dynamic-library-alist` to be defined to know the
  name of the libgccjit DLL. I have defined this in syms_of_emacs(). This
  definition should be active only while dumping.

- This last bug is kinda confusing. I'm not sure about my diagnosis. The list
  `delayed_comp_unit_disposal_list` has nodes allocated with xmalloc(). It seems
  that these blocks allocated with xmalloc() get GC'd or they get corrupted
  somehow and thus they don't survive until Emacs is about to close, which is
  when we need the list. I solved it by allocating the data and nodes with
  HeapAlloc().

Thanks, Nico.

[-- Attachment #2: 0001-Determine-the-emacs-root-dir-only-when-necessary.patch --]
[-- Type: application/octet-stream, Size: 4650 bytes --]

From 42e5d34f9f17ec85da376fb6c18569ba1cc353d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=A1s=20B=C3=A9rtolo?= <nicolasbertolo@gmail.com>
Date: Mon, 25 May 2020 17:55:23 -0300
Subject: [PATCH 1/4] Determine the emacs root dir only when necessary.

Fexpand_file_name() tries to obtain the root directory even when it
is not necessary because `default-directory' is not nil.

Avoid a call to getenv () unless it's necessary.

* src/fileio.c: Introduce function emacs_root_dir(). Refactor
`expand-file-name` to use it.
* src/lisp.h: Separate emacs_root_dir() into dos_emacs_root_dir() and
w32_emacs_root_dir().
* src/msdos.c: Rename emacs_root_dir() to dos_emacs_root_dir().
* src/w32.c: Rename emacs_root_dir() to w32_emacs_root_dir().
---
 src/fileio.c | 45 ++++++++++++++++++++++++++-------------------
 src/lisp.h   | 11 +++++++----
 src/msdos.c  |  2 +-
 src/w32.c    |  2 +-
 4 files changed, 35 insertions(+), 25 deletions(-)

diff --git a/src/fileio.c b/src/fileio.c
index 2f1d2f8243..11374ac56d 100644
--- a/src/fileio.c
+++ b/src/fileio.c
@@ -781,6 +781,30 @@ user_homedir (char const *name)
   return pw->pw_dir;
 }
 
+/* As a last resort, we may have to use the root as
+   default_directory in `expand-file-name'.
+
+   "/" is not considered a root directory on DOS_NT, so using it
+   as default_directory causes an infinite recursion in, e.g.,
+   the following:
+
+   (let (default-directory)
+          (expand-file-name "a"))
+
+   To avoid this, we use the root of the current drive. */
+
+static Lisp_Object
+emacs_root_dir (void)
+{
+#ifdef DOS
+  return build_string (dos_emacs_root_dir ());
+#elif defined (WINDOWSNT)
+  return build_string (w32_emacs_root_dir ());
+#else
+  return build_string ("/");
+#endif
+}
+
 DEFUN ("expand-file-name", Fexpand_file_name, Sexpand_file_name, 1, 2, 0,
        doc: /* Convert filename NAME to absolute, and canonicalize it.
 Second arg DEFAULT-DIRECTORY is directory to start with if NAME is relative
@@ -850,23 +874,6 @@ DEFUN ("expand-file-name", Fexpand_file_name, Sexpand_file_name, 1, 2, 0,
       error ("Invalid handler in `file-name-handler-alist'");
     }
 
-  /* As a last resort, we may have to use the root as
-     default_directory below.  */
-  Lisp_Object root;
-#ifdef DOS_NT
-      /* "/" is not considered a root directory on DOS_NT, so using it
-	 as default_directory causes an infinite recursion in, e.g.,
-	 the following:
-
-            (let (default-directory)
-	      (expand-file-name "a"))
-
-	 To avoid this, we use the root of the current drive.  */
-      root = build_string (emacs_root_dir ());
-#else
-      root = build_string ("/");
-#endif
-
   /* Use the buffer's default-directory if DEFAULT_DIRECTORY is omitted.  */
   if (NILP (default_directory))
     {
@@ -891,13 +898,13 @@ DEFUN ("expand-file-name", Fexpand_file_name, Sexpand_file_name, 1, 2, 0,
 	      Lisp_Object absdir
 		= STRINGP (Vinvocation_directory)
 		&& file_name_absolute_no_tilde_p (Vinvocation_directory)
-		? Vinvocation_directory : root;
+		? Vinvocation_directory : emacs_root_dir ();
 	      default_directory = Fexpand_file_name (dir, absdir);
 	    }
 	}
     }
   if (! STRINGP (default_directory))
-    default_directory = root;
+    default_directory = emacs_root_dir ();
 
   handler = Ffind_file_name_handler (default_directory, Qexpand_file_name);
   if (!NILP (handler))
diff --git a/src/lisp.h b/src/lisp.h
index 23b105e550..9704e78cf4 100644
--- a/src/lisp.h
+++ b/src/lisp.h
@@ -4746,10 +4746,13 @@ maybe_disable_address_randomization (int argc, char **argv)
 extern void malloc_probe (size_t);
 extern void syms_of_profiler (void);
 
-#ifdef DOS_NT
-/* Defined in msdos.c, w32.c.  */
-extern char *emacs_root_dir (void);
-#endif /* DOS_NT */
+#ifdef MSDOS
+/* Defined in msdos.c.  */
+extern char *dos_emacs_root_dir (void);
+#elif defined (WINDOWSNT)
+/* Defined in w32.c.  */
+extern char *w32_emacs_root_dir (void);
+#endif /* MSDOS */
 
 #ifdef HAVE_NATIVE_COMP
 INLINE bool
diff --git a/src/msdos.c b/src/msdos.c
index b5f06c99c3..0827cc96cd 100644
--- a/src/msdos.c
+++ b/src/msdos.c
@@ -3350,7 +3350,7 @@ getdefdir (int drive, char *dst)
 }
 
 char *
-emacs_root_dir (void)
+dos_emacs_root_dir (void)
 {
   static char root_dir[4];
 
diff --git a/src/w32.c b/src/w32.c
index 3e71d0d383..bce8371164 100644
--- a/src/w32.c
+++ b/src/w32.c
@@ -3147,7 +3147,7 @@ #define SET_ENV_BUF_SIZE (4 * MAX_PATH)	/* to cover EMACSLOADPATH */
 /* Called from expand-file-name when default-directory is not a string.  */
 
 char *
-emacs_root_dir (void)
+w32_emacs_root_dir (void)
 {
   static char root_dir[MAX_UTF8_PATH];
   const char *p;
-- 
2.25.1.windows.1


[-- Attachment #3: 0002-Do-not-call-gensym-too-early-when-loading-a-dump-fil.patch --]
[-- Type: application/octet-stream, Size: 1354 bytes --]

From fc2b770805d064d3369ddbee0468ffb85b3c3a72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=A1s=20B=C3=A9rtolo?= <nicolasbertolo@gmail.com>
Date: Fri, 29 May 2020 21:03:00 -0300
Subject: [PATCH 2/4] Do not call `gensym' too early when loading a dump file.

This happened when subr.eln was not the first native compilation unit
to be loaded. register_native_comp_unit() is called when loading a
native compilation unit and that in turn used to call `gensym', which
was not loaded yet. This led to a SIGSEGV.

* src/comp.c (register_native_comp_unit): Replace the call to `gensym'
with an ad-hoc counter.
---
 src/comp.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/comp.c b/src/comp.c
index 32a98173d5..9698393f70 100644
--- a/src/comp.c
+++ b/src/comp.c
@@ -4120,7 +4120,15 @@ finish_delayed_disposal_of_comp_units (void)
 register_native_comp_unit (Lisp_Object comp_u)
 {
 #ifdef WINDOWSNT
-  Fputhash (CALL1I (gensym, Qnil), comp_u, all_loaded_comp_units_h);
+  /* We have to do this since we can't use `gensym'. This function is
+     called early when loading a dump file and subr.el may not have
+     been loaded yet. */
+  static EMACS_UINT count;
+
+  if (count >= MOST_POSITIVE_FIXNUM)
+    return;
+
+  Fputhash(make_fixnum(count++), comp_u, all_loaded_comp_units_h);
 #endif
 }
 
-- 
2.25.1.windows.1


[-- Attachment #4: 0003-Fix-loading-of-libgccjit.dll-while-dumping-in-Window.patch --]
[-- Type: application/octet-stream, Size: 1544 bytes --]

From 314fc82ae3a5050ed1b6fd06bb057f26b740ee2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=A1s=20B=C3=A9rtolo?= <nicolasbertolo@gmail.com>
Date: Fri, 29 May 2020 21:08:37 -0300
Subject: [PATCH 3/4] Fix loading of libgccjit.dll while dumping in Windows.

loadup.el calls `native-comp-available-p', that calls
load_gccjit_if_necessary() in Windows. That function tries to load
libgccjit using the mappings defined in `dynamic-library-alist'. That
mapping is filled by term/w32-win.el, but that file may be loaded too
late.

* src/emacs.c (syms_of_emacs): Add libgccjit to the
`dynamic-library-alist' used when starting to dump so
`native-comp-available-p' always works in Windows.
---
 src/emacs.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/emacs.c b/src/emacs.c
index 35ae9d0e66..90035bc3a3 100644
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -3053,7 +3053,15 @@ syms_of_emacs (void)
 
 Also note that this is not a generic facility for accessing external
 libraries; only those already known by Emacs will be loaded.  */);
+#ifdef WINDOWSNT
+  /* We may need to load libgccjit when dumping before term/w32-win.el
+     defines `dynamic-library-alist`. This will fail if that variable
+     is empty, so add libgccjit.dll to it. */
+  Vdynamic_library_alist = list1(list2(Qgccjit,
+                                       build_string("libgccjit.dll")));
+#else
   Vdynamic_library_alist = Qnil;
+#endif
   Fput (intern_c_string ("dynamic-library-alist"), Qrisky_local_variable, Qt);
 
 #ifdef WINDOWSNT
-- 
2.25.1.windows.1


[-- Attachment #5: 0004-Use-Windows-allocation-APIs-for-delayed_comp_unit_di.patch --]
[-- Type: application/octet-stream, Size: 3187 bytes --]

From ee82ba57fb5397719296d882fc7b0bf5131e19b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=A1s=20B=C3=A9rtolo?= <nicolasbertolo@gmail.com>
Date: Fri, 29 May 2020 21:25:55 -0300
Subject: [PATCH 4/4] Use Windows allocation APIs for
 delayed_comp_unit_disposal_list.

That list and associated information need to be available right until
Emacs closes. This is not realiable when using xmalloc().

* src/comp.c: Use HeapFree and HeapAlloc instead of xfree() and
xmalloc() when allocating nodes of delayed_comp_unit_disposal_list and
the "cfile" member of Lisp_Native_Comp_Unit.
* src/pdumper.c (dump_do_dump_relocation): Use HeapAlloc for
allocating the "cfile" member of Lisp_Native_Comp_Unit.
---
 src/comp.c    | 13 ++++++++-----
 src/pdumper.c |  4 +++-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/comp.c b/src/comp.c
index 9698393f70..996168bbb5 100644
--- a/src/comp.c
+++ b/src/comp.c
@@ -4108,8 +4108,8 @@ finish_delayed_disposal_of_comp_units (void)
       Lisp_Object dirname = internal_condition_case_1 (
           Ffile_name_directory, build_string (item->filename), Qt, return_nil);
       clean_comp_unit_directory (dirname);
-      xfree (item->filename);
-      xfree (item);
+      HeapFree (GetProcessHeap (), 0, item->filename);
+      HeapFree (GetProcessHeap (), 0, item);
     }
 }
 #endif
@@ -4152,13 +4152,15 @@ dispose_comp_unit (struct Lisp_Native_Comp_Unit *comp_handle, bool delay)
           return_nil);
       if (!NILP (dirname))
         clean_comp_unit_directory (dirname);
-      xfree (comp_handle->cfile);
+      HeapFree (GetProcessHeap (), 0, comp_handle->cfile);
       comp_handle->cfile = NULL;
     }
   else
     {
       struct delayed_comp_unit_disposal *head;
-      head = xmalloc (sizeof (struct delayed_comp_unit_disposal));
+      head = HeapAlloc (GetProcessHeap (),
+                        0,
+                        sizeof (struct delayed_comp_unit_disposal));
       head->next = delayed_comp_unit_disposal_list;
       head->filename = comp_handle->cfile;
       comp_handle->cfile = NULL;
@@ -4552,7 +4554,8 @@ DEFUN ("native-elisp-load", Fnative_elisp_load, Snative_elisp_load, 1, 2, 0,
     xsignal2 (Qnative_lisp_load_failed, file, build_string (dynlib_error ()));
   comp_u->file = file;
 #ifdef WINDOWSNT
-  comp_u->cfile = xlispstrdup (file);
+  comp_u->cfile = HeapAlloc (GetProcessHeap (), 0, 1 + SBYTES (file));
+  strcpy (comp_u->cfile, SSDATA (file));
 #endif
   comp_u->data_vec = Qnil;
   comp_u->lambda_gc_guard = CALLN (Fmake_hash_table, QCtest, Qeq);
diff --git a/src/pdumper.c b/src/pdumper.c
index 29e3560ee5..68a7a9a536 100644
--- a/src/pdumper.c
+++ b/src/pdumper.c
@@ -5321,7 +5321,9 @@ dump_do_dump_relocation (const uintptr_t dump_base,
 		   installation_state == INSTALLED
 		   ? XCAR (comp_u->file) : XCDR (comp_u->file));
 #ifdef WINDOWSNT
-	comp_u->cfile = xlispstrdup (comp_u->file);
+        comp_u->cfile = HeapAlloc (GetProcessHeap (), 0,
+                                   1 + SBYTES (comp_u->file));
+        strcpy (comp_u->cfile, SSDATA (comp_u->file));
 #endif
 	comp_u->handle = dynlib_open (SSDATA (comp_u->file));
 	if (!comp_u->handle)
-- 
2.25.1.windows.1