bug#41646: Startup in Windows is very slow when load-path contains many

[Lin Sun <sunlin7.mail@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 1268 bytes --]

Hi Eli,

I wrote the code and tested it, the patch file attached.
In the patch, a new "load-hints" is introduced with a default value nil.
When its value is nil, everything works as before.
When its value is set as a list of hints, the openp function calling
count is reduced a lot.
I had an example for require "org", the "load-hints" can reduce the
access attempts from 123 to 6, here is the test command lines, strace
the emacs in batch mode, a single (require 'org) trigger more than
100+ open attempts; with correct load-hints, the count reduce to 6.
> strace src/emacs -batch -eval "(require 'org)" 2>&1| grep 'open.*/org\.' | wc -l
>    123
> strace src/emacs -batch -eval "(let ((load-hints '((\"org*\" \"~/tmp/emacs.debug/lisp/org/\"))))(require 'org))" 2>&1| grep 'open.*/org\.' | wc -l
>      6

And you had mentioned scan the load-path to build a full hints list,
it absolutely should work; I'm looking to change package.el to
generate the "<package>-autoloads.el" work with "load-hints",
currently the autoloads.el will add its folder into "load-path",
just change it to add its  path into "load-hints" should work, then we
do NOT need to provide a function to build the "load-hints" for the
packages installed by package.el won't bother the "load-path".

[-- Attachment #2: 0001-New-variable-load-hints-to-speedup-searching-file-fo.patch --]
[-- Type: text/x-patch, Size: 7048 bytes --]

From f194d69445c1ef0e08db29b07570ee3ed7b2ed62 Mon Sep 17 00:00:00 2001
From: Lin Sun <sunlin7@hotmail.com>
Date: Wed, 16 Oct 2024 07:31:59 +0000
Subject: [PATCH] New variable load-hints to speedup searching file for (load)
 function

* lisp/subr.el: (locate-library) support the `load-hints' variable
* src/lread.c: (load) function support the `load-hints' variable
---
 lisp/subr.el |  9 ++---
 src/lread.c  | 93 ++++++++++++++++++++++++++++++++++++++++++----------
 2 files changed, 81 insertions(+), 21 deletions(-)

diff --git a/lisp/subr.el b/lisp/subr.el
index 2eaed682406..3d9599270ed 100644
--- a/lisp/subr.el
+++ b/lisp/subr.el
@@ -3141,10 +3141,11 @@ locate-library
 string.  When run interactively, the argument INTERACTIVE-CALL is t,
 and the file name is displayed in the echo area."
   (interactive (list (read-library-name) nil nil t))
-  (let ((file (locate-file library
-			   (or path load-path)
-			   (append (unless nosuffix (get-load-suffixes))
-				   load-file-rep-suffixes))))
+  (let ((file (locate-file-internal
+               library (or path load-path)
+               (append (unless nosuffix (get-load-suffixes))
+                       load-file-rep-suffixes)
+               nil (unless path load-hints))))
     (if interactive-call
 	(if file
 	    (message "Library is file %s" (abbreviate-file-name file))
diff --git a/src/lread.c b/src/lread.c
index 95c6891c205..6de1c5be5eb 100644
--- a/src/lread.c
+++ b/src/lread.c
@@ -1271,6 +1271,53 @@ close_file_unwind_android_fd (void *ptr)
 
 #endif
 
+static bool
+complete_filename_p (Lisp_Object pathname)
+{
+  const unsigned char *s = SDATA (pathname);
+  return (IS_DIRECTORY_SEP (s[0])
+	  || (SCHARS (pathname) > 2 && IS_DEVICE_SEP (s[1])
+	      && IS_DIRECTORY_SEP (s[2])));
+}
+
+/* search the file in load hints to get a path list */
+static Lisp_Object
+search_load_hints(Lisp_Object load_hints, Lisp_Object file) {
+  Lisp_Object load_path = Qnil;
+  Lisp_Object tail = load_hints;
+  FOR_EACH_TAIL_SAFE (tail)
+    {
+      bool fullmatch = false;
+      ptrdiff_t len = -1;
+      Lisp_Object row = XCAR (tail);
+      Lisp_Object key = XCAR (row);
+      CHECK_STRING (key);
+
+      if (SBYTES (key) - 1 <= SBYTES (file))
+	{
+	  if (SBYTES (key) >= 1
+	      && SDATA (key)[SBYTES (key) - 1] == '*')
+	    len = SBYTES (key) - 1; /* "file-*" format */
+	  else if (SBYTES (key) == SBYTES (file))
+	    {
+	      len = SBYTES (key);
+	      fullmatch = true;
+	    }
+	}
+
+      if (len >= 0 && 0 == memcmp (SDATA (key), SDATA (file), len))
+	{
+	  if (fullmatch)
+	    {
+	      load_path = CALLN (Fappend, XCDR (row));
+	      break;
+	    }
+	  load_path = CALLN (Fappend, load_path, XCDR (row));
+	}
+    }
+  return load_path;
+}
+
 DEFUN ("load", Fload, Sload, 1, 5, 0,
        doc: /* Execute a file of Lisp code named FILE.
 First try FILE with `.elc' appended, then try with `.el', then try
@@ -1278,7 +1325,9 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
 then try FILE unmodified (the exact suffixes in the exact order are
 determined by `load-suffixes').  Environment variable references in
 FILE are replaced with their values by calling `substitute-in-file-name'.
-This function searches the directories in `load-path'.
+This function searches the entry in `load-hints` first, if some entries
+matched, searches in the matched pathes. Otherwise, searches directories
+in `load-path'.
 
 If optional second arg NOERROR is non-nil,
 report no error if FILE doesn't exist.
@@ -1327,7 +1376,7 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
 #endif
   specpdl_ref fd_index UNINIT;
   specpdl_ref count = SPECPDL_INDEX ();
-  Lisp_Object found, efound, hist_file_name;
+  Lisp_Object found, efound, hist_file_name, load_path = Qnil;
   /* True means we printed the ".el is newer" message.  */
   bool newer = 0;
   /* True means we are loading a compiled file.  */
@@ -1409,12 +1458,18 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
 	    suffixes = CALLN (Fappend, suffixes, Vload_file_rep_suffixes);
 	}
 
+      if (!complete_filename_p (file))
+	load_path = search_load_hints(Vload_hints, file);
+
+      if (NILP (load_path))
+	load_path = Vload_path;
+
 #if !defined USE_ANDROID_ASSETS
-      fd = openp (Vload_path, file, suffixes, &found, Qnil,
+      fd = openp (load_path, file, suffixes, &found, Qnil,
 		  load_prefer_newer, no_native, NULL);
 #else
       asset = NULL;
-      rc = openp (Vload_path, file, suffixes, &found, Qnil,
+      rc = openp (load_path, file, suffixes, &found, Qnil,
 		  load_prefer_newer, no_native, &asset);
       fd.fd = rc;
       fd.asset = asset;
@@ -1780,16 +1835,7 @@ save_match_data_load (Lisp_Object file, Lisp_Object noerror,
   return unbind_to (count, result);
 }
 \f
-static bool
-complete_filename_p (Lisp_Object pathname)
-{
-  const unsigned char *s = SDATA (pathname);
-  return (IS_DIRECTORY_SEP (s[0])
-	  || (SCHARS (pathname) > 2
-	      && IS_DEVICE_SEP (s[1]) && IS_DIRECTORY_SEP (s[2])));
-}
-
-DEFUN ("locate-file-internal", Flocate_file_internal, Slocate_file_internal, 2, 4, 0,
+DEFUN ("locate-file-internal", Flocate_file_internal, Slocate_file_internal, 2, 5, 0,
        doc: /* Search for FILENAME through PATH.
 Returns the file's name in absolute form, or nil if not found.
 If SUFFIXES is non-nil, it should be a list of suffixes to append to
@@ -1797,12 +1843,18 @@ DEFUN ("locate-file-internal", Flocate_file_internal, Slocate_file_internal, 2,
 If non-nil, PREDICATE is used instead of `file-readable-p'.
 PREDICATE can also be an integer to pass to the faccessat(2) function,
 in which case file-name-handlers are ignored.
+LOAD-HINTS is a list same as `load-hints'.
 This function will normally skip directories, so if you want it to find
 directories, make sure the PREDICATE function returns `dir-ok' for them.  */)
-  (Lisp_Object filename, Lisp_Object path, Lisp_Object suffixes, Lisp_Object predicate)
+  (Lisp_Object filename, Lisp_Object path, Lisp_Object suffixes, Lisp_Object predicate,
+   Lisp_Object load_hints)
 {
-  Lisp_Object file;
-  int fd = openp (path, filename, suffixes, &file, predicate, false, true,
+  Lisp_Object file, dirs = Qnil;
+  if (!NILP(load_hints))
+    dirs = search_load_hints(load_hints, filename);
+  if (NILP(dirs))
+    dirs = path;
+  int fd = openp (dirs, filename, suffixes, &file, predicate, false, true,
 		  NULL);
   if (NILP (predicate) && fd >= 0)
     emacs_close (fd);
@@ -5851,6 +5903,13 @@ syms_of_lread (void)
 	       doc: /* Non-nil means read recursive structures using #N= and #N# syntax.  */);
   Vread_circle = Qt;
 
+  DEFVAR_LISP ("load-hints", Vload_hints,
+	       doc: /* A list for name to directory-list to search for files
+to load, before the load-path.  Eache entry is a file name to directory list,
+file name ends with a '*' means prefix matching. Example:
+  '(("name1-*" "/path1" "/path2")).  */);
+  Vload_hints = Qnil;
+
   DEFVAR_LISP ("load-path", Vload_path,
 	       doc: /* List of directories to search for files to load.
 Each element is a string (directory file name) or nil (meaning
-- 
2.34.1