From dc8ff75037f90938f1d3eb9676f674aad868c2da Mon Sep 17 00:00:00 2001
From: Radon Rosborough <radon.neon@gmail.com>
Date: Sun, 17 Sep 2017 22:17:17 -0700
Subject: [PATCH] Add early init file, stop package-initialize insertion

* src/lread.c (Vearly_init_file): New variable, for the filename of
  the early init file that was loaded.

* lisp/startup.el: Load the early init file, if it is found. Move the
  check for an invalid username to just before that, and move the
  initialization of the package system to just after.

* lisp/emacs-lisp/package.el (package--ensure-init-file): Remove. Burn
  with fire. Remove related code and documentation.

* doc/lispref/os.texi: Document early init file.

* doc/lispref/package.texi: Document changes to when
  package-initialize is called, and advise against calling it in the
  init file.

* doc/emacs/package.texi: Document changes to when package-initialize
  is called.

* doc/misc/org.texi: Don't recommend to call package-initialize in the
  init file.

* etc/NEWS: Document changes to startup and package.el.

Discussion on emacs-devel leading up to this change (approximately 150
messages):

- https://lists.gnu.org/archive/html/emacs-devel/2017-08/msg00154.html
- https://lists.gnu.org/archive/html/emacs-devel/2017-08/msg00433.html
- https://lists.gnu.org/archive/html/emacs-devel/2017-09/msg00023.html
---
 doc/emacs/package.texi     |  28 ++-----
 doc/lispref/os.texi        |  14 ++++
 doc/lispref/package.texi   |  16 +++-
 doc/misc/org.texi          |   4 +-
 etc/NEWS                   |  18 ++++
 lisp/emacs-lisp/package.el |  71 +---------------
 lisp/startup.el            | 204 +++++++++++++++++++++++++++++++++------------
 src/lread.c                |   9 ++
 8 files changed, 212 insertions(+), 152 deletions(-)

diff --git a/doc/emacs/package.texi b/doc/emacs/package.texi
index 215f50cb40..bd7cccce6c 100644
--- a/doc/emacs/package.texi
+++ b/doc/emacs/package.texi
@@ -253,30 +253,16 @@ Package Installation
 consult the package's help buffer.
 
   By default, Emacs also automatically loads all installed packages in
-subsequent Emacs sessions.  This happens at startup, after processing
-the init file (@pxref{Init File}).  As an exception, Emacs does not
-load packages at startup if invoked with the @samp{-q} or
-@samp{--no-init-file} options (@pxref{Initial Options}).
+subsequent Emacs sessions.  This happens at startup, before processing
+the init file but after processing the early init file (@pxref{Early
+Init File,,, elisp, The Emacs Lisp Reference Manual}).  As an
+exception, Emacs does not load packages at startup if invoked with the
+@samp{-q} or @samp{--no-init-file} options (@xref{Initial Options}).
 
 @vindex package-enable-at-startup
   To disable automatic package loading, change the variable
-@code{package-enable-at-startup} to @code{nil}.
-
-@findex package-initialize
-  The reason automatic package loading occurs after loading the init
-file is that user options only receive their customized values after
-loading the init file, including user options which affect the
-packaging system.  In some circumstances, you may want to load
-packages explicitly in your init file (usually because some other code
-in your init file depends on a package).  In that case, your init file
-should call the function @code{package-initialize}.  It is up to you
-to ensure that relevant user options, such as @code{package-load-list}
-(see below), are set up prior to the @code{package-initialize} call.
-This will automatically set @code{package-enable-at-startup} to @code{nil}, to
-avoid loading the packages again after processing the init file.
-Alternatively, you may choose to completely inhibit package loading at
-startup, and invoke the command @kbd{M-x package-initialize} to load
-your packages manually.
+@code{package-enable-at-startup} to @code{nil}.  You must do this in
+the early init file.  Currently it cannot be done via Customize.
 
 @vindex package-load-list
   For finer control over package loading, you can use the variable
diff --git a/doc/lispref/os.texi b/doc/lispref/os.texi
index 441fda5d82..062aa28222 100644
--- a/doc/lispref/os.texi
+++ b/doc/lispref/os.texi
@@ -361,6 +361,7 @@ Init File
 @cindex init file
 @cindex @file{.emacs}
 @cindex @file{init.el}
+@cindex @file{early-init.el}
 
   When you start Emacs, it normally attempts to load your @dfn{init
 file}.  This is either a file named @file{.emacs} or @file{.emacs.el}
@@ -384,6 +385,19 @@ Init File
 file.  If those environment variables are absent, though, Emacs uses
 your user-id to find your home directory.
 
+@cindex early init file
+  Emacs also attempts to load a second init file, called the
+  @dfn{early init file}, if it exists.  This is a file named
+  @file{early-init.el} in a subdirectory named @file{.emacs.d} in your
+  home directory.  The difference is that the early init file is
+  loaded much earlier during the startup process, so you can use it to
+  customize some things that are initialized before loading the
+  regular init file.  For example, here you can customize the process
+  of loading installed packages, by setting variables such as
+  @var{package-load-list} or
+  @var{package-enable-at-startup}. @xref{Package Installation,,,
+  emacs,The GNU Emacs Manual}.
+
 @cindex default init file
   An Emacs installation may have a @dfn{default init file}, which is a
 Lisp library named @file{default.el}.  Emacs finds this file through
diff --git a/doc/lispref/package.texi b/doc/lispref/package.texi
index 153ee48741..2e93f29bd2 100644
--- a/doc/lispref/package.texi
+++ b/doc/lispref/package.texi
@@ -106,10 +106,12 @@ Packaging Basics
 
   Whenever Emacs starts up, it automatically calls the function
 @code{package-initialize} to load installed packages.  This is done
-after loading the init file and abbrev file (if any) and before
-running @code{after-init-hook} (@pxref{Startup Summary}).  Automatic
-package loading is disabled if the user option
-@code{package-enable-at-startup} is @code{nil}.
+after loading the early init file, but before loading the early init
+file and abbrev file (if any) and before running
+@code{after-init-hook} (@pxref{Startup Summary}).  Automatic package
+loading is disabled if the user option
+@code{package-enable-at-startup} is @code{nil}.  (Of course, the
+setting of this user option must be done in the early init file.)
 
 @deffn Command package-initialize &optional no-activate
 This function initializes Emacs' internal record of which packages are
@@ -123,6 +125,12 @@ Packaging Basics
 The optional argument @var{no-activate}, if non-@code{nil}, causes
 Emacs to update its record of installed packages without actually
 loading them; it is for internal use only.
+
+In most cases, you should not need to call @code{package-initialize},
+as this is done automatically during startup.  Simply make sure to put
+any code that should run before @code{package-initialize} in the early
+init file, and any code that should run after in the primary init
+file (@xref{Init File,,, emacs, The GNU Emacs Manual}).
 @end deffn
 
 @node Simple Packages
diff --git a/doc/misc/org.texi b/doc/misc/org.texi
index ca57501f3d..c487d43a13 100644
--- a/doc/misc/org.texi
+++ b/doc/misc/org.texi
@@ -891,9 +891,7 @@ Installation
 been visited, i.e., where no Org built-in function have been loaded.
 Otherwise autoload Org functions will mess up the installation.
 
-Then, to make sure your Org configuration is taken into account, initialize
-the package system with @code{(package-initialize)} in your Emacs init file
-before setting any Org option.  If you want to use Org's package repository,
+If you want to use Org's package repository,
 check out the @uref{http://orgmode.org/elpa.html, Org ELPA page}.
 
 @subsubheading Downloading Org as an archive
diff --git a/etc/NEWS b/etc/NEWS
index 371cdf686c..80678551c7 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -28,6 +28,24 @@ When you add a new item, use the appropriate mark if you are sure it applies,
 
 * Startup Changes in Emacs 27.1
 
++++
+** Emacs can now be configured using an early init file.
+The file is called early-init.el, in `user-emacs-directory'.  It is
+loaded very early in the startup process: in particular, before
+graphical elements such as the tool bar are initialized, and before
+the package manager is initialized.
+
++++
+** Emacs now initializes package.el before loading the init-file.
+This is part of a change intended to eliminate the behavior of
+package.el inserting a call to (package-initialize) into the
+init-file, which was previously done when Emacs was started.  Users
+who do not configure package.el variables such as `package-load-list'
+and `package-user-dir' need not make any configuration changes.  Users
+who do configure such variables should place the configuration into
+the newly introduced early init file, which is loaded before
+package.el is initialized.
+
 
 * Changes in Emacs 27.1
 
diff --git a/lisp/emacs-lisp/package.el b/lisp/emacs-lisp/package.el
index 8b101c1323..305453a4ba 100644
--- a/lisp/emacs-lisp/package.el
+++ b/lisp/emacs-lisp/package.el
@@ -1436,16 +1436,11 @@ package-read-all-archive-contents
 ;; available on disk.
 (defvar package--initialized nil)
 
-(defvar package--init-file-ensured nil
-  "Whether we know the init file has package-initialize.")
-
 ;;;###autoload
 (defun package-initialize (&optional no-activate)
   "Load Emacs Lisp packages, and activate them.
 The variable `package-load-list' controls which packages to load.
 If optional arg NO-ACTIVATE is non-nil, don't activate packages.
-If `user-init-file' does not mention `(package-initialize)', add
-it to the file.
 If called as part of loading `user-init-file', set
 `package-enable-at-startup' to nil, to prevent accidentally
 loading packages twice.
@@ -1454,13 +1449,7 @@ package-initialize
 taken care of by `package-initialize'."
   (interactive)
   (setq package-alist nil)
-  (if after-init-time
-      (package--ensure-init-file)
-    ;; If `package-initialize' is before we finished loading the init
-    ;; file, it's obvious we don't need to ensure-init.
-    (setq package--init-file-ensured t
-          ;; And likely we don't need to run it again after init.
-          package-enable-at-startup nil))
+  (setq package-enable-at-startup nil)
   (package-load-all-descriptors)
   (package-read-all-archive-contents)
   (unless no-activate
@@ -1877,64 +1866,6 @@ package-download-transaction
 using `package-compute-transaction'."
   (mapc #'package-install-from-archive packages))
 
-(defun package--ensure-init-file ()
-  "Ensure that the user's init file has `package-initialize'.
-`package-initialize' doesn't have to be called, as long as it is
-present somewhere in the file, even as a comment.  If it is not,
-add a call to it along with some explanatory comments."
-  ;; Don't mess with the init-file from "emacs -Q".
-  (when (and (stringp user-init-file)
-             (not package--init-file-ensured)
-             (file-readable-p user-init-file)
-             (file-writable-p user-init-file))
-    (let* ((buffer (find-buffer-visiting user-init-file))
-           buffer-name
-           (contains-init
-            (if buffer
-                (with-current-buffer buffer
-                  (save-excursion
-                    (save-restriction
-                      (widen)
-                      (goto-char (point-min))
-                      (re-search-forward "(package-initialize\\_>" nil 'noerror))))
-              ;; Don't visit the file if we don't have to.
-              (with-temp-buffer
-                (insert-file-contents user-init-file)
-                (goto-char (point-min))
-                (re-search-forward "(package-initialize\\_>" nil 'noerror)))))
-      (unless contains-init
-        (with-current-buffer (or buffer
-                                 (let ((delay-mode-hooks t)
-                                       (find-file-visit-truename t))
-                                   (find-file-noselect user-init-file)))
-          (when buffer
-            (setq buffer-name (buffer-file-name))
-            (set-visited-file-name (file-chase-links user-init-file)))
-          (save-excursion
-            (save-restriction
-              (widen)
-              (goto-char (point-min))
-              (while (and (looking-at-p "[[:blank:]]*\\(;\\|$\\)")
-                          (not (eobp)))
-                (forward-line 1))
-              (insert
-               "\n"
-               ";; Added by Package.el.  This must come before configurations of\n"
-               ";; installed packages.  Don't delete this line.  If you don't want it,\n"
-               ";; just comment it out by adding a semicolon to the start of the line.\n"
-               ";; You may delete these explanatory comments.\n"
-               "(package-initialize)\n")
-              (unless (looking-at-p "$")
-                (insert "\n"))
-              (let ((file-precious-flag t))
-                (save-buffer))
-              (if buffer
-                  (progn
-                    (set-visited-file-name buffer-name)
-                    (set-buffer-modified-p nil))
-                (kill-buffer (current-buffer)))))))))
-  (setq package--init-file-ensured t))
-
 ;;;###autoload
 (defun package-install (pkg &optional dont-select)
   "Install the package PKG.
diff --git a/lisp/startup.el b/lisp/startup.el
index 7cf6fee425..56f688e334 100644
--- a/lisp/startup.el
+++ b/lisp/startup.el
@@ -1021,6 +1021,156 @@ command-line
     (and command-line-args
          (setcdr command-line-args args)))
 
+  ;; Warn for invalid user name.
+  (when init-file-user
+    (if (string-match "[~/:\n]" init-file-user)
+        (display-warning 'initialization
+                         (format "Invalid user name %s"
+                                 init-file-user)
+                         :error)
+      (if (file-directory-p (expand-file-name
+                             ;; We don't support ~USER on MS-Windows
+                             ;; and MS-DOS except for the current
+                             ;; user, and always load .emacs from
+                             ;; the current user's home directory
+                             ;; (see below).  So always check "~",
+                             ;; even if invoked with "-u USER", or
+                             ;; if $USER or $LOGNAME are set to
+                             ;; something different.
+                             (if (memq system-type '(windows-nt ms-dos))
+                                 "~"
+                               (concat "~" init-file-user))))
+          nil
+        (display-warning 'initialization
+                         (format "User %s has no home directory"
+                                 (if (equal init-file-user "")
+                                     (user-real-login-name)
+                                   init-file-user))
+                         :error))))
+
+  ;; Load the early init file, if found.
+  (let ((debug-on-error-from-init-file nil)
+        (debug-on-error-should-be-set nil)
+        (debug-on-error-initial (if (eq init-file-debug t)
+                                    'startup
+                                  init-file-debug))
+        (orig-enable-multibyte (default-value 'enable-multibyte-characters)))
+    (let ((debug-on-error debug-on-error-initial)
+          (inner
+           (lambda ()
+             ;; If no username, don't load the init file.
+             (when init-file-user
+               (let ((early-init-file-1
+                      (expand-file-name
+                       "early-init"
+                       (file-name-as-directory
+                        (concat "~" init-file-user "/.emacs.d")))))
+                 ;; Setting `user-init-file' to t tells `load' to
+                 ;; store the name of the file that was loaded, if
+                 ;; possible, into `user-init-file'. We're not using
+                 ;; `user-init-file' yet, so we can re-use it here for
+                 ;; the early init file.
+                 (setq user-init-file t)
+
+                 ;; Attempt to load the early init file. If it doesn't
+                 ;; exist, do nothing.
+                 (load early-init-file-1 t t)
+
+                 ;; If the init file could be loaded, move its
+                 ;; discovered filename from `user-init-file' into
+                 ;; `early-init-file', where it belongs.
+                 (unless (eq user-init-file t)
+                   (setq early-init-file user-init-file)
+                   (when (and early-init-file
+                              (equal (file-name-extension early-init-file)
+                                     "elc"))
+                     (let* ((source (file-name-sans-extension early-init-file))
+                            (alt (concat source ".el")))
+                       (setq source (cond ((file-exists-p alt) alt)
+                                          ((file-exists-p source) source)
+                                          (t nil)))
+                       (when source
+                         (when (file-newer-than-file-p source early-init-file)
+                           (message "Warning: %s is newer than %s"
+                                    source early-init-file)
+                           (sit-for 1))
+                         (setq early-init-file source))))))))))
+      (if init-file-debug
+          (funcall inner)
+        (condition-case error
+            (progn
+              (funcall inner)
+              (setq init-file-had-error nil))
+          (error
+           (display-warning
+            'initialization
+            (format-message "\
+An error occurred while loading `%s':\n\n%s%s%s\n\n\
+To ensure normal operation, you should investigate and remove the
+cause of the error in your initialization file.  Start Emacs with
+the `--debug-init' option to view a complete error backtrace."
+                            ;; Use `user-init-file' here because if
+                            ;; there was an error while loading the
+                            ;; init file, then `early-init-file' may
+                            ;; not have been reassigned; but
+                            ;; `user-init-file' will still be set by
+                            ;; `load'.
+                            user-init-file
+                            (get (car error) 'error-message)
+                            (if (cdr error) ": " "")
+                            (mapconcat (lambda (s) (prin1-to-string s t))
+                                       (cdr error) ", "))
+            :warning)
+           (setq init-file-had-error t))))
+      (or (eq debug-on-error debug-on-error-initial)
+          (setq debug-on-error-should-be-set t
+                debug-on-error-from-init-file debug-on-error)))
+    (if debug-on-error-should-be-set
+	  (setq debug-on-error debug-on-error-from-init-file))
+      (unless (or (default-value 'enable-multibyte-characters)
+		  (eq orig-enable-multibyte (default-value
+					      'enable-multibyte-characters)))
+	;; Init file changed to unibyte.  Reset existing multibyte
+	;; buffers (probably *scratch*, *Messages*, *Minibuf-0*).
+	;; Arguably this should only be done if they're free of
+	;; multibyte characters.
+	(mapc (lambda (buffer)
+		(with-current-buffer buffer
+		  (if enable-multibyte-characters
+		      (set-buffer-multibyte nil))))
+	      (buffer-list))
+	;; Also re-set the language environment in case it was
+	;; originally done before unibyte was set and is sensitive to
+	;; unibyte (display table, terminal coding system &c).
+	(set-language-environment current-language-environment)))
+
+  ;; If any package directory exists, initialize the package system.
+  (and user-init-file
+       package-enable-at-startup
+       (catch 'package-dir-found
+	 (let (dirs)
+	   (if (boundp 'package-directory-list)
+	       (setq dirs package-directory-list)
+	     (dolist (f load-path)
+	       (and (stringp f)
+		    (equal (file-name-nondirectory f) "site-lisp")
+		    (push (expand-file-name "elpa" f) dirs))))
+	   (push (if (boundp 'package-user-dir)
+		     package-user-dir
+		   (locate-user-emacs-file "elpa"))
+		 dirs)
+	   (dolist (dir dirs)
+	     (when (file-directory-p dir)
+	       (dolist (subdir (directory-files dir))
+		 (when (let ((subdir (expand-file-name subdir dir)))
+                         (and (file-directory-p subdir)
+                              (file-exists-p
+                               (expand-file-name
+                                (package--description-file subdir)
+                                subdir))))
+		   (throw 'package-dir-found t)))))))
+       (package-initialize))
+
   ;; Make sure window system's init file was loaded in loadup.el if
   ;; using a window system.
   ;; Initialize the window-system only after processing the command-line
@@ -1127,33 +1277,6 @@ command-line
     ;; the startup screen.
     (setq inhibit-startup-screen nil)
 
-    ;; Warn for invalid user name.
-    (when init-file-user
-      (if (string-match "[~/:\n]" init-file-user)
-	  (display-warning 'initialization
-			   (format "Invalid user name %s"
-				   init-file-user)
-			   :error)
-	(if (file-directory-p (expand-file-name
-			       ;; We don't support ~USER on MS-Windows
-			       ;; and MS-DOS except for the current
-			       ;; user, and always load .emacs from
-			       ;; the current user's home directory
-			       ;; (see below).  So always check "~",
-			       ;; even if invoked with "-u USER", or
-			       ;; if $USER or $LOGNAME are set to
-			       ;; something different.
-			       (if (memq system-type '(windows-nt ms-dos))
-				   "~"
-				 (concat "~" init-file-user))))
-	    nil
-	  (display-warning 'initialization
-			   (format "User %s has no home directory"
-				   (if (equal init-file-user "")
-				       (user-real-login-name)
-				     init-file-user))
-			   :error))))
-
     ;; Load that user's init file, or the default one, or none.
     (let (debug-on-error-from-init-file
 	  debug-on-error-should-be-set
@@ -1312,33 +1435,6 @@ command-line
 		 (eq face-ignored-fonts old-face-ignored-fonts))
       (clear-face-cache)))
 
-  ;; If any package directory exists, initialize the package system.
-  (and user-init-file
-       package-enable-at-startup
-       (catch 'package-dir-found
-	 (let (dirs)
-	   (if (boundp 'package-directory-list)
-	       (setq dirs package-directory-list)
-	     (dolist (f load-path)
-	       (and (stringp f)
-		    (equal (file-name-nondirectory f) "site-lisp")
-		    (push (expand-file-name "elpa" f) dirs))))
-	   (push (if (boundp 'package-user-dir)
-		     package-user-dir
-		   (locate-user-emacs-file "elpa"))
-		 dirs)
-	   (dolist (dir dirs)
-	     (when (file-directory-p dir)
-	       (dolist (subdir (directory-files dir))
-		 (when (let ((subdir (expand-file-name subdir dir)))
-                         (and (file-directory-p subdir)
-                              (file-exists-p
-                               (expand-file-name
-                                (package--description-file subdir)
-                                subdir))))
-		   (throw 'package-dir-found t)))))))
-       (package-initialize))
-
   (setq after-init-time (current-time))
   ;; Display any accumulated warnings after all functions in
   ;; `after-init-hook' like `desktop-read' have finalized possible
diff --git a/src/lread.c b/src/lread.c
index 6bc93b1481..b4823956e7 100644
--- a/src/lread.c
+++ b/src/lread.c
@@ -4910,6 +4910,15 @@ While Emacs loads and evaluates the init file, value is the real name
 of the file, regardless of whether or not it has the `.elc' extension.  */);
   Vuser_init_file = Qnil;
 
+  DEFVAR_LISP ("early-init-file", Vearly_init_file,
+               doc: /* File name, including directory, of user's early init file.
+If the file loaded had extension `.elc', and the corresponding source file
+exists, this variable contains the name of source file, suitable for use
+by functions like `custom-save-all' which edit the init file.
+While Emacs loads and evaluates the init file, value is the real name
+of the file, regardless of whether or not it has the `.elc' extension.  */);
+  Vearly_init_file = Qnil;
+
   DEFVAR_LISP ("current-load-list", Vcurrent_load_list,
 	       doc: /* Used for internal purposes by `load'.  */);
   Vcurrent_load_list = Qnil;
-- 
2.13.3

