diff --git a/.gitignore b/.gitignore index dd4eab759c..65d560dc4f 100644 --- a/.gitignore +++ b/.gitignore @@ -274,6 +274,7 @@ etc/emacs.tmpdesktop # Built by 'make-dist'. /MANIFEST +/MANIFEST-submodules # Distribution directories. /emacs-[1-9]*/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..e36c5a3285 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,12 @@ +[submodule "elpa/ada-mode"] + path = elpa/ada-mode + url = /c/Projects/emacs/master/.git + branch = emacs-28/ada-mode +[submodule "elpa/uniquify-files"] + path = elpa/uniquify-files + url = /c/Projects/emacs/master/.git + branch = emacs-28/uniquify-files +[submodule "elpa/wisi"] + path = elpa/wisi + url = /c/Projects/emacs/master/.git + branch = emacs-28/wisi diff --git a/admin/notes/elpa b/admin/notes/elpa index 1e9e7a9f52..ce84aacaa4 100644 --- a/admin/notes/elpa +++ b/admin/notes/elpa @@ -33,3 +33,178 @@ the package. It is easy to use the elpa branch to deploy a "local" copy of the package archive. For details, see the README file in the elpa branch. + +ELPA PACKAGES BUNDLED IN THE DISTRIBUTION TARBALL + +Why Bundle? + + - To provide backward compatibility for users while allowing more + flexible releases, and more convenient maintenance. + + - ada-mode was originally included in the emacs tarball, but was + removed when the ELPA version was capable enough. Some users + complained; bundling ada-mode would (almost) restore the + original behavior. + + - org and other packages currently maintain two copies of their + code, in emacs.git master and elpa.git. Synchronizing them is + difficult; it is not a simple `git push/pull'. Bundling would + allow using 'git push/pull' to synchonize between emacs.git + and elpa.git. + + - To provide a better "out-of-the-box" Emacs experience for new + Emacs users, by bundling new useful packages. + +After the initial install of Emacs, users do not need to do anything +special to update a bundled ELPA package; the normal package.el +mechanisms recognize all the versions properly. + +Emacs core code may not depend on (bundled or non-bundled) GNU ELPA +packages unless they are ":core" packages. + + - A simple git clone/checkout of emacs.git master creates a + workspace will all code required to build and run Emacs. + + - Any ELPA package can be changed without affecting Emacs core. + +For single file packages whose upstream repository is emacs.git, the +package is marked ":core" in the ELPA "elpa-packages" file. For +example, eldoc and flymake are :core packages. This mechanism is not +supported for packages with a separate upstream repository, because +maintaining sync between the files in emacs.git and upstream is +problematic; it is not a simple 'git push'. + +To bundle a multi-file package in Emacs, add to elpa.git a branch +`emacs-NN/[PKGNAME]` which holds the version to be included in the +next Emacs release of version NN: + + cd elpa/packages/[PKGNAME] + git checkout -b emacs-NN/[PKGNAME] + git checkout externals/[PKGNAME] + +Note that every GNU ELPA package has a branch `externals/[PKGNAME]` in +elpa.git. + +Also push the release branch to emacs.git: + + git push [USERNAME]@git.savannah.gnu.org:/srv/git/emacs.git emacs-NN/[PKGNAME] + + This allows emacs developers to checkout the bundled package + without requiring a separate clone of elpa.git. + +In emacs/master, add a git submodule for that branch: + + cd emacs/master + git submodule add --reference .git -b emacs-NN/[PKGNAME] \ + [USERNAME]@git.savannah.gnu.org:/srv/git/emacs.git ./elpa/[PKGNAME] + +Emacs developers should add '--recurse-submodules' to the 'git clone' +command when cloning Emacs; that will checkout the submodules with the +rest of Emacs. To checkout the submodules in an existing Emacs +directory tree: + + git submodule update --reference . --init + + However, the git worktree manual (https://git-scm.com/docs/git-worktree), in + the Bugs section at the end, says: + + Multiple checkout in general is still experimental, and the + support for submodules is incomplete. It is NOT recommended to + make multiple checkouts of a superproject. + + git worktrees allow keeping several checked out directory trees + while having only one local copy of the emacs.git repository. Some + emacs developers use worktrees, for example to keep emacs master, + emacs-27, and some feature branches checked out and built. + + Emacs developers that wish to use worktrees must not execute any + submodule commands; do not include --recurse-submodules' on 'git + clone', do not execute 'git submodule update'. Then the local git + repository is not a "superproject", and worktrees can be safely + used. + + We provide a script `checkout_git_elpa_worktrees.sh' that uses 'git + worktree' to checkout each bundled ELPA branch into emac/elpa, + instead of 'git submodule update'. + +There are currently (jan 2020) some multi-file packages with code in +both elpa.git and emacs.git; org, gnus for example. To convert those +packages into bundled packages, they are deleted from emacs.git, and +then follow the process above. The net result in the emacs directory +tree is that the code moves from emacs/lisp/[PKGNAME] to +emacs/elpa/[PKGNAME]. + +'make' treats emacs/elpa the same as it treats emacs/lisp and +emacs/doc; .el files are byte compiled, .texi files are compiled to +info. Note that documentation source .texi files for bundled packages +reside in emacs/elpa/[PKGNAME], not in emacs/doc. + +'make dist' treats emacs/elpa the same as it treats emacs/lisp; the +files are included in the distribution tarball. + +'make install' places the installed emacs/elpa directory next to the +installed emacs/lisp directory. + +At Emacs startup, the (installed or source) `emacs/elpa' directory is +added to `package-directory-list', and is treated the same as other +entries in that list. Note that this is different from files in +emacs/lisp; those are added to `load-path'. Using +`package-directory-list' allows users to disable bundled packages via +`package-load-list'. + +Emacs developers use the various 'git submodule' commands to maintain +packages in emacs/elpa. It is expected that very few edits will be +made there; the primary development of ELPA packages is done in a +checkout from elpa.git. Changes can be made in emacs/elpa to fix a bug +for a release, or to change code to be consistent with some emacs core +change. Such changes can be pushed to elpa.git and merged into the +package development branch. + +UNRESOLVED ISSUES/TODO: + +- One issue is whether the autoloads of bundled ELPA packages are + processed when we dump Emacs (like we do for all the packages that + are in Emacs core), or whether that's done during + `package-activate-all`. The baseline design does it during + `package-activate-all'. + + Doing it at dump time gives better startup times, at the cost of + making it impossible for the end-user to prevent activation of a + package. + + Users who care about startup time can do their own dump. + +- emacs/elpa/[PKNAME] vs emacs/lisp/[PKGNAME] + + Note that this choice can be made on a per-package basis; + emacs/.gitmodules records where the package is checked out. + + The baseline design keeps all bundled ELPA packages in + emacs/elpa/[PKNAME], both in the source and installed directory + trees. This makes it very easy to distinguish an emacs directory + tree with bundled packages from one without. + + An alternative is to keep the bundled ELPA packages in + emacs/lisp/[PKGNAME]; that minimizes the change when a package that + is current in emacs.git converts to a bundled package. This would + also mean that bundled packages are put in `load-path' at Emacs + startup, not in `package-directory-list'. + +- Does vc support submodules and nested worktrees? + +- Define a useful emacs lisp project, so `C-x p f' searches + `load-path' via `project-find-file' + + That reduces the need to remember what packages are bundled, and + where you have them checked out. + +- Update the mechanism that populates + https://www.gnu.org/software/emacs/manual/html_mono/* + + For example, ada-mode.html there still reflects the ancient version + that was in emacs core. + +- Automating compiling C, Ada, Rust or other language code required by + the package, to executables or modules. This is more important for + bundled packages; users expect Emacs to "just work" out of the box. + diff --git a/elpa/ada-mode b/elpa/ada-mode new file mode 160000 index 0000000000..05d61057ec --- /dev/null +++ b/elpa/ada-mode @@ -0,0 +1 @@ +Subproject commit 05d61057ecb532e813489f85d00620363798d72d diff --git a/elpa/uniquify-files b/elpa/uniquify-files new file mode 160000 index 0000000000..7b15736c17 --- /dev/null +++ b/elpa/uniquify-files @@ -0,0 +1 @@ +Subproject commit 7b15736c17ab4b60ebd269c9e7b62194ae6d2a15 diff --git a/elpa/wisi b/elpa/wisi new file mode 160000 index 0000000000..dd09dcf376 --- /dev/null +++ b/elpa/wisi @@ -0,0 +1 @@ +Subproject commit dd09dcf3768903e74f9d9f6a579f5dd2acd7a5d7 diff --git a/lisp/Makefile.in b/lisp/Makefile.in index 72f7f1676b..7fc63e0b4f 100644 --- a/lisp/Makefile.in +++ b/lisp/Makefile.in @@ -53,11 +53,14 @@ FIND_DELETE = # You can specify a different executable on the make command line, # e.g. "make EMACS=../src/emacs ...". -# We never change directory before running Emacs, so a relative file -# name is fine, and makes life easier. If we need to change -# directory, we can use emacs --chdir. +# We never change directory before running Emacs (except in git +# submodule commands), so a relative file name is fine, and makes +# life easier. If we need to change directory, we can use emacs +# --chdir. EMACS = ../src/emacs${EXEEXT} +ABS_EMACS = $(CURDIR)/$(EMACS) $(EMACSOPT) + # Command line flags for Emacs. EMACSOPT = -batch --no-site-file --no-site-lisp @@ -110,6 +113,7 @@ MAIN_FIRST = # The actual Emacs command run in the targets below. # Prevent any setting of EMACSLOADPATH in user environment causing problems. +# FIXME: why not 'unexport EMACSLOADPATH'? emacs = EMACSLOADPATH= '$(EMACS)' $(EMACSOPT) ## Subdirectories, relative to builddir. @@ -194,7 +198,7 @@ $(lisp)/finder-inf.el: # Use expand-file-name rather than $abs_scrdir so that Emacs does not # get confused when it compares file-names for equality. -autoloads .PHONY: $(lisp)/loaddefs.el +autoloads .PHONY: elpa-autoloads $(lisp)/loaddefs.el $(lisp)/loaddefs.el: gen-lisp $(LOADDEFS) $(AM_V_GEN)$(emacs) -l autoload \ --eval '(setq autoload-ensure-writable t)' \ @@ -202,6 +206,9 @@ $(lisp)/loaddefs.el: --eval '(setq generated-autoload-file (expand-file-name (unmsys--file-name "$@")))' \ -f batch-update-autoloads ${SUBDIRS_ALMOST} +elpa-autoloads : + EMACSLOADPATH= git submodule foreach $(ABS_EMACS) -l elpa-bundle -f elpa-bundle-generate-autoloads + # autoloads only runs when loaddefs.el is nonexistent, although it # generates a number of different files. Provide a force option to enable # regeneration of all these files. @@ -314,7 +321,7 @@ compile-targets: # Compile all the Elisp files that need it. Beware: it approximates # 'no-byte-compile', so watch out for false-positives! -compile-main: gen-lisp compile-clean +compile-main: gen-lisp compile-clean compile-bundled-elpa @(cd $(lisp) && \ els=`echo "${SUBDIRS_REL} " | sed -e 's|/\./|/|g' -e 's|/\. | |g' -e 's| |/*.el |g'`; \ for el in ${MAIN_FIRST} $$els; do \ @@ -340,11 +347,14 @@ compile-clean: fi; \ done +compile-bundled-elpa : + EMACSLOADPATH= git submodule foreach $(ABS_EMACS) -l elpa-bundle -f elpa-bundle-byte-compile + .PHONY: gen-lisp leim semantic ## make -C ../admin/unidata all should be here, but that would race ## with ../src. See comments above for loaddefs. -gen-lisp: leim semantic +gen-lisp: leim semantic gen-bundled-elpa # (re)compile titdic-cnv before recursing into `leim` since its used to # generate some of the Quail source files from tables. @@ -354,6 +364,9 @@ leim: semantic: $(MAKE) -C ../admin/grammars all EMACS="$(EMACS:.%=../.%)" +gen-bundled-elpa: + EMACSLOADPATH= git submodule foreach $(ABS_EMACS) -l elpa-bundle -f elpa-bundle-generate-pkg-file + # Compile all Lisp files, but don't recompile those that are up to # date. Some .el files don't get compiled because they set the # local variable no-byte-compile. @@ -452,9 +465,12 @@ $(CAL_DIR)/hol-loaddefs.el: --eval "(setq generated-autoload-file (expand-file-name (unmsys--file-name \"$@\")))" \ -f batch-update-autoloads $(CAL_DIR) -.PHONY: bootstrap-clean distclean maintainer-clean extraclean +.PHONY: elpa-bundle-clean bootstrap-clean distclean maintainer-clean extraclean + +elpa-bundle-clean: + git submodule foreach 'rm -f `basename $$name`-pkg.el `basename $$name`-autloads.el *.elc' -bootstrap-clean: +bootstrap-clean: elpa-bundle-clean find $(lisp) -name '*.elc' $(FIND_DELETE) rm -f $(AUTOGENEL) diff --git a/lisp/emacs-lisp/elpa-bundle.el b/lisp/emacs-lisp/elpa-bundle.el new file mode 100644 index 0000000000..f4e8c83c99 --- /dev/null +++ b/lisp/emacs-lisp/elpa-bundle.el @@ -0,0 +1,167 @@ +;;; elpa-bundle.el --- maintain bundled elpa packages -*- lexical-binding: t -*- + +(require 'package) +(require 'lisp-mnt) + +;;; functions named 'elpaa--' are copied from elpa-admin.el, modified as indicated + +(defvar elpaa--url "https://elpa.gnu.org/nongnu/") +(defun elpaa--default-url (pkgname) (concat elpaa--url pkgname ".html")) + +(defun elpaa--alist-to-plist-args (alist) + (mapcar (lambda (x) + (if (and (not (consp x)) + (or (keywordp x) + (not (symbolp x)) + (memq x '(nil t)))) + x `',x)) + (apply #'nconc + (mapcar (lambda (pair) (list (car pair) (cdr pair))) alist)))) + +(defun elpaa--override-version (pkg-spec orig-fun header) + (let ((str (funcall orig-fun header))) + (or (if (or (equal header "version") + (and str (equal header "package-version"))) + (let ((version-map (plist-get (cdr pkg-spec) :version-map)) + (dont-release (plist-get (cdr pkg-spec) :dont-release))) + (or (cadr (assoc str version-map)) + (and str dont-release + (string-match dont-release str) + (replace-match "snapshot" t t str))))) + str))) + +;;; mainfile inlined. +(defun elpaa--metadata (dir pkg-spec) + "Return a list (SIMPLE VERSION DESCRIPTION REQ EXTRAS), +where SIMPLE is non-nil if the package is simple; +VERSION is the version string of the simple package; +DESCRIPTION is the brief description of the package; +REQ is a list of requirements; +EXTRAS is an alist with additional metadata. + +PKG is the name of the package and DIR is the directory where it is." + (let* ((pkg (car pkg-spec)) + (mainfile (concat pkg ".el")) ;; FIXME: ignoring :main-file :lisp-dir + (files (directory-files dir nil "\\`dir\\'\\|\\.el\\'"))) + (setq files (delete (concat pkg "-pkg.el") files)) + (setq files (delete (concat pkg "-autoloads.el") files)) + (cond + ((file-exists-p mainfile) + (with-temp-buffer + (insert-file-contents mainfile) + (goto-char (point-min)) + (let* ((pkg-desc + (unwind-protect + (progn + (when (or (plist-get (cdr pkg-spec) :version-map) + (plist-get (cdr pkg-spec) :dont-release)) + (advice-add 'lm-header :around + (apply-partially + #'elpaa--override-version + pkg-spec))) + (package-buffer-info)) + (advice-remove 'lm-header + #'elpaa--override-version))) + (extras (package-desc-extras pkg-desc)) + (version (package-desc-version pkg-desc)) + (keywords (lm-keywords-list)) + ;; (_ (elpaa--version-to-list version)) ; Sanity check! + (pt (lm-header "package-type")) + (simple (if pt (equal pt "simple") (= (length files) 1))) + (found-url (alist-get :url extras)) + (found-keywords (alist-get :keywords extras))) + + (when (and keywords (not found-keywords)) + ;; Using an old package-buffer-info which doesn't include + ;; keywords. Fix it by hand. + (push (cons :keywords keywords) extras)) + (unless found-url + ;; Provide a good default URL. + (push (cons :url (elpaa--default-url pkg)) extras)) + (list simple + (package-version-join version) + (package-desc-summary pkg-desc) + (package-desc-reqs pkg-desc) + extras)))) + (t + (error "Can't find main file %s file in %s" mainfile dir))))) + +;; elpa--temp-file deleted +(defun elpaa--write-pkg-file (pkg-dir name metadata) + (let ((pkg-file (expand-file-name (concat name "-pkg.el") pkg-dir)) + (print-level nil) + (print-quoted t) + (print-length nil)) + (write-region + (concat (format ";; Generated package description from %s.el -*- no-byte-compile: t -*-\n" + name) + (prin1-to-string + (cl-destructuring-bind (version desc requires extras) + (cdr metadata) + (nconc + (list 'define-package + name + version + desc + (list 'quote + ;; Turn version lists into string form. + (mapcar + (lambda (elt) + (list (car elt) + (package-version-join (cadr elt)))) + requires))) + (elpaa--alist-to-plist-args extras)))) + "\n") + nil + pkg-file))) + +(defun elpa-bundle-pkg-spec () + "Return elpa admin package spec for package in default-directory." + (let* ((dir default-directory) + (pkg (file-name-nondirectory (directory-file-name dir)))) + + ;; This is the elpa admin package spec, _not_ the package.el + ;; package descriptor. We don't need :url here. + ;; + ;; FIXME: we do need :lisp-dir; so far, we can just look for + ;; a 'lisp' subdirectory. + ;; + ;; FIXME: we need :main-file to generate -pkg.el. So far + ;; that's only tramp; we could hard-code an exceptions list + ;; here. + (list pkg))) + +(defun elpa-bundle-generate-pkg-file () + "Generate package descriptor file for package in default-directory." + (let* ((dir default-directory) + (pkg (file-name-nondirectory (directory-file-name dir))) + (pkg-spec (elpa-bundle-pkg-spec)) + (metadata (elpaa--metadata dir pkg-spec))) + (elpaa--write-pkg-file dir pkg metadata))) + +(defun elpa-bundle-generate-autoloads () + "Generate autoloads file for package in default-directory." + (package-generate-autoloads (elpa-bundle-pkg-spec) default-directory)) + +(defun elpa-bundle-activate-all () + "Activate all bundled packages." + ;; `default-directory' is emacs/elpa/[PKGNAME], unless the package + ;; is under emacs/lisp somewhere. + + ;; FIXME: currently this only supports emacs/elpa; use 'git + ;; submodule foreach' to get list of package dirs. + (let ((package-user-dir (expand-file-name "elpa" (getenv "emacs_dir"))) + (package-directory-list nil)) + (package-load-all-descriptors) + (dolist (elt package-alist) + (package-activate-1 (car (cdr elt)))))) + +(defun elpa-bundle-byte-compile () + "Byte-compile package in default-directory." + ;; bundled packages may depend on other bundled packages + (elpa-bundle-activate-all) + + ;; This is how package.el compiles packages. + (byte-recompile-directory default-directory 0 t)) + +;;; elpa-bundle.el ends here diff --git a/lisp/emacs-lisp/package.el b/lisp/emacs-lisp/package.el index 90b7b88d58..a1503702df 100644 --- a/lisp/emacs-lisp/package.el +++ b/lisp/emacs-lisp/package.el @@ -305,12 +305,14 @@ package-user-dir ;;;###autoload (defcustom package-directory-list - ;; Defaults are subdirs named "elpa" in the site-lisp dirs. + ;; Defaults are subdirs named "elpa" in the site-lisp dirs, and + ;; bundled elpa packages. (let (result) (dolist (f load-path) (and (stringp f) (equal (file-name-nondirectory f) "site-lisp") (push (expand-file-name "elpa" f) result))) + (push (expand-file-name "../elpa" data-directory) result) (nreverse result)) "List of additional directories containing Emacs Lisp packages. Each directory name should be absolute. diff --git a/make-dist b/make-dist index 606fdd9e3a..dd55e3b8aa 100755 --- a/make-dist +++ b/make-dist @@ -407,6 +407,11 @@ manifest= else git ls-files | grep -v '^test' >$manifest fi || exit + +# Bundled ELPA packages are not included by 'git ls-files' +# above. + git submodule foreach git ls-files >> $manifest + printf '%s\n' $possibly_non_vc_files $info_files >>$manifest || exit sort -u -o $manifest $manifest || exit fi