[PATCH] WIP experimental extraction of elisp docstrings -> sphinx

[PATCH] WIP experimental extraction of elisp docstrings -> sphinx

[David Bremner]

---
This is a proof of concept extracting docstrings for inclusion in the emacs manual.
It uses a new emacs lisp "package" called rstdoc.

Among the hinky things

. this |br| hack is quite gross, but it was the best I could do

. the dependency on the ${emacs_bytecode} needs to be guarded proprely so we can still build
  most of the docs without emacs. I don't really understand the whole .eldeps thing, it
  seems to cause constant rebuilds for me anyway.,

. using (rstdoc-defcustom ) instead of (defcustom ) confuses the
  byte-compiler to think the variable is undefined. A simple "fix" would
  be to throw in a (defvar ) for each of those.

On the plus side, this introduces no external dependencies.

 doc/Makefile.local    |  4 +--
 doc/notmuch-emacs.rst |  6 +++-
 emacs/Makefile.local  |  3 +-
 emacs/notmuch-tag.el  |  5 ++-
 emacs/rstdoc.el       | 77 +++++++++++++++++++++++++++++++++++++++++++
 emacs/rstdoc.rsti     | 12 +++++++
 6 files changed, 102 insertions(+), 5 deletions(-)
 create mode 100644 emacs/rstdoc.el
 create mode 100644 emacs/rstdoc.rsti

diff --git a/doc/Makefile.local b/doc/Makefile.local
index 16459e35..fed8e1e4 100644
--- a/doc/Makefile.local
+++ b/doc/Makefile.local
@@ -37,10 +37,10 @@ INFO_INFO_FILES := $(INFO_TEXI_FILES:.texi=.info)
 %.gz: %
 	rm -f $@ && gzip --stdout $^ > $@
 
-sphinx-html:
+sphinx-html: ${emacs_bytecode}
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(DOCBUILDDIR)/html
 
-sphinx-texinfo:
+sphinx-texinfo: ${emacs_bytecode}
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(DOCBUILDDIR)/texinfo
 
 sphinx-info: sphinx-texinfo
diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst
index ce2e358e..1979da03 100644
--- a/doc/notmuch-emacs.rst
+++ b/doc/notmuch-emacs.rst
@@ -275,7 +275,7 @@ operations specified in ``notmuch-tagging-keys``; i.e. each
 
 :index:`notmuch-tagging-keys`
 
-   A list of keys and corresponding tagging operations.
+  |docstring::notmuch-tagging-keys|
 
 Configuration
 =============
@@ -300,3 +300,7 @@ suffix exist it will be read instead (just one of these, chosen in this
 order). Most often users create ``~/.emacs.d/notmuch-config.el`` and just
 work with it. If Emacs was invoked with the ``-q`` or ``--no-init-file``
 options, ``notmuch-init-file`` is not read.
+
+.. include:: ../emacs/rstdoc.rsti
+
+.. include:: ../emacs/notmuch-tag-doc.rsti
diff --git a/emacs/Makefile.local b/emacs/Makefile.local
index 1b3ef584..f0557800 100644
--- a/emacs/Makefile.local
+++ b/emacs/Makefile.local
@@ -117,4 +117,5 @@ ifeq ($(WITH_DESKTOP),1)
 	-update-desktop-database "$(DESTDIR)$(desktop_dir)"
 endif
 
-CLEAN := $(CLEAN) $(emacs_bytecode) $(dir)/notmuch-version.el $(dir)/notmuch-pkg.el
+CLEAN := $(CLEAN) $(emacs_bytecode) $(dir)/notmuch-version.el $(dir)/notmuch-pkg.el\
+	$(dir)/notmuch-tag-doc.rsti
diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el
index 0500927d..372517df 100644
--- a/emacs/notmuch-tag.el
+++ b/emacs/notmuch-tag.el
@@ -28,6 +28,9 @@
 (require 'crm)
 (require 'notmuch-lib)
 
+(eval-when-compile (require 'rstdoc)
+		   (rstdoc-init "notmuch-tag-doc.rsti"))
+
 (declare-function notmuch-search-tag "notmuch" tag-changes)
 (declare-function notmuch-show-tag "notmuch-show" tag-changes)
 (declare-function notmuch-tree-tag "notmuch-tree" tag-changes)
@@ -44,7 +47,7 @@
 		       (variable :tag "Tag variable"))
 		(string :tag "Name"))))
 
-(defcustom notmuch-tagging-keys
+(rstdoc-defcustom notmuch-tagging-keys
   `((,(kbd "a") notmuch-archive-tags "Archive")
     (,(kbd "u") notmuch-show-mark-read-tags "Mark read")
     (,(kbd "f") ("+flagged") "Flag")
diff --git a/emacs/rstdoc.el b/emacs/rstdoc.el
new file mode 100644
index 00000000..d5e78568
--- /dev/null
+++ b/emacs/rstdoc.el
@@ -0,0 +1,77 @@
+;;; rstdoc.el --- help generate documentation from docstrings -*-lexical-binding: t-*-
+     
+;; Copyright (C) 2018 David Bremner
+     
+;; Author: David Bremner <david@tethera.net>
+;; Created: 26 May 2018
+;; Keywords: emacs lisp, documentation
+;; Homepage: https://notmuchmail.org
+
+;; This file is not part of GNU Emacs.
+
+;; rstdoc.el is free software: you can redistribute it and/or modify it
+;; under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+;;
+;; rstdoc.el is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+;; General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with rstdoc.el.  If not, see <https://www.gnu.org/licenses/>.
+;;
+
+;;; Commentary:
+;; 
+
+;; Rstdoc provides some macros that wrap standard emacs forms for
+;; defining variables and functions, while capturing the docstrings
+;; for later re-use.
+
+;;; Code:
+
+(provide 'rstdoc)
+
+(defvar rstdoc-file nil
+    "File into which to write docstrings.")
+
+(defun rstdoc-init (&optional file-name)
+  "Initialize output of docstrings to FILE-NAME"
+  (when (or file-name rstdoc-file)
+    (make-local-variable 'rstdoc-file)
+    (let ((out-file (or file-name rstdoc-file))) ;; save buffer-local binding
+      (setq rstdoc-file out-file)
+      (with-temp-buffer
+	(insert (format ".. generated by rstdoc on %s \n" (current-time-string)))
+	(write-region (point-min) (point-max) out-file)))))
+
+(defmacro rstdoc-defcustom (symbol standard doc &rest args)
+  `(defcustom symbol standard doc ,@args)
+  (rstdoc-save-docstring symbol doc))
+
+(defun rstdoc-save-docstring (symbol docstring)
+  (when rstdoc-file
+    (let ((out-file rstdoc-file)) ;; save buffer local value
+      (with-temp-buffer
+	(insert (format "\n.. |docstring::%s| replace::\n" symbol))
+	(insert (replace-regexp-in-string "^" "    " (rstdoc--rst-quote-string docstring)))
+	(insert "\n")
+	(append-to-file (point-min) (point-max) out-file)))))
+
+(defvar rst--escape-alist
+  '( ("'" . "`")
+     ("^[[:space:]\t]*$" . "|br|"))
+    "list of (regex . replacement) pairs")
+
+(defun rstdoc--rst-quote-string (str)
+  (with-temp-buffer
+    (insert str)
+    (dolist (pair rst--escape-alist)
+      (goto-char (point-min))
+      (while (re-search-forward (car pair) nil t)
+	(replace-match (cdr pair))))
+    (buffer-substring (point-min) (point-max))))
+
+;;; rstdoc.el ends here
diff --git a/emacs/rstdoc.rsti b/emacs/rstdoc.rsti
new file mode 100644
index 00000000..857dab58
--- /dev/null
+++ b/emacs/rstdoc.rsti
@@ -0,0 +1,12 @@
+.. -*- rst -*-
+
+.. |br| replace:: |br-texinfo| |br-html|
+       
+.. |br-texinfo| raw:: texinfo
+
+   @* @*
+
+.. |br-html| raw:: html
+
+   <br /><br />
+
-- 
2.17.0

WIP2, automagic docstrings for notmuch-emacs.rst

[David Bremner]

This is a big improvement over
id:20180527185530.6795-1-david@tethera.net since it doesn't require
any source code modifications to generate the docstrings. It also
fixes the whining from the byte-compiler, and works for function
docstrings as well as variable docstrings.

The pseudo markup using |br| and |indent| is kindof gross still, but I
guess I can live with it.

I did test building html docs (make sphinx-html) and thost seem to
work OK as well, although not installed anywhere by default.

The use of docstrings in the manuals is still not guarded properly for
the no-emacs case; presumably we'd want the other documents to still
build.

I'm also not really sure about what sphinx markup to use for
keybindings with functions (see the "Copy to kill ring" section)

[PATCH 1/4] emacs: initial version of rstdoc.el

[David Bremner]

This small library is intended to support batch extraction of Emacs
Lisp docstrings from source files. Clients will need to include (or
replace) rstdoc.rsti.
---
 emacs/rstdoc.el   | 84 +++++++++++++++++++++++++++++++++++++++++++++++
 emacs/rstdoc.rsti | 22 +++++++++++++
 2 files changed, 106 insertions(+)
 create mode 100644 emacs/rstdoc.el
 create mode 100644 emacs/rstdoc.rsti

diff --git a/emacs/rstdoc.el b/emacs/rstdoc.el
new file mode 100644
index 00000000..f79a0cdd
--- /dev/null
+++ b/emacs/rstdoc.el
@@ -0,0 +1,84 @@
+;;; rstdoc.el --- help generate documentation from docstrings -*-lexical-binding: t-*-
+
+;; Copyright (C) 2018 David Bremner
+
+;; Author: David Bremner <david@tethera.net>
+;; Created: 26 May 2018
+;; Keywords: emacs lisp, documentation
+;; Homepage: https://notmuchmail.org
+
+;; This file is not part of GNU Emacs.
+
+;; rstdoc.el is free software: you can redistribute it and/or modify it
+;; under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+;;
+;; rstdoc.el is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+;; General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with rstdoc.el.  If not, see <https://www.gnu.org/licenses/>.
+;;
+
+;;; Commentary:
+;;
+
+;; Rstdoc provides a facility to extract all of the docstrings defined in
+;; an elisp source file. Usage:
+;;
+;; emacs -Q --batch -L . -l rstdoc -f rstdoc-batch-extract foo.el foo.rsti
+
+;;; Code:
+
+(provide 'rstdoc)
+
+(defun rstdoc-batch-extract ()
+  "Extract docstrings to and from the files on the command line"
+  (apply #'rstdoc-extract command-line-args-left))
+
+(defun rstdoc-extract (in-file out-file)
+  "Write docstrings from IN-FILE to OUT-FILE"
+  (load-file in-file)
+  (let* ((definitions (cdr (assoc (expand-file-name in-file) load-history)))
+	 (doc-hash (make-hash-table :test 'eq)))
+    (mapc
+     (lambda (elt)
+       (let ((pair
+	      (pcase elt
+		(`(defun . ,name) (cons name (documentation name)))
+		(`(,_ . ,_)  nil)
+		(sym (cons sym (get sym 'variable-documentation))))))
+	 (when (and pair (cdr pair))
+	   (puthash (car pair) (cdr pair) doc-hash))))
+     definitions)
+    (with-temp-buffer
+      (maphash
+       (lambda (key val)
+	 (rstdoc--insert-docstring key val))
+       doc-hash)
+      (write-region (point-min) (point-max) out-file))))
+
+(defun rstdoc--insert-docstring (symbol docstring)
+  (insert (format "\n.. |docstring::%s| replace::\n" symbol))
+  (insert (replace-regexp-in-string "^" "    " (rstdoc--rst-quote-string docstring)))
+  (insert "\n"))
+
+(defvar rst--escape-alist
+  '( ("'" . "`")
+     ("^[[:space:]\t]*$" . "|br|")
+     ("^[[:space:]\t]" . "|indent|"))
+    "list of (regex . replacement) pairs")
+
+(defun rstdoc--rst-quote-string (str)
+  (with-temp-buffer
+    (insert str)
+    (dolist (pair rst--escape-alist)
+      (goto-char (point-min))
+      (while (re-search-forward (car pair) nil t)
+	(replace-match (cdr pair))))
+    (buffer-substring (point-min) (point-max))))
+
+;;; rstdoc.el ends here
diff --git a/emacs/rstdoc.rsti b/emacs/rstdoc.rsti
new file mode 100644
index 00000000..f138b7bf
--- /dev/null
+++ b/emacs/rstdoc.rsti
@@ -0,0 +1,22 @@
+.. -*- rst -*-
+
+.. |br| replace:: |br-texinfo| |br-html|
+       
+.. |br-texinfo| raw:: texinfo
+
+   @* @*
+
+.. |br-html| raw:: html
+
+   <br /><br />
+
+.. |indent| replace:: |indent-texinfo| |indent-html|
+
+.. |indent-texinfo| raw:: texinfo
+
+   @* @ @ @ @
+
+.. |indent-html| raw:: html
+
+   <br />&nbsp;&nbsp;&nbsp;&nbsp;
+
-- 
2.17.0

[PATCH 2/4] emacs: require notmuch-lib from notmuch-wash.el

[David Bremner]

This is needed so that notmuch-wash.el is loadable by itself; in
particular for the docstring processing.
---
 emacs/notmuch-wash.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/emacs/notmuch-wash.el b/emacs/notmuch-wash.el
index 5f8b9267..54108d93 100644
--- a/emacs/notmuch-wash.el
+++ b/emacs/notmuch-wash.el
@@ -24,7 +24,7 @@
 ;;; Code:
 
 (require 'coolj)
-
+(require 'notmuch-lib)
 (declare-function notmuch-show-insert-bodypart "notmuch-show" (msg part depth &optional hide))
 (defvar notmuch-show-indent-messages-width)
 
-- 
2.17.0

[PATCH 3/4] emacs: build docstring (rsti) files

[David Bremner]

These are intended to included in the sphinx manual for notmuch-emacs.
---
 emacs/Makefile.local | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/emacs/Makefile.local b/emacs/Makefile.local
index 1b3ef584..c94b47c5 100644
--- a/emacs/Makefile.local
+++ b/emacs/Makefile.local
@@ -45,6 +45,7 @@ emacs_images := \
 	$(srcdir)/$(dir)/notmuch-logo.png
 
 emacs_bytecode = $(emacs_sources:.el=.elc)
+emacs_docstrings = $(emacs_sources:.el=.rsti)
 
 # Because of defmacro's and defsubst's, we have to account for load
 # dependencies between Elisp files when byte compiling.  Otherwise,
@@ -76,6 +77,8 @@ CLEAN+=$(dir)/.eldeps $(dir)/.eldeps.tmp $(dir)/.eldeps.x
 ifeq ($(HAVE_EMACS),1)
 %.elc: %.el $(global_deps)
 	$(call quiet,EMACS) --directory emacs -batch -f batch-byte-compile $<
+%.rsti: %.el
+	$(call quiet,EMACS) -batch -L emacs -l rstdoc -f rstdoc-batch-extract $< $@
 endif
 
 elpa: $(ELPA_FILE)
@@ -93,7 +96,7 @@ endif
 
 ifeq ($(WITH_EMACS),1)
 ifeq ($(HAVE_EMACS),1)
-all: $(emacs_bytecode)
+all: $(emacs_bytecode) $(emacs_docstrings)
 install-emacs: $(emacs_bytecode)
 endif
 
@@ -117,4 +120,5 @@ ifeq ($(WITH_DESKTOP),1)
 	-update-desktop-database "$(DESTDIR)$(desktop_dir)"
 endif
 
-CLEAN := $(CLEAN) $(emacs_bytecode) $(dir)/notmuch-version.el $(dir)/notmuch-pkg.el
+CLEAN := $(CLEAN) $(emacs_bytecode) $(dir)/notmuch-version.el $(dir)/notmuch-pkg.el \
+	$(emacs_docstrings)
-- 
2.17.0

[PATCH 4/4] WIP/doc: start to use docstrings in emacs manual.

[David Bremner]

this is still pretty much proof of concept. It needs guarding against
the no-emacs case.
---
 doc/Makefile.local    |  4 +--
 doc/notmuch-emacs.rst | 81 ++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/doc/Makefile.local b/doc/Makefile.local
index 16459e35..c0db2bc9 100644
--- a/doc/Makefile.local
+++ b/doc/Makefile.local
@@ -37,10 +37,10 @@ INFO_INFO_FILES := $(INFO_TEXI_FILES:.texi=.info)
 %.gz: %
 	rm -f $@ && gzip --stdout $^ > $@
 
-sphinx-html:
+sphinx-html: ${emacs_docstrings}
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(DOCBUILDDIR)/html
 
-sphinx-texinfo:
+sphinx-texinfo: ${emacs_docstrings}
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(DOCBUILDDIR)/texinfo
 
 sphinx-info: sphinx-texinfo
diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst
index ce2e358e..47bd6102 100644
--- a/doc/notmuch-emacs.rst
+++ b/doc/notmuch-emacs.rst
@@ -36,14 +36,14 @@ a mouse or by positioning the cursor and pressing ``<return>``
 |
 | Saved searches: **[edit]**
 |
-|	  52 **inbox**           52 **unread**
+|         52 **inbox**           52 **unread**
 |
 | Search: ____________________________________
 |
 | All tags: **[show]**
 |
-|	 Hit \`?' for context-sensitive help in any Notmuch screen.
-|		      Customize Notmuch or this page.
+|        Hit \`?' for context-sensitive help in any Notmuch screen.
+|                     Customize Notmuch or this page.
 
 You can change the overall appearance of the notmuch-hello screen by
 customizing the variable :index:`notmuch-hello-sections`.
@@ -190,6 +190,9 @@ pressing RET after positioning the cursor on a hidden part.
     advance to the next message, or advance to the next thread (if
     already on the last message of a thread).
 
+``c``
+    :ref:`show-copy`
+
 ``N``
     Move to next message
 
@@ -208,6 +211,63 @@ pressing RET after positioning the cursor on a hidden part.
 ``?``
     Display full set of key bindings
 
+Display of messages can be controlled by the following variables
+
+:index:`notmuch-message-headers`
+       |docstring::notmuch-message-headers|
+
+:index:`notmuch-message-headers-visible`
+       |docstring::notmuch-message-headers-visible|
+
+.. _show-copy:
+
+Copy to kill-ring
+-----------------
+
+You can use the usually Emacs ways of copying text to the kill-ring,
+but notmuch also provides some shortcuts. These keys are available in
+:ref:`notmuch-show`, and :ref:`notmuch-tree`. A subset are available
+in :ref:`notmuch-search`.
+
+``c F``	``notmuch-show-stash-filename``
+   |docstring::notmuch-show-stash-filename|
+
+``c G`` ``notmuch-show-stash-git-send-email``
+   |docstring::notmuch-show-stash-git-send-email|
+
+``c I`` ``notmuch-show-stash-message-id-stripped``
+   |docstring::notmuch-show-stash-message-id-stripped|
+
+``c L`` ``notmuch-show-stash-mlarchive-link-and-go``
+   |docstring::notmuch-show-stash-mlarchive-link-and-go|
+
+``c T`` ``notmuch-show-stash-tags``
+   |docstring::notmuch-show-stash-tags|
+
+``c c`` ``notmuch-show-stash-cc``
+   |docstring::notmuch-show-stash-cc|
+
+``c d`` ``notmuch-show-stash-date``
+   |docstring::notmuch-show-stash-date|
+
+``c f`` ``notmuch-show-stash-from``
+   |docstring::notmuch-show-stash-from|
+
+``c i`` ``notmuch-show-stash-message-id``
+   |docstring::notmuch-show-stash-message-id|
+
+``c l`` ``notmuch-show-stash-mlarchive-link``
+   |docstring::notmuch-show-stash-mlarchive-link|
+
+``c s`` ``notmuch-show-stash-subject``
+   |docstring::notmuch-show-stash-subject|
+
+``c t`` ``notmuch-show-stash-to``
+   |docstring::notmuch-show-stash-to|
+
+``c ?``
+    Show all available copying commands
+
 .. _notmuch-tree:
 
 notmuch-tree
@@ -218,6 +278,9 @@ email archives. Each line in the buffer represents a single
 message giving the relative date, the author, subject, and any
 tags.
 
+``c``
+    :ref:`show-copy`
+
 ``<return>``
    Displays that message.
 
@@ -275,7 +338,7 @@ operations specified in ``notmuch-tagging-keys``; i.e. each
 
 :index:`notmuch-tagging-keys`
 
-   A list of keys and corresponding tagging operations.
+  |docstring::notmuch-tagging-keys|
 
 Configuration
 =============
@@ -286,8 +349,10 @@ Importing Mail
 --------------
 
 :index:`notmuch-poll`
+   |docstring::notmuch-poll|
 
 :index:`notmuch-poll-script`
+   |docstring::notmuch-poll-script|
 
 Init File
 ---------
@@ -300,3 +365,11 @@ suffix exist it will be read instead (just one of these, chosen in this
 order). Most often users create ``~/.emacs.d/notmuch-config.el`` and just
 work with it. If Emacs was invoked with the ``-q`` or ``--no-init-file``
 options, ``notmuch-init-file`` is not read.
+
+.. include:: ../emacs/rstdoc.rsti
+
+.. include:: ../emacs/notmuch-lib.rsti
+
+.. include:: ../emacs/notmuch-show.rsti
+
+.. include:: ../emacs/notmuch-tag.rsti
-- 
2.17.0