find-file-project

find-file-project

[Stephen Leake]

[-- Attachment #1: Type: text/plain, Size: 604 bytes --]

Attached is a patch that implements find-file-project, with completion
of file-name on the project search path. It handles duplicate filenames
by uniquifying them witht trailing directory names.

The patch also adds small projects for elisp and global, to show that
this approach works for multiple backends.

Comments?

I can break this into smaller commits on master, if that seems like a
good idea.

I didn't add a NEWS entry. I don't think we are putting project related
changes in NEWS yet, since it is all new in Emacs 25. But the
file-name-all-completion change needs a NEWS entry.

-- 
-- Stephe

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: find-file-project.diff --]
[-- Type: text/x-patch, Size: 35919 bytes --]

diff --git a/lisp/cedet/cedet-global.el b/lisp/cedet/cedet-global.el
index 3773ba0..9fb46a0 100644
--- a/lisp/cedet/cedet-global.el
+++ b/lisp/cedet/cedet-global.el
@@ -120,6 +120,60 @@ Return a fully qualified filename."
 	(error "No file found")))
     ans))
 
+(defun find-file-complete-global-table (prefix)
+  "Do completion for file names in `find-file-complete-global'"
+  ;; Returned paths are relative to default-directory
+  (cond
+   ((string-match find-file-uniquify-regexp prefix)
+    ;; User has selected one match; return it.
+    (list prefix))
+
+   (t
+    (let* ((paths ;; Matching relative paths, as returned by global.
+	    (with-current-buffer (cedet-gnu-global-call (list "--ignore-case" "-P" prefix))
+	      (split-string (buffer-substring (point-min) (point-max)) "\n" t)))
+	   (dir-names
+	    (cl-mapcar (lambda (path) (cons (file-name-directory path) (file-name-nondirectory path)))
+		       paths))
+	   )
+
+      ;; "global -P `prefix'" matches in middle of the file name, and
+      ;; in the directory portion. The calling completion function
+      ;; rejects any completions that don't start with `prefix'.
+
+      (find-file-uniquify dir-names)
+      ))
+   ))
+
+(defun find-file-complete-global (filename)
+  "Prompt for completion of FILENAME in a Gnu global project."
+    (setq filename
+	  (completing-read
+	   "file: " ;; prompt
+	   (completion-table-with-cache #'find-file-complete-global-table) ;; collection
+	   nil ;; predicate
+	   t ;; require match
+	   filename
+	   ))
+
+    (when (string-match find-file-uniquify-regexp filename)
+      ;; Get partial dir from conflict
+      (setq filename (concat (match-string 2 filename) (match-string 1 filename))))
+
+    ;; If there are two files like:
+    ;;
+    ;; src/keyboard.c
+    ;; test/etags/c-src/emacs/src/keyboard.c
+    ;;
+    ;; and the user completes to the first, the following global call
+    ;; will return both. The desired result is always the shortest.
+    (with-current-buffer (cedet-gnu-global-call (list "--ignore-case" "-Pa" filename))
+      (let ((paths (split-string (buffer-substring (point-min) (point-max)) "\n" t)))
+	(setq paths (sort paths (lambda (a b) (< (length a) (length b)))))
+	(car paths)))
+
+    )
+
 (defun cedet-gnu-global-show-root ()
   "Show the root of a GNU Global area under the current buffer."
   (interactive)
@@ -193,6 +247,19 @@ If a database already exists, then just update it."
       )
     ))
 
+;;; project.el integration
+
+(defun project-try-global (dir)
+  (when (cedet-gnu-global-version-check t)
+    (let ((root (locate-dominating-file dir "GTAGS")))
+      (when root
+	(list 'global root)))))
+
+(cl-defmethod project-find-file ((prj (head global)) filename)
+  (let ((default-directory (file-name-as-directory (nth 1 prj))))
+    (find-file (find-file-complete-global filename))))
+
+
 (provide 'cedet-global)
 
 ;;; cedet-global.el ends here
diff --git a/lisp/files.el b/lisp/files.el
index c309f86..ad4fb4b 100644
--- a/lisp/files.el
+++ b/lisp/files.el
@@ -1691,6 +1691,85 @@ killed."
 	;; We already ran these; don't run them again.
 	(let (kill-buffer-query-functions kill-buffer-hook)
 	  (kill-buffer obuf))))))
+
+(defconst find-file-uniquify-regexp "^\\(.*\\)<\\(.*\\)>"
+  "Regexp matching uniqufied file name.
+Match 1 is the filename, match 2 is the relative directory.")
+
+(defun find-file-uniquify-conflicts (conflicts)
+  "Subroutine of `find-file-uniquify'."
+  (let ((common-root ;; shared prefix of dirs in conflicts - may be nil
+	 (fill-common-string-prefix (car (nth 0 conflicts)) (car (nth 1 conflicts)))))
+
+    (let ((temp (cddr conflicts))
+	  dir-name)
+      (while (and common-root
+		  temp)
+	(setq dir-name (pop temp))
+	(setq common-root (fill-common-string-prefix common-root (car dir-name)))))
+
+    (when common-root
+      ;; Trim `common-root' back to last '/'
+      (let ((i (1- (length common-root))))
+	(while (and (> i 0)
+		    (not (= (aref common-root i) ?/)))
+	  (setq i (1- i)))
+	(setq common-root (substring common-root 0 (1+ i)))))
+
+    (cl-mapcar
+     (lambda (dir-name)
+       (concat (cdr dir-name)
+	       "<" (substring (car dir-name) (length common-root)) ">"))
+     conflicts)
+    ))
+
+(defun find-file-uniquify (dir-names)
+  "Return a flat list of names from DIR-NAMES with duplicate filenames extended by directories.
+DIR-NAMES is a list of (dir . name)."
+  (let (result
+	conflicts ;; list of (dir . name) where all `name' are the same.
+	)
+
+    ;; Sort dir-names so duplicates are grouped together
+    (setq dir-names (sort dir-names (lambda (a b)
+				      (string< (cdr a) (cdr b)))))
+
+    (while dir-names
+      (setq conflicts (list (pop dir-names)))
+      (while (string= (cdr (car conflicts)) (cdr (car dir-names)))
+	(push (pop dir-names) conflicts))
+
+      (if (= 1 (length conflicts))
+	  (push (cdr (car conflicts)) result)
+	(setq result (append (find-file-uniquify-conflicts conflicts) result)))
+      )
+    (nreverse result)
+    ))
+
+(defun find-file-path-completion-table (path predicate prefix)
+  "Do completion for file names in `find-file-project'."
+  (cond
+   ((string-match find-file-uniquify-regexp prefix)
+    ;; User has selected one match; return it.
+    (list prefix))
+
+   ;; FIXME: handle prefix = "Makefile<dvc"; show /Projects/org.emacs.dvc/lisp/Makefile
+
+   (t
+    (let ((dir-names nil)) ;; list of (dir . name), where `name' is a completion
+      (dolist (dir path)
+        (when (file-directory-p dir)
+	  ;; `file-name-all-completions' applies
+	  ;; `completion-ignored-extensions' and `completion-regexp-list'
+          (setq dir-names
+		(append dir-names
+			(cl-mapcar (lambda (filename) (cons dir filename))
+				   (file-name-all-completions prefix dir predicate))))
+	  ))
+
+      (find-file-uniquify dir-names)))
+   ))
+
 \f
 ;; FIXME we really need to fold the uniquify stuff in here by default,
 ;; not using advice, and add it to the doc string.
diff --git a/lisp/progmodes/project.el b/lisp/progmodes/project.el
index 186840a..e9abf17 100644
--- a/lisp/progmodes/project.el
+++ b/lisp/progmodes/project.el
@@ -76,6 +76,101 @@ should take into account the value returned by
     (project-roots project)
     (funcall project-search-path-function))))
 
+;; Conversion between recursive and flat paths.
+(defun project--directory-directories-1 (dir ignore-regexp)
+  "Return a list of all directories in DIR (non-recursively).
+Ignores directories that match IGNORE-REGEXP (a regular expression).
+DIR must not end in ?/."
+  (let ((dirs (directory-files dir t nil t))
+	(dir-dot (concat dir "/."))
+	(dir-dotdot (concat dir "/.."))
+	result)
+    (while dirs
+      (let ((dir1 (pop dirs)))
+	(when (file-directory-p dir1)
+	  (unless (or
+		   (string= dir-dot dir1)
+		   (string= dir-dotdot dir1)
+		   (and ignore-regexp
+			(string-match ignore-regexp dir1)))
+	    (push dir1 result)))))
+    result))
+
+
+(defun project--directory-directories-recurse (dir ignore-regexp)
+  "Return a list of all directories in DIR, recursively.
+Ignores directories that match IGNORE-REGEXP (a regular expression).
+DIR must not end in ?/."
+  (let ((dirs (project--directory-directories-1 dir ignore-regexp))
+	result dir)
+    (while dirs
+      (setq dir (pop dirs))
+      (push dir result)
+      (setq result (append (project--directory-directories-recurse dir ignore-regexp) result)))
+
+    result))
+
+(defun project-recursive-ignores-to-flat (recursive-path ignore-dirs)
+  "Return a flat path constructed from RECURSIVE-PATH and IGNORE-DIRS."
+  (let ((dirs recursive-path)
+	(ignore-regexp (regexp-opt ignore-dirs))
+	result dir)
+
+    (while dirs
+      (setq dir (pop dirs))
+      (push dir result)
+      (setq result (append (project--directory-directories-recurse (directory-file-name dir) ignore-regexp) result)))
+
+    result))
+
+(defun project-flat-to-recursive-ignores (flat-path)
+  "Return a cons (RECURSIVE-PATH . IGNORE-DIRS) computed from FLAT-PATH."
+  (let* ((dirs (mapcar
+                 (lambda (dir)
+                   (file-name-as-directory (expand-file-name dir)))
+                 flat-path))
+	 search-path
+	 include-dirs
+	 ignore-dirs
+	 root)
+
+    ;; FIXME: combine with above loop?
+    (cl-delete-if-not #'file-exists-p dirs)
+
+    ;; Recursively delete subdirectories under a parent that is also
+    ;; in dirs. Add remaining to search-path, add unmentioned dirs to
+    ;; ignores.
+    ;;
+    ;; Start by sorting by name, so the roots that are
+    ;; present are just before their children.
+    (setq dirs (sort dirs (lambda (a b) (string< a b))))
+
+    (while (setq root (pop dirs))
+      (setq include-dirs nil)
+      (while (string-prefix-p root (car dirs))
+	(push (pop dirs) include-dirs))
+      (push root search-path)
+      (dolist (subdir (directory-files root t))
+	(when (file-directory-p subdir)
+	  (unless (or
+		   (string= (substring subdir -1) ".")
+		   (string= (substring subdir -2) "..")
+		   (member (file-name-as-directory subdir) include-dirs))
+	    (push (file-name-as-directory subdir) ignore-dirs)))))
+
+    (cons search-path ignore-dirs)
+    ))
+
+(cl-defgeneric project-ignore-dirs (_prj)
+  "Return list of absolute directory names that should be ignored.
+The names should be matched to directories when searching in
+`project-search-path'."
+  nil)
+
+(cl-defgeneric project-flat-search-path (prj)
+  "Return a flat search path for PRJ."
+  (project-recursive-ignores-to-flat (project-search-path prj) (project-ignore-dirs prj)))
+
 (cl-defgeneric project-roots (project)
   "Return the list of directory roots related to the current project.
 It should include the current project root, as well as the roots
@@ -83,6 +178,7 @@ of any other currently open projects, if they're meant to be
 edited together.  The directory names should be absolute.")
 
 (cl-defgeneric project-ignores (_project _dir)
+  ;; FIXME: do we need both project-ignores and project-ignore-files-globs?
   "Return the list of glob patterns to ignore inside DIR.
 Patterns can match both regular files and directories.
 To root an entry, start it with `./'.  To match directories only,
@@ -97,6 +193,79 @@ an element of `project-search-path'."
     vc-directory-exclusion-list)
    grep-find-ignored-files))
 
+(cl-defgeneric project-ignore-files-globs (_prj)
+  "Return list of file glob patterns that should be ignored in all directories.
+The globs should be matched to file and directory names sans path
+when searching in `project-search-path'."
+  (require 'grep)
+  (defvar grep-find-ignored-files)
+  (append
+    vc-directory-exclusion-list ;; preloaded
+    grep-find-ignored-files))
+
+(cl-defgeneric project-ignore-files-regexp (prj)
+  "Return a regular expression equivalent to `project-ignore-files-globs'."
+  (let ((globs (project-ignore-files-globs prj)))
+    (cond
+     ((= 1 (length globs))
+      (dired-glob-regexp (car globs)))
+
+     (t
+      (concat "\\(" (mapconcat #'dired-glob-regexp globs "\\)\\|\\(") "\\)"))
+     )))
+
+(cl-defgeneric project-find-file (prj filename)
+  "Find FILENAME with completion in current project PRJ."
+  (let* ((flat-path (project-flat-search-path prj))
+	 (regexp (project-ignore-files-regexp prj))
+	 (predicate
+	  (lambda (filename)
+	    (not (string-match regexp filename)))))
+
+    (setq filename
+	  (completing-read
+	   "file: " ;; prompt
+	   (completion-table-dynamic (apply-partially 'find-file-path-completion-table flat-path predicate))
+	   nil
+	   t ;; require match
+	   filename
+	   ))
+
+    ;; If text properties were preserved by completing-read, we could
+    ;; store the full directory in a text property on the
+    ;; conflicts. But they are not, so we construct a relative path to
+    ;; ensure the filename is found on `flat-path'.
+    (when (string-match find-file-uniquify-regexp filename)
+	(let ((dir (match-string 2 filename))
+	      (prefix "../")
+	      (i 0))
+
+	  (while (< i (length dir))
+	    (when (= (aref dir i) ?/)
+	      (setq prefix (concat prefix "../")))
+	    (setq i (1+ i)))
+
+	  (setq filename
+		(concat prefix
+			dir
+			"/"
+			(match-string 1 filename)))
+	  ))
+
+    (let ((absfilename (locate-file filename flat-path nil)))
+      (if absfilename
+	  (find-file absfilename)
+	;; FIXME: need human-readable name for project
+	(error "'%s' not found in project." filename)))
+    ))
+
+(defun find-file-project (filename)
+  "Find FILENAME (default prompt) with completion in current project.
+With prefix arg, FILENAME defaults to filename at point."
+  (interactive (list (when current-prefix-arg (thing-at-point 'filename))))
+  (project-find-file (project-current) filename))
+
+
 (defgroup project-vc nil
   "Project implementation using the VC package."
   :group 'tools)
diff --git a/src/dired.c b/src/dired.c
index 9773667..5bb7c80 100644
--- a/src/dired.c
+++ b/src/dired.c
@@ -349,7 +349,7 @@ If NOSORT is non-nil, the list is not sorted--its order is unpredictable.
   handler = Ffind_file_name_handler (directory, Qdirectory_files);
   if (!NILP (handler))
     return call5 (handler, Qdirectory_files, directory,
-                  full, match, nosort);
+		  full, match, nosort);
 
   return directory_files_internal (directory, full, match, nosort, 0, Qnil);
 }
@@ -420,10 +420,12 @@ determined by the variable `completion-ignored-extensions', which see.  */)
 }
 
 DEFUN ("file-name-all-completions", Ffile_name_all_completions,
-       Sfile_name_all_completions, 2, 2, 0,
+       Sfile_name_all_completions, 2, 3, 0,
        doc: /* Return a list of all completions of file name FILE in directory DIRECTORY.
-These are all file names in directory DIRECTORY which begin with FILE.  */)
-  (Lisp_Object file, Lisp_Object directory)
+These are all file names in directory DIRECTORY which begin with FILE,
+and for which PREDICATE returns non-nil. If PREDICATE is nil (the default),
+it is ignored.*/)
+  (Lisp_Object file, Lisp_Object directory, Lisp_Object predicate)
 {
   Lisp_Object handler;
   directory = Fexpand_file_name (directory, Qnil);
@@ -440,7 +442,7 @@ These are all file names in directory DIRECTORY which begin with FILE.  */)
   if (!NILP (handler))
     return call3 (handler, Qfile_name_all_completions, file, directory);
 
-  return file_name_completion (file, directory, 1, Qnil);
+  return file_name_completion (file, directory, 1, predicate);
 }
 
 static int file_name_completion_stat (int, struct dirent *, struct stat *);
diff --git a/test/automated/files.el b/test/automated/files.el
index 0522e0c..789fb01 100644
--- a/test/automated/files.el
+++ b/test/automated/files.el
@@ -166,6 +166,73 @@ form.")
       (delete-file tempfile))))
 
 
+(ert-deftest find-file-path-completion-table-duplicates ()
+  "Test completion when there are two files with the same name in
+different directories on path."
+  (let* ((root (make-temp-file "find-file-path-test" t))
+	 (dir1 (concat root "/dir1"))
+	 (dir2 (concat root "/dir2")))
+
+    (mkdir dir1)
+    (mkdir dir2)
+
+    (with-temp-file (concat dir1 "/file1.el")
+      (insert "dir1/file1.el"))
+    (with-temp-file (concat dir1 "/file2.el")
+      (insert "dir1/file2.el"))
+
+    (with-temp-file (concat dir2 "/file1.el")
+      (insert "dir2/file1.el"))
+    (with-temp-file (concat dir2 "/file3.el")
+      (insert "dir2/file3.el"))
+
+    ;; multiple completions, some with same name
+    (should (equal (find-file-path-completion-table (list dir1 dir2) nil "fi")
+		   '("file1.el<dir1>" "file1.el<dir2>" "file2.el" "file3.el")))
+
+    ;; only one completion (after user selects first of the above)
+    (should (equal (find-file-path-completion-table (list dir1 dir2) nil "file1.el<dir1>")
+		   '("file1.el<dir1>")))
+    ))
+
+(ert-deftest find-file-path-completion-table-predicate ()
+  "Test completion when there are two files with the same name in
+different directories on path, and a predicate."
+  (let* ((root (make-temp-file "find-file-path-test" t))
+	 (dir1 (concat root "/dir1"))
+	 (dir2 (concat root "/dir2"))
+         (regexp (dired-glob-regexp "*.elc"))
+	 (pred (lambda (name) (not (string-match regexp name)))))
+
+    (mkdir dir1)
+    (mkdir dir2)
+
+    (with-temp-file (concat dir1 "/file1.el")
+      (insert "dir1/file1.el"))
+    (with-temp-file (concat dir1 "/file1.elc")
+      (insert "dir1/file1.elc"))
+    (with-temp-file (concat dir1 "/file2.el")
+      (insert "dir1/file2.el"))
+    (with-temp-file (concat dir1 "/file2.elc")
+      (insert "dir1/file2.elc"))
+
+    (with-temp-file (concat dir2 "/file1.el")
+      (insert "dir2/file1.el"))
+    (with-temp-file (concat dir2 "/file1.elc")
+      (insert "dir2/file1.elc"))
+    (with-temp-file (concat dir2 "/file3.el")
+      (insert "dir2/file3.el"))
+    (with-temp-file (concat dir2 "/file3.elc")
+      (insert "dir2/file3.elc"))
+
+    ;; multiple completions, some with same name, predicate eliminates some
+    (should (equal (find-file-path-completion-table (list dir1 dir2) pred "fi")
+		   '("file1.el<dir1>" "file1.el<dir2>" "file2.el" "file3.el")))
+
+    ;; only one completion (after user selects first of the above)
+    (should (equal (find-file-path-completion-table (list dir1 dir2) pred "file1.el<dir1>")
+		   '("file1.el<dir1>")))
+    ))
 ;; Stop the above "Local Var..." confusing Emacs.
 \f
 
diff --git a/doc/lispref/files.texi b/doc/lispref/files.texi
index 735e08e..edfb045 100644
--- a/doc/lispref/files.texi
+++ b/doc/lispref/files.texi
@@ -2421,11 +2421,13 @@ the file, which in some cases may cause a security hole.
   This section describes low-level subroutines for completing a file
 name.  For higher level functions, see @ref{Reading File Names}.
 
-@defun file-name-all-completions partial-filename directory
-This function returns a list of all possible completions for a file
-whose name starts with @var{partial-filename} in directory
-@var{directory}.  The order of the completions is the order of the files
-in the directory, which is unpredictable and conveys no useful
+@defun file-name-all-completions partial-filename directory &optional predicate
+This function returns a list of all possible completions for a file in
+directory @var{directory} whose name starts with
+@var{partial-filename} and for which @var{predicate} (called with the
+filename) returns non-nil. If @var{predicate} is nil (the default), it
+is ignored. The order of the completions is the order of the files in
+the directory, which is unpredictable and conveys no useful
 information.
 
 The argument @var{partial-filename} must be a file name containing no
diff --git a/lisp/cedet/cedet-global.el b/lisp/cedet/cedet-global.el
index 3773ba0..9fb46a0 100644
--- a/lisp/cedet/cedet-global.el
+++ b/lisp/cedet/cedet-global.el
@@ -120,6 +120,60 @@ Return a fully qualified filename."
 	(error "No file found")))
     ans))
 
+(defun find-file-complete-global-table (prefix)
+  "Do completion for file names in `find-file-complete-global'"
+  ;; Returned paths are relative to default-directory
+  (cond
+   ((string-match find-file-uniquify-regexp prefix)
+    ;; User has selected one match; return it.
+    (list prefix))
+
+   (t
+    (let* ((paths ;; Matching relative paths, as returned by global.
+	    (with-current-buffer (cedet-gnu-global-call (list "--ignore-case" "-P" prefix))
+	      (split-string (buffer-substring (point-min) (point-max)) "\n" t)))
+	   (dir-names
+	    (cl-mapcar (lambda (path) (cons (file-name-directory path) (file-name-nondirectory path)))
+		       paths))
+	   )
+
+      ;; "global -P `prefix'" matches in middle of the file name, and
+      ;; in the directory portion. The calling completion function
+      ;; rejects any completions that don't start with `prefix'.
+
+      (find-file-uniquify dir-names)
+      ))
+   ))
+
+(defun find-file-complete-global (filename)
+  "Prompt for completion of FILENAME in a Gnu global project."
+    (setq filename
+	  (completing-read
+	   "file: " ;; prompt
+	   (completion-table-with-cache #'find-file-complete-global-table) ;; collection
+	   nil ;; predicate
+	   t ;; require match
+	   filename
+	   ))
+
+    (when (string-match find-file-uniquify-regexp filename)
+      ;; Get partial dir from conflict
+      (setq filename (concat (match-string 2 filename) (match-string 1 filename))))
+
+    ;; If there are two files like:
+    ;;
+    ;; src/keyboard.c
+    ;; test/etags/c-src/emacs/src/keyboard.c
+    ;;
+    ;; and the user completes to the first, the following global call
+    ;; will return both. The desired result is always the shortest.
+    (with-current-buffer (cedet-gnu-global-call (list "--ignore-case" "-Pa" filename))
+      (let ((paths (split-string (buffer-substring (point-min) (point-max)) "\n" t)))
+	(setq paths (sort paths (lambda (a b) (< (length a) (length b)))))
+	(car paths)))
+
+    )
+
 (defun cedet-gnu-global-show-root ()
   "Show the root of a GNU Global area under the current buffer."
   (interactive)
@@ -193,6 +247,19 @@ If a database already exists, then just update it."
       )
     ))
 
+;;; project.el integration
+
+(defun project-try-global (dir)
+  (when (cedet-gnu-global-version-check t)
+    (let ((root (locate-dominating-file dir "GTAGS")))
+      (when root
+	(list 'global root)))))
+
+(cl-defmethod project-find-file ((prj (head global)) filename)
+  (let ((default-directory (file-name-as-directory (nth 1 prj))))
+    (find-file (find-file-complete-global filename))))
+
+
 (provide 'cedet-global)
 
 ;;; cedet-global.el ends here
diff --git a/lisp/files.el b/lisp/files.el
index c309f86..ad4fb4b 100644
--- a/lisp/files.el
+++ b/lisp/files.el
@@ -1691,6 +1691,85 @@ killed."
 	;; We already ran these; don't run them again.
 	(let (kill-buffer-query-functions kill-buffer-hook)
 	  (kill-buffer obuf))))))
+
+(defconst find-file-uniquify-regexp "^\\(.*\\)<\\(.*\\)>"
+  "Regexp matching uniqufied file name.
+Match 1 is the filename, match 2 is the relative directory.")
+
+(defun find-file-uniquify-conflicts (conflicts)
+  "Subroutine of `find-file-uniquify'."
+  (let ((common-root ;; shared prefix of dirs in conflicts - may be nil
+	 (fill-common-string-prefix (car (nth 0 conflicts)) (car (nth 1 conflicts)))))
+
+    (let ((temp (cddr conflicts))
+	  dir-name)
+      (while (and common-root
+		  temp)
+	(setq dir-name (pop temp))
+	(setq common-root (fill-common-string-prefix common-root (car dir-name)))))
+
+    (when common-root
+      ;; Trim `common-root' back to last '/'
+      (let ((i (1- (length common-root))))
+	(while (and (> i 0)
+		    (not (= (aref common-root i) ?/)))
+	  (setq i (1- i)))
+	(setq common-root (substring common-root 0 (1+ i)))))
+
+    (cl-mapcar
+     (lambda (dir-name)
+       (concat (cdr dir-name)
+	       "<" (substring (car dir-name) (length common-root)) ">"))
+     conflicts)
+    ))
+
+(defun find-file-uniquify (dir-names)
+  "Return a flat list of names from DIR-NAMES with duplicate filenames extended by directories.
+DIR-NAMES is a list of (dir . name)."
+  (let (result
+	conflicts ;; list of (dir . name) where all `name' are the same.
+	)
+
+    ;; Sort dir-names so duplicates are grouped together
+    (setq dir-names (sort dir-names (lambda (a b)
+				      (string< (cdr a) (cdr b)))))
+
+    (while dir-names
+      (setq conflicts (list (pop dir-names)))
+      (while (string= (cdr (car conflicts)) (cdr (car dir-names)))
+	(push (pop dir-names) conflicts))
+
+      (if (= 1 (length conflicts))
+	  (push (cdr (car conflicts)) result)
+	(setq result (append (find-file-uniquify-conflicts conflicts) result)))
+      )
+    (nreverse result)
+    ))
+
+(defun find-file-path-completion-table (path predicate prefix)
+  "Do completion for file names in `find-file-project'."
+  (cond
+   ((string-match find-file-uniquify-regexp prefix)
+    ;; User has selected one match; return it.
+    (list prefix))
+
+   ;; FIXME: handle prefix = "Makefile<dvc"; show /Projects/org.emacs.dvc/lisp/Makefile
+
+   (t
+    (let ((dir-names nil)) ;; list of (dir . name), where `name' is a completion
+      (dolist (dir path)
+        (when (file-directory-p dir)
+	  ;; `file-name-all-completions' applies
+	  ;; `completion-ignored-extensions' and `completion-regexp-list'
+          (setq dir-names
+		(append dir-names
+			(cl-mapcar (lambda (filename) (cons dir filename))
+				   (file-name-all-completions prefix dir predicate))))
+	  ))
+
+      (find-file-uniquify dir-names)))
+   ))
+
 \f
 ;; FIXME we really need to fold the uniquify stuff in here by default,
 ;; not using advice, and add it to the doc string.
diff --git a/lisp/progmodes/project.el b/lisp/progmodes/project.el
index 186840a..e9abf17 100644
--- a/lisp/progmodes/project.el
+++ b/lisp/progmodes/project.el
@@ -76,6 +76,101 @@ should take into account the value returned by
     (project-roots project)
     (funcall project-search-path-function))))
 
+;; Conversion between recursive and flat paths.
+(defun project--directory-directories-1 (dir ignore-regexp)
+  "Return a list of all directories in DIR (non-recursively).
+Ignores directories that match IGNORE-REGEXP (a regular expression).
+DIR must not end in ?/."
+  (let ((dirs (directory-files dir t nil t))
+	(dir-dot (concat dir "/."))
+	(dir-dotdot (concat dir "/.."))
+	result)
+    (while dirs
+      (let ((dir1 (pop dirs)))
+	(when (file-directory-p dir1)
+	  (unless (or
+		   (string= dir-dot dir1)
+		   (string= dir-dotdot dir1)
+		   (and ignore-regexp
+			(string-match ignore-regexp dir1)))
+	    (push dir1 result)))))
+    result))
+
+
+(defun project--directory-directories-recurse (dir ignore-regexp)
+  "Return a list of all directories in DIR, recursively.
+Ignores directories that match IGNORE-REGEXP (a regular expression).
+DIR must not end in ?/."
+  (let ((dirs (project--directory-directories-1 dir ignore-regexp))
+	result dir)
+    (while dirs
+      (setq dir (pop dirs))
+      (push dir result)
+      (setq result (append (project--directory-directories-recurse dir ignore-regexp) result)))
+
+    result))
+
+(defun project-recursive-ignores-to-flat (recursive-path ignore-dirs)
+  "Return a flat path constructed from RECURSIVE-PATH and IGNORE-DIRS."
+  (let ((dirs recursive-path)
+	(ignore-regexp (regexp-opt ignore-dirs))
+	result dir)
+
+    (while dirs
+      (setq dir (pop dirs))
+      (push dir result)
+      (setq result (append (project--directory-directories-recurse (directory-file-name dir) ignore-regexp) result)))
+
+    result))
+
+(defun project-flat-to-recursive-ignores (flat-path)
+  "Return a cons (RECURSIVE-PATH . IGNORE-DIRS) computed from FLAT-PATH."
+  (let* ((dirs (mapcar
+                 (lambda (dir)
+                   (file-name-as-directory (expand-file-name dir)))
+                 flat-path))
+	 search-path
+	 include-dirs
+	 ignore-dirs
+	 root)
+
+    ;; FIXME: combine with above loop?
+    (cl-delete-if-not #'file-exists-p dirs)
+
+    ;; Recursively delete subdirectories under a parent that is also
+    ;; in dirs. Add remaining to search-path, add unmentioned dirs to
+    ;; ignores.
+    ;;
+    ;; Start by sorting by name, so the roots that are
+    ;; present are just before their children.
+    (setq dirs (sort dirs (lambda (a b) (string< a b))))
+
+    (while (setq root (pop dirs))
+      (setq include-dirs nil)
+      (while (string-prefix-p root (car dirs))
+	(push (pop dirs) include-dirs))
+      (push root search-path)
+      (dolist (subdir (directory-files root t))
+	(when (file-directory-p subdir)
+	  (unless (or
+		   (string= (substring subdir -1) ".")
+		   (string= (substring subdir -2) "..")
+		   (member (file-name-as-directory subdir) include-dirs))
+	    (push (file-name-as-directory subdir) ignore-dirs)))))
+
+    (cons search-path ignore-dirs)
+    ))
+
+(cl-defgeneric project-ignore-dirs (_prj)
+  "Return list of absolute directory names that should be ignored.
+The names should be matched to directories when searching in
+`project-search-path'."
+  nil)
+
+(cl-defgeneric project-flat-search-path (prj)
+  "Return a flat search path for PRJ."
+  (project-recursive-ignores-to-flat (project-search-path prj) (project-ignore-dirs prj)))
+
 (cl-defgeneric project-roots (project)
   "Return the list of directory roots related to the current project.
 It should include the current project root, as well as the roots
@@ -83,6 +178,7 @@ of any other currently open projects, if they're meant to be
 edited together.  The directory names should be absolute.")
 
 (cl-defgeneric project-ignores (_project _dir)
+  ;; FIXME: do we need both project-ignores and project-ignore-files-globs?
   "Return the list of glob patterns to ignore inside DIR.
 Patterns can match both regular files and directories.
 To root an entry, start it with `./'.  To match directories only,
@@ -97,6 +193,79 @@ an element of `project-search-path'."
     vc-directory-exclusion-list)
    grep-find-ignored-files))
 
+(cl-defgeneric project-ignore-files-globs (_prj)
+  "Return list of file glob patterns that should be ignored in all directories.
+The globs should be matched to file and directory names sans path
+when searching in `project-search-path'."
+  (require 'grep)
+  (defvar grep-find-ignored-files)
+  (append
+    vc-directory-exclusion-list ;; preloaded
+    grep-find-ignored-files))
+
+(cl-defgeneric project-ignore-files-regexp (prj)
+  "Return a regular expression equivalent to `project-ignore-files-globs'."
+  (let ((globs (project-ignore-files-globs prj)))
+    (cond
+     ((= 1 (length globs))
+      (dired-glob-regexp (car globs)))
+
+     (t
+      (concat "\\(" (mapconcat #'dired-glob-regexp globs "\\)\\|\\(") "\\)"))
+     )))
+
+(cl-defgeneric project-find-file (prj filename)
+  "Find FILENAME with completion in current project PRJ."
+  (let* ((flat-path (project-flat-search-path prj))
+	 (regexp (project-ignore-files-regexp prj))
+	 (predicate
+	  (lambda (filename)
+	    (not (string-match regexp filename)))))
+
+    (setq filename
+	  (completing-read
+	   "file: " ;; prompt
+	   (completion-table-dynamic (apply-partially 'find-file-path-completion-table flat-path predicate))
+	   nil
+	   t ;; require match
+	   filename
+	   ))
+
+    ;; If text properties were preserved by completing-read, we could
+    ;; store the full directory in a text property on the
+    ;; conflicts. But they are not, so we construct a relative path to
+    ;; ensure the filename is found on `flat-path'.
+    (when (string-match find-file-uniquify-regexp filename)
+	(let ((dir (match-string 2 filename))
+	      (prefix "../")
+	      (i 0))
+
+	  (while (< i (length dir))
+	    (when (= (aref dir i) ?/)
+	      (setq prefix (concat prefix "../")))
+	    (setq i (1+ i)))
+
+	  (setq filename
+		(concat prefix
+			dir
+			"/"
+			(match-string 1 filename)))
+	  ))
+
+    (let ((absfilename (locate-file filename flat-path nil)))
+      (if absfilename
+	  (find-file absfilename)
+	;; FIXME: need human-readable name for project
+	(error "'%s' not found in project." filename)))
+    ))
+
+(defun find-file-project (filename)
+  "Find FILENAME (default prompt) with completion in current project.
+With prefix arg, FILENAME defaults to filename at point."
+  (interactive (list (when current-prefix-arg (thing-at-point 'filename))))
+  (project-find-file (project-current) filename))
+
+
 (defgroup project-vc nil
   "Project implementation using the VC package."
   :group 'tools)
diff --git a/src/dired.c b/src/dired.c
index 9773667..5bb7c80 100644
--- a/src/dired.c
+++ b/src/dired.c
@@ -349,7 +349,7 @@ If NOSORT is non-nil, the list is not sorted--its order is unpredictable.
   handler = Ffind_file_name_handler (directory, Qdirectory_files);
   if (!NILP (handler))
     return call5 (handler, Qdirectory_files, directory,
-                  full, match, nosort);
+		  full, match, nosort);
 
   return directory_files_internal (directory, full, match, nosort, 0, Qnil);
 }
@@ -420,10 +420,12 @@ determined by the variable `completion-ignored-extensions', which see.  */)
 }
 
 DEFUN ("file-name-all-completions", Ffile_name_all_completions,
-       Sfile_name_all_completions, 2, 2, 0,
+       Sfile_name_all_completions, 2, 3, 0,
        doc: /* Return a list of all completions of file name FILE in directory DIRECTORY.
-These are all file names in directory DIRECTORY which begin with FILE.  */)
-  (Lisp_Object file, Lisp_Object directory)
+These are all file names in directory DIRECTORY which begin with FILE,
+and for which PREDICATE returns non-nil. If PREDICATE is nil (the default),
+it is ignored.*/)
+  (Lisp_Object file, Lisp_Object directory, Lisp_Object predicate)
 {
   Lisp_Object handler;
   directory = Fexpand_file_name (directory, Qnil);
@@ -440,7 +442,7 @@ These are all file names in directory DIRECTORY which begin with FILE.  */)
   if (!NILP (handler))
     return call3 (handler, Qfile_name_all_completions, file, directory);
 
-  return file_name_completion (file, directory, 1, Qnil);
+  return file_name_completion (file, directory, 1, predicate);
 }
 
 static int file_name_completion_stat (int, struct dirent *, struct stat *);
diff --git a/test/automated/files.el b/test/automated/files.el
index 0522e0c..789fb01 100644
--- a/test/automated/files.el
+++ b/test/automated/files.el
@@ -166,6 +166,73 @@ form.")
       (delete-file tempfile))))
 
 
+(ert-deftest find-file-path-completion-table-duplicates ()
+  "Test completion when there are two files with the same name in
+different directories on path."
+  (let* ((root (make-temp-file "find-file-path-test" t))
+	 (dir1 (concat root "/dir1"))
+	 (dir2 (concat root "/dir2")))
+
+    (mkdir dir1)
+    (mkdir dir2)
+
+    (with-temp-file (concat dir1 "/file1.el")
+      (insert "dir1/file1.el"))
+    (with-temp-file (concat dir1 "/file2.el")
+      (insert "dir1/file2.el"))
+
+    (with-temp-file (concat dir2 "/file1.el")
+      (insert "dir2/file1.el"))
+    (with-temp-file (concat dir2 "/file3.el")
+      (insert "dir2/file3.el"))
+
+    ;; multiple completions, some with same name
+    (should (equal (find-file-path-completion-table (list dir1 dir2) nil "fi")
+		   '("file1.el<dir1>" "file1.el<dir2>" "file2.el" "file3.el")))
+
+    ;; only one completion (after user selects first of the above)
+    (should (equal (find-file-path-completion-table (list dir1 dir2) nil "file1.el<dir1>")
+		   '("file1.el<dir1>")))
+    ))
+
+(ert-deftest find-file-path-completion-table-predicate ()
+  "Test completion when there are two files with the same name in
+different directories on path, and a predicate."
+  (let* ((root (make-temp-file "find-file-path-test" t))
+	 (dir1 (concat root "/dir1"))
+	 (dir2 (concat root "/dir2"))
+         (regexp (dired-glob-regexp "*.elc"))
+	 (pred (lambda (name) (not (string-match regexp name)))))
+
+    (mkdir dir1)
+    (mkdir dir2)
+
+    (with-temp-file (concat dir1 "/file1.el")
+      (insert "dir1/file1.el"))
+    (with-temp-file (concat dir1 "/file1.elc")
+      (insert "dir1/file1.elc"))
+    (with-temp-file (concat dir1 "/file2.el")
+      (insert "dir1/file2.el"))
+    (with-temp-file (concat dir1 "/file2.elc")
+      (insert "dir1/file2.elc"))
+
+    (with-temp-file (concat dir2 "/file1.el")
+      (insert "dir2/file1.el"))
+    (with-temp-file (concat dir2 "/file1.elc")
+      (insert "dir2/file1.elc"))
+    (with-temp-file (concat dir2 "/file3.el")
+      (insert "dir2/file3.el"))
+    (with-temp-file (concat dir2 "/file3.elc")
+      (insert "dir2/file3.elc"))
+
+    ;; multiple completions, some with same name, predicate eliminates some
+    (should (equal (find-file-path-completion-table (list dir1 dir2) pred "fi")
+		   '("file1.el<dir1>" "file1.el<dir2>" "file2.el" "file3.el")))
+
+    ;; only one completion (after user selects first of the above)
+    (should (equal (find-file-path-completion-table (list dir1 dir2) pred "file1.el<dir1>")
+		   '("file1.el<dir1>")))
+    ))
 ;; Stop the above "Local Var..." confusing Emacs.
 \f
 

Re: find-file-project

[Dmitry Gutov]

Hi Stephen,

We've discussed this before: I think this approach adds too much 
infrastructure (like forces through bits of "flat" project support, 
which aren't optimal for most of the projects I know of).

Why not just implement completion on file paths relative to the project 
root? The user could input a base file name, if they like, and TAB would 
expand it to one of the relative paths if it's unique, or allow them to 
input a directory. You won't need any other uniquification then.

On 09/15/2015 11:23 PM, Stephen Leake wrote:
> Attached is a patch that implements find-file-project, with completion
> of file-name on the project search path. It handles duplicate filenames
> by uniquifying them witht trailing directory names.

Maybe call it find-file-in-project?

> The patch also adds small projects for elisp and global, to show that
> this approach works for multiple backends.

I don't see the elisp backend.

Regarding global, is there no related semantic-symref-tool command? 
Maybe it would make sense to implement this in a generic fashion.

 > I can break this into smaller commits on master, if that seems like a
good idea.

If you want to test it out, please commit it to a feature branch instead.

> I didn't add a NEWS entry. I don't think we are putting project related
> changes in NEWS yet, since it is all new in Emacs 25. But the
> file-name-all-completion change needs a NEWS entry.

We will add NEWS entries before the release.

> +(defun find-file-complete-global (filename)
> +  "Prompt for completion of FILENAME in a Gnu global project."
> +    (setq filename
> +	  (completing-read
> +	   "file: " ;; prompt
> +	   (completion-table-with-cache #'find-file-complete-global-table) ;; collection
> +	   nil ;; predicate
> +	   t ;; require match
> +	   filename
> +	   ))

I don't think each implementation should do its own completing-read. 
Rather, they should just return the completion table from a generic 
method. E.g. (cl-defmethod project-file-completion-table ...).

> +    ;; and the user completes to the first, the following global call
> +    ;; will return both. The desired result is always the shortest.

I'd say the better match wins (not the shortest path), but if you want 
to add sorting, completion-metadata can include display-sort-function.

> +;;; project.el integration
> +
> +(defun project-try-global (dir)
> +  (when (cedet-gnu-global-version-check t)
> +    (let ((root (locate-dominating-file dir "GTAGS")))
> +      (when root
> +	(list 'global root)))))
> +
> +(cl-defmethod project-find-file ((prj (head global)) filename)
> +  (let ((default-directory (file-name-as-directory (nth 1 prj))))
> +    (find-file (find-file-complete-global filename))))

...if you're sure it's a good idea. After all, GNU Global is just a 
tool, it doesn't (and cannot) know too much about a project.

If it's used as just-another project implementation, you won't be able 
to integrate it with some more advanced kind of project definition.

> +(defconst find-file-uniquify-regexp "^\\(.*\\)<\\(.*\\)>"
> +  "Regexp matching uniqufied file name.
> +Match 1 is the filename, match 2 is the relative directory.")

This is way complicated. completion-at-point interface will help with 
typing, no real need to shorten the file names.

Another benefit of delegating all that to a completion table, is that 
different kinds of frontends would be able to use it.

> +;; Conversion between recursive and flat paths.
> +(defun project--directory-directories-1 (dir ignore-regexp)
> ...
> +(defun project--directory-directories-recurse (dir ignore-regexp)
> ...
> +(defun project-recursive-ignores-to-flat (recursive-path ignore-dirs)
> ...
> +(defun project-flat-to-recursive-ignores (flat-path)
> ...
> +(cl-defgeneric project-ignore-dirs (_prj)
 > ...
> +(cl-defgeneric project-flat-search-path (prj)
> ...

Do you really need all this?

> +  ;; FIXME: do we need both project-ignores and project-ignore-files-globs?

Just having project-ignores should suffice.

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> We've discussed this before: I think this approach adds too much
> infrastructure (like forces through bits of "flat" project support,
> which aren't optimal for most of the projects I know of).

Only code that needs flat paths uses them. As we discussed earlier, we
can either compute the flat path, and do completion on it, or compute
the flat path while doing completion.

And as we also discussed earlier, for _some_ projects, flat paths _are_
optimal.

What is the downside of a few dozen lines of code to support a few
projects? 

> Why not just implement completion on file paths relative to the
> project root? 

For Emacs elisp, there is no single root, except perhaps "/".

> The user could input a base file name, if they like, and TAB would
> expand it to one of the relative paths if it's unique, or allow them
> to input a directory. You won't need any other uniquification then.

That requires the user to know what directory the file is in, as
find-file completion does now.

Using a flat path avoids that. I find it quite useful to just type
"locate", and immediately see that there are two choices, one of which I
was unaware of.

On the other hand, what you are describing is pretty much what
find-file-project does. Have you tried it?

> Maybe call it find-file-in-project?

That does sound better.

>> The patch also adds small projects for elisp and global, to show that
>> this approach works for multiple backends.
>
> I don't see the elisp backend.

Oops; that got left out of the patch. This code is supposed to be in
project.el:
    
(cl-defstruct project-elisp
  search-path-flat
  ;; list of non-recursive directory names.
)

(defun project-elisp-make (extra-load-path)
  "Return a `project-elisp' object initialized from EXTRA-LOAD-PATH and `load-path'.
For example, add \"lisp/cedet/ede\" for core emacs."
  (make-project-elisp
    :search-path-flat (append extra-load-path load-path)))

(cl-defmethod project-flat-search-path ((prj project-elisp))
  "Override `project-flat-search-path' for elisp projects."
  (project-elisp-search-path-flat prj))

(cl-defmethod project-search-path ((_prj project-elisp))
  "Override `project-search-path' for elisp projects."
  load-path)

(cl-defmethod project-root ((_this project-elisp))
  ;; no meaningful root
  nil)

(cl-defmethod project-identifier-completion-table ((_this project-elisp))
  (elisp--xref-identifier-completion-table))

(cl-defmethod project-ignore-files-globs ((_this project-elisp))
  '("*.elc"))

;; example project; used for emacs

(defvar project-emacs
  (let ((cedet-root (file-name-directory (locate-file "cedet.el" load-path))))
    (project-elisp-make
     (project-recursive-ignores-to-flat
      (list
       (concat cedet-root "ede")
       (concat cedet-root "semantic")
       (concat cedet-root "srecode"))
      nil)
     )))

(defun project-try-elisp-emacs (_dir)
  (when (memq major-mode '(emacs-lisp-mode lisp-interaction-mode))
    project-emacs))


> Regarding global, is there no related semantic-symref-tool command?
> Maybe it would make sense to implement this in a generic fashion.

I've also implemented ede-find-file, which dispatches to the
semantic-symref global backend. Next patch.

>> I can break this into smaller commits on master, if that seems like a
> good idea.
>
> If you want to test it out, please commit it to a feature branch
> instead.

I've already tested it. I guess if you mean if I want others to test it.
I was hoping this patch would be enough.

>> +(defun find-file-complete-global (filename)
>> +  "Prompt for completion of FILENAME in a Gnu global project."
>> +    (setq filename
>> +	  (completing-read
>> +	   "file: " ;; prompt
>> +	   (completion-table-with-cache #'find-file-complete-global-table) ;; collection
>> +	   nil ;; predicate
>> +	   t ;; require match
>> +	   filename
>> +	   ))
>
> I don't think each implementation should do its own completing-read.
> Rather, they should just return the completion table from a generic
> method. E.g. (cl-defmethod project-file-completion-table ...).

For the default find-file-in-project, that would require computing the
flat path and the predicate on each use of the completion table. Other
instances might want to bind completion-ignore-case or
completion-regexp-list.

More importantly, the result of the completion is treated differently; the
default instance calls locate-file after rearranging the uniquification,
while the global instance calls ceded-gnu-global-call.

>> +    ;; and the user completes to the first, the following global call
>> +    ;; will return both. The desired result is always the shortest.
>
> I'd say the better match wins (not the shortest path), but if you want
> to add sorting, completion-metadata can include display-sort-function.

I don't follow; the completion does not need to be sorted. This code is
dealing with the result of completion.

A better way to accomplish this would be to somehow encode the full
directory path in the completion result, but I didn't find a way to do
that. In particular, text properties are not returned from completion.

>> +;;; project.el integration
>> +
>> +(defun project-try-global (dir)
>> +  (when (cedet-gnu-global-version-check t)
>> +    (let ((root (locate-dominating-file dir "GTAGS")))
>> +      (when root
>> +	(list 'global root)))))
>> +
>> +(cl-defmethod project-find-file ((prj (head global)) filename)
>> +  (let ((default-directory (file-name-as-directory (nth 1 prj))))
>> +    (find-file (find-file-complete-global filename))))
>
> ...if you're sure it's a good idea. After all, GNU Global is just a
> tool, it doesn't (and cannot) know too much about a project.
>
> If it's used as just-another project implementation, you won't be able
> to integrate it with some more advanced kind of project definition.

It's on a par with the existing EDE "generic" projects for vc tools. And
with the existing project-vc.

>> +(defconst find-file-uniquify-regexp "^\\(.*\\)<\\(.*\\)>"
>> +  "Regexp matching uniqufied file name.
>> +Match 1 is the filename, match 2 is the relative directory.")
>
> This is way complicated. completion-at-point interface will help with
> typing, no real need to shorten the file names.

I don't follow; what file names are being shortened? On the contrary,
they are being lengthened, with enough directory names to make them unique.

> Another benefit of delegating all that to a completion table, is that
> different kinds of frontends would be able to use it.

This code is used in the completion table, and can be used in any
completion table that has similarly colliding names.

For example, if the user enters "locate" <tab> at the find-file-project
prompt, the display then shows:

file: locate(.){c, rnc, el<lisp>, el<lisp/cedet/ede>}

showing that there are four possible completions, two with the same
filename. 

>> +;; Conversion between recursive and flat paths.
>> +(defun project--directory-directories-1 (dir ignore-regexp)
>> ...
>> +(defun project--directory-directories-recurse (dir ignore-regexp)
>> ...
>> +(defun project-recursive-ignores-to-flat (recursive-path ignore-dirs)
>> ...
>> +(defun project-flat-to-recursive-ignores (flat-path)
>> ...
>> +(cl-defgeneric project-ignore-dirs (_prj)
>> ...
>> +(cl-defgeneric project-flat-search-path (prj)
>> ...
>
> Do you really need all this?

Yes.

I guess you are asking for some rationale. This code allows converting
between the two equivalent representations of search paths;
recursive/ignores and flat. As we have discussed before, there are cases
where both of these conversions are useful.

>> +  ;; FIXME: do we need both project-ignores and project-ignore-files-globs?
>
> Just having project-ignores should suffice.

I guess I could just ignore 'dir' in project-ignores. 

-- 
-- Stephe

Re: find-file-project

[Stephen Leake]

Stephen Leake <stephen_leake@stephe-leake.org> writes:

> Dmitry Gutov <dgutov@yandex.ru> writes:
>
>>> +  ;; FIXME: do we need both project-ignores and project-ignore-files-globs?
>>
>> Just having project-ignores should suffice.
>
> I guess I could just ignore 'dir' in project-ignores. 

On the other hand, if some implementation of project-ignores does not
ignore the `dir' argument, then the predicate in project-find-file must
call project-ignores for each dir.

Do you have a use case where the `dir' argument to `project-ignores' is
not ignored?

-- 
-- Stephe

Re: find-file-project

[Stephen Leake]

Stephen Leake <stephen_leake@stephe-leake.org> writes:

> Stephen Leake <stephen_leake@stephe-leake.org> writes:
>
>> Dmitry Gutov <dgutov@yandex.ru> writes:
>>
>>>> +  ;; FIXME: do we need both project-ignores and project-ignore-files-globs?
>>>
>>> Just having project-ignores should suffice.
>>
>> I guess I could just ignore 'dir' in project-ignores. 
>
> On the other hand, if some implementation of project-ignores does not
> ignore the `dir' argument, then the predicate in project-find-file must
> call project-ignores for each dir.
>

Similarly, project-recursive-ignores-to-flat should call project-ignores
for each dir, rather than using a simple ignore-dirs list. Which would
eliminate the need for project-ignore-dirs.

But that is much more complicated, so I'd rather remove the `dir'
argument from project-ignores, if it is not needed for some known use
case.

-- 
-- Stephe

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 06:26 AM, Stephen Leake wrote:

> Do you have a use case where the `dir' argument to `project-ignores' is
> not ignored?

DIR is just as described: one of the project roots, for example. If 
those roots are different Git repositories, naturally they'll have 
different sets of ignores. That's a use case I expect in practice.

The project-vc frontend ignores this argument only because it works only 
with one root anyway.

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 05:49 AM, Stephen Leake wrote:

> Only code that needs flat paths uses them. As we discussed earlier, we
> can either compute the flat path, and do completion on it, or compute
> the flat path while doing completion.

If you absolutely need flat paths for completion here, just having the 
function that do one-way conversion should suffice. There's no need for 
project-flat-to-recursive-ignores, for instance.

> And as we also discussed earlier, for _some_ projects, flat paths _are_
> optimal.

That just means that the respective backends will store user settings in 
a certain fashion. There's no need to add functional duplication to the 
project API.

> What is the downside of a few dozen lines of code to support a few
> projects?

Implementation uncertainty and API complexity.

>> Why not just implement completion on file paths relative to the
>> project root?
>
> For Emacs elisp, there is no single root, except perhaps "/".

That's a decent argument. But then, when there are several roots, you 
could uniquify just their names, and prepend them to file names.

Your way may be better, but I'm not too sure yet.

>> The user could input a base file name, if they like, and TAB would
>> expand it to one of the relative paths if it's unique, or allow them
>> to input a directory. You won't need any other uniquification then.
>
> That requires the user to know what directory the file is in, as
> find-file completion does now.

It doesn't - you complete using the files list you've collected during 
the initial walk, and you match the typed file name against it. There's 
no reason to require that the input matches the beginning of the string.

> Using a flat path avoids that. I find it quite useful to just type
> "locate", and immediately see that there are two choices, one of which I
> was unaware of.

On the other hand, if I understand it right, I can't type "test unit 
foo", to see the unit test for foo.

> On the other hand, what you are describing is pretty much what
> find-file-project does. Have you tried it?

Not yet. I will, when you push it to the feature branch.

>> Maybe call it find-file-in-project?
>
> That does sound better.

Or just call the command project-find-file. It's inside project.el, 
after all.

>>> The patch also adds small projects for elisp and global, to show that
>>> this approach works for multiple backends.
>>
>> I don't see the elisp backend.
>
> Oops; that got left out of the patch. This code is supposed to be in
> project.el:

I see, thanks.

> (defvar project-emacs
>    (let ((cedet-root (file-name-directory (locate-file "cedet.el" load-path))))
>      (project-elisp-make
>       (project-recursive-ignores-to-flat
>        (list
>         (concat cedet-root "ede")
>         (concat cedet-root "semantic")
>         (concat cedet-root "srecode"))
>        nil)
>       )))

And this is ridiculous. Emacs obviously isn't a "flat" project. Just use 
the current format.

> I've also implemented ede-find-file, which dispatches to the
> semantic-symref global backend. Next patch.

Is there a reason to keep the gtags backend, then?

> I've already tested it. I guess if you mean if I want others to test it.

Yes.

> I was hoping this patch would be enough.

It isn't. Sorry.

>> I don't think each implementation should do its own completing-read.
>> Rather, they should just return the completion table from a generic
>> method. E.g. (cl-defmethod project-file-completion-table ...).
>
> For the default find-file-in-project, that would require computing the
> flat path and the predicate on each use of the completion table.

Why? The generic function could just as well return 
(completion-table-with-cache #'find-file-complete-global-table).

> Other
> instances might want to bind completion-ignore-case or
> completion-regexp-list.

completion-ignore-case is no business of theirs. Instead of 
completion-regexp-list, they can use a completion table with predicate.

> More importantly, the result of the completion is treated differently; the
> default instance calls locate-file after rearranging the uniquification,
> while the global instance calls ceded-gnu-global-call.

Why? If we've identified the requested file, let's just open it.

> I don't follow; the completion does not need to be sorted. This code is
> dealing with the result of completion.

I see.

> A better way to accomplish this would be to somehow encode the full
> directory path in the completion result, but I didn't find a way to do
> that. In particular, text properties are not returned from completion.

This seems to be a good argument against using base file names, and in 
favor of using full relative paths against project roots, combined with 
abbreviated roots.

> It's on a par with the existing EDE "generic" projects for vc tools. And
> with the existing project-vc.

project-vc is an minimum-viable-product for those who don't bother 
configuring anything else. There's no reason to have many of them.

> I don't follow; what file names are being shortened? On the contrary,
> they are being lengthened, with enough directory names to make them unique.

Either way, you're trying to same on typing during completion. 
completion-at-point helps with that as well.

> I guess you are asking for some rationale. This code allows converting
> between the two equivalent representations of search paths;
> recursive/ignores and flat. As we have discussed before, there are cases
> where both of these conversions are useful.

Not really. You don't need the ability to convert from flat paths to 
recursive if the project API consistently uses the "recursive" format. 
All data is already in it.

Re: find-file-project

[Eli Zaretskii]

> From: Dmitry Gutov <dgutov@yandex.ru>
> Date: Wed, 16 Sep 2015 01:21:30 +0300
> 
> > I didn't add a NEWS entry. I don't think we are putting project related
> > changes in NEWS yet, since it is all new in Emacs 25. But the
> > file-name-all-completion change needs a NEWS entry.

New packages should be mentioned in NEWS, there's a special section
for that.

> We will add NEWS entries before the release.

No, NEWS entries should be made together with pushing the changes.
It's the changes in the manuals that can (but don't have to) wait
until before the release.

If an important user-visible change is not mentioned even in NEWS,
people who track the development code will have no good means of
discovering them.  (Telling them to read emacs-diffs is too much, I
think.)

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/16/2015 06:26 AM, Stephen Leake wrote:
>
>> Do you have a use case where the `dir' argument to `project-ignores' is
>> not ignored?
>
> DIR is just as described: one of the project roots, for example. If
> those roots are different Git repositories, naturally they'll have
> different sets of ignores. That's a use case I expect in practice.

Right. And .gitignore, .cvsignore can occur in any directory.

.mtn-ignore also can contain path information.

I'll rework my implementation to handle this, using project-ignores.

One question; what does "To root an entry, start it with `./'.  " mean?
How does that affect the intended matching?

-- 
-- Stephe

Re: find-file-project

[Stefan Monnier]

>>> I don't think each implementation should do its own completing-read.
>>> Rather, they should just return the completion table from a generic
>>> method. E.g. (cl-defmethod project-file-completion-table ...).

100% agreement.


        Stefan

Re: find-file-project

[Stefan Monnier]

>> We will add NEWS entries before the release.
> No, NEWS entries should be made together with pushing the changes.

Yes, please.  If there's a good reason to think that the functionality
is not settled, we tolerate that the NEWS entry be incomplete (in which
case it should state that it's incomplete), but we do want a NEWS entry
right away.


        Stefan

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/16/2015 05:49 AM, Stephen Leake wrote:
>
>> Only code that needs flat paths uses them. As we discussed earlier, we
>> can either compute the flat path, and do completion on it, or compute
>> the flat path while doing completion.
>
> If you absolutely need flat paths for completion here, just having the
> function that do one-way conversion should suffice. There's no need
> for project-flat-to-recursive-ignores, for instance.

I found it convenient for building a project object, and for testing.
But it is otherwise not used in this patch.

>>> The user could input a base file name, if they like, and TAB would
>>> expand it to one of the relative paths if it's unique, or allow them
>>> to input a directory. You won't need any other uniquification then.
>>
>> That requires the user to know what directory the file is in, as
>> find-file completion does now.
>
> It doesn't - you complete using the files list you've collected during
> the initial walk, and you match the typed file name against it.
> There's no reason to require that the input matches the beginning of
> the string.

Except that 'completing-read' does require exactly that. I did try to
prepend directories for uniquification; it doesn't work. Or at least it
would require major surgery on completing-read to make it work.

>> Using a flat path avoids that. I find it quite useful to just type
>> "locate", and immediately see that there are two choices, one of which I
>> was unaware of.
>
> On the other hand, if I understand it right, I can't type "test unit
> foo", to see the unit test for foo.

It's not google search.

Can you be more specific? What is the file name you are trying to
complete to?


One thing it should handle but doesn't yet; when presented with:

foo.el<bar> foo.el<baz> foo.el<car>

The user should be able to type <b to narrow to the first two.

>> On the other hand, what you are describing is pretty much what
>> find-file-project does. Have you tried it?
>
> Not yet. I will, when you push it to the feature branch.

Ok, I'll do that.

>>> Maybe call it find-file-in-project?
>>
>> That does sound better.
>
> Or just call the command project-find-file. It's inside project.el,
> after all.

There's already a project-find-file.

There are two similarly named functions; one interactive, the other
dispatching. EDE uses -impl suffix for the latter; that would work here.

>>>> The patch also adds small projects for elisp and global, to show that
>>>> this approach works for multiple backends.
>>>
>>> I don't see the elisp backend.
>>
>> Oops; that got left out of the patch. This code is supposed to be in
>> project.el:
>
> I see, thanks.
>
>> (defvar project-emacs
>>    (let ((cedet-root (file-name-directory (locate-file "cedet.el" load-path))))
>>      (project-elisp-make
>>       (project-recursive-ignores-to-flat
>>        (list
>>         (concat cedet-root "ede")
>>         (concat cedet-root "semantic")
>>         (concat cedet-root "srecode"))
>>        nil)
>>       )))
>
> And this is ridiculous. Emacs obviously isn't a "flat" project. 

I don't know what you mean by "a flat project".

load-path is neither purely flat (cedet requires recursion) nor purely
recursive (it has both emacs/lisp and emacs/lisp/progmodes etc). So it
has to be converted one way or another.

I did try to use the recursive approach; this seemed simpler.

I'll use the recursive approach in the feature branch, for a true
comparison.

>> More importantly, the result of the completion is treated differently; the
>> default instance calls locate-file after rearranging the uniquification,
>> while the global instance calls ceded-gnu-global-call.
>
> Why? If we've identified the requested file, let's just open it.

`completing-read' does _not_ return an absolute or unique path. See below.

>> A better way to accomplish this would be to somehow encode the full
>> directory path in the completion result, but I didn't find a way to do
>> that. In particular, text properties are not returned from completion.
>
> This seems to be a good argument against using base file names, and in
> favor of using full relative paths against project roots, combined
> with abbreviated roots.

That would add significant clutter to the UI when filenames collide; you
often don't need the full path to a root to distinguish them.

And in the global case, the elisp code has no idea what path global is
using; it must always call global to get an absolute path.


It would make sense to modify completing-read to be able to return
meta-information with the completion. I looked at that briefly, and I
have no idea how to go about it.

>> I guess you are asking for some rationale. This code allows converting
>> between the two equivalent representations of search paths;
>> recursive/ignores and flat. As we have discussed before, there are cases
>> where both of these conversions are useful.
>
> Not really. You don't need the ability to convert from flat paths to
> recursive if the project API consistently uses the "recursive" format.
> All data is already in it.

You keep ignoring the fact that in some use cases, the available project
path information is already flat, and has to be converted to
recursive/ignores. I presented the AdaCore gpr file example.

In addition, a mix of flat and recursive/ignore is convenient for users
constructing a path.

This does need some unit tests for the conversion functions.

-- 
-- Stephe

Re: find-file-project

[Stephen Leake]

Eli Zaretskii <eliz@gnu.org> writes:

>> From: Dmitry Gutov <dgutov@yandex.ru>
>> Date: Wed, 16 Sep 2015 01:21:30 +0300
>> 
>> > I didn't add a NEWS entry. I don't think we are putting project related
>> > changes in NEWS yet, since it is all new in Emacs 25. But the
>> > file-name-all-completion change needs a NEWS entry.
>
> New packages should be mentioned in NEWS, there's a special section
> for that.
>
>> We will add NEWS entries before the release.
>
> No, NEWS entries should be made together with pushing the changes.

Ok, I'll add a brief mention of package.el under "New Modes and Packages
in Emacs 25.1", so we don't forget it.

-- 
-- Stephe

Re: find-file-project

[Stephen Leake]

Stephen Leake <stephen_leake@stephe-leake.org> writes:

> Dmitry Gutov <dgutov@yandex.ru> writes:
>
>> On 09/16/2015 05:49 AM, Stephen Leake wrote:
>>
>>> (defvar project-emacs
>>>    (let ((cedet-root (file-name-directory (locate-file "cedet.el" load-path))))
>>>      (project-elisp-make
>>>       (project-recursive-ignores-to-flat
>>>        (list
>>>         (concat cedet-root "ede")
>>>         (concat cedet-root "semantic")
>>>         (concat cedet-root "srecode"))
>>>        nil)
>>>       )))
>>
>> And this is ridiculous. Emacs obviously isn't a "flat" project. 
>
> I don't know what you mean by "a flat project".
>
> load-path is neither purely flat (cedet requires recursion) nor purely
> recursive (it has both emacs/lisp and emacs/lisp/progmodes etc). So it
> has to be converted one way or another.

And the user has to provide additional information as to what parts are
flat vs recursive, for the conversion to be done properly.

For example, SourceForge CEDET contains the "cogre" directory under
"cedet". Since that is not part of Emacs core yet, if I'm testing
SourceForge Emacs prior to a merge with Emacs core, it is not
appropriate to include "cogre" in the project search path.

-- 
-- Stephe

Re: find-file-project

[Eli Zaretskii]

> From: Stephen Leake <stephen_leake@stephe-leake.org>
> Date: Wed, 16 Sep 2015 08:40:50 -0500
> 
> Ok, I'll add a brief mention of package.el under "New Modes and Packages
> in Emacs 25.1", so we don't forget it.

Thanks.

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 03:56 PM, Stephen Leake wrote:

> One question; what does "To root an entry, start it with `./'.  " mean?
> How does that affect the intended matching?

That means that ./f* matches ./foo, but not ./bar/foo (these are paths 
relative to the root).

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 04:40 PM, Stephen Leake wrote:

> Ok, I'll add a brief mention of package.el under "New Modes and Packages
> in Emacs 25.1", so we don't forget it.

Please also mention project.el in there somewhere. Thanks. ;-)

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 05:13 PM, Stephen Leake wrote:

>>> And this is ridiculous. Emacs obviously isn't a "flat" project.
>>
>> I don't know what you mean by "a flat project".

It's not an Ada project.

>> load-path is neither purely flat (cedet requires recursion) nor purely
>> recursive (it has both emacs/lisp and emacs/lisp/progmodes etc). So it
>> has to be converted one way or another.

"emacs/lisp and emacs/lisp/progmodes" is not a good counter-argument.

search-path may have entries where one is a subdirectory of another, but 
each such entry is "recursive" anyway (we have to consider all files 
inside each respective tree). When search path used for file traversal, 
the caller will simply have to filter out the entries that are redundant 
for its purposes.

> And the user has to provide additional information as to what parts are
> flat vs recursive, for the conversion to be done properly.

I don't see that. If you need this complexity, just use the "recursive" 
format (it's more powerful), and specify any exclusions via ignores.

> For example, SourceForge CEDET contains the "cogre" directory under
> "cedet". Since that is not part of Emacs core yet, if I'm testing
> SourceForge Emacs prior to a merge with Emacs core, it is not
> appropriate to include "cogre" in the project search path.

Well, ignore it, then.

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/16/2015 03:56 PM, Stephen Leake wrote:
>
>> One question; what does "To root an entry, start it with `./'.  " mean?
>> How does that affect the intended matching?
>
> That means that ./f* matches ./foo, but not ./bar/foo (these are paths
> relative to the root).

By "root" I guess you mean the argument "dir", since that is an element
of the recursive project-search-path.

To be very clear, suppose we have this directory structure:

dir1/
    dir11/
        file1.el
        file1.elc

dir2/
    dir21/
        obj/
        Makefile
        foo.exe
        bar.exe
        test_1.text
        test_2.text

    dir22/
        able.exe

dir3/
    dir31/
        file3.texi
        file3.dvi
        file4.text

project-search-path returns (".../dir1" ".../dir2" ".../dir3"); thus
project-ignores is only called with one of these three values. (I'm
ignoring project-roots, because I don't understand when it would be
different from project-search-path).

The user wants to ignore "*/*.elc", "*/*.dvi", "*/*.exe",
"dir2/dir21/obj/*", "dir2/dir21/*.text", "dir3/dir31/*.dvi".

So one implementation of project-ignores could be:

(cond
 ((string-equal dir ".../dir1")
  '("*.elc" "*.dvi" "*.exe"))

 ((string-equal dir ".../dir2")
  '("*.elc" "*.dvi" "*.exe" "./dir2/dir21/obj/" "dir2/dir21/*.text"))

 ((string-equal dir ".../dir3")
  '("*.elc" "*.dvi" "*.exe" "./dir3/dir31/*.dvi"))

)

A real project implementation would use an alist or hashmap.

Then find-file-path-completion-table could take a recursive
project-search-path as the "path" argument, and recurse thru the
directory tree, calling project-ignores at each directory, with a parent
directory from project-search-path as the argument, and comparing the
path of each file - relative to the parent directory - to the glob
patterns.

For example, when processing dir2/dir21, it calls (project-ignores
"dir2"), and compares "./dir2/dir21/obj/" to "dir2/dir21/obj", and
"*.exe" to "dir2/dir21/foo.exe", both yeilding "ignore".

Is that right?


I gather this was designed to match the semantics of find? I don't see
any uses of project-ignores in the current code, but it could be used
with find-grep in a project-aware xref.

Since the compare for directories is different from that for files, it
would simplify the use of project-ignores (at least for this use case)
if it was split into project-ignore-files and project-ignore-dirs. Or
took an arg to specify that. But I'll use it as is, and see how much
that would really gain.


I was stuck on using a flat path for find-file-path-completion-table
because I started out trying to reuse locate-file-completion-table. But
now that I'm implementing the completion function from scratch, and it
is reading the disk and ignoring some files anyway, it does make more
sense start with a recursive path.

--
-- Stephe

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/16/2015 05:13 PM, Stephen Leake wrote:
>
>>>> And this is ridiculous. Emacs obviously isn't a "flat" project.
>>>
>>> I don't know what you mean by "a flat project".
>
> It's not an Ada project.

That's not a useful definition, and therefore not helpful in moving this
discussion towards closure.

>>> load-path is neither purely flat (cedet requires recursion) nor purely
>>> recursive (it has both emacs/lisp and emacs/lisp/progmodes etc). So it
>>> has to be converted one way or another.
>
> "emacs/lisp and emacs/lisp/progmodes" is not a good counter-argument.
>
> search-path may have entries where one is a subdirectory of another,
> but each such entry is "recursive" anyway (we have to consider all
> files inside each respective tree). 

That's clearly suboptimal; the subdirectory will be processed twice in
every use of project-search-path. Why would that ever be a good idea?

project-prune-directories optimizes that usage, so that
project-seach-path is purely recursive; I assume that's why it's there.
project-recursive-ignores-to-flat does the same conversion, more
accurately (it also produces the ignores list).

> When search path used for file traversal, the caller will simply have
> to filter out the entries that are redundant for its purposes.

It clearly complicates completion; we have to delete lots of actual
duplicates, rather than only worrying about files in different
directories having the same name.

When would such a definition of project-search-path actually be useful?


You are going a long way merely to justify calling "load-path" recursive.

>> And the user has to provide additional information as to what parts are
>> flat vs recursive, for the conversion to be done properly.
>
> I don't see that. If you need this complexity, just use the
> "recursive" format (it's more powerful), 

It is not "more powerful"; we have already agreed that one format can be
converted accurately to the other; the code in the patch implements those
conversions. 


You are missing my main point; the user must specify information in
addition to "load-path" in order to determine the correct search path
for an elisp project.

Whether that additional information is specified as "extra includes" or
"extra ignores" is a secondary issue.

>> For example, SourceForge CEDET contains the "cogre" directory under
>> "cedet". Since that is not part of Emacs core yet, if I'm testing
>> SourceForge Emacs prior to a merge with Emacs core, it is not
>> appropriate to include "cogre" in the project search path.
>
> Well, ignore it, then.

Yes, that is the user intent. The question is, how is the user supposed
to specify that intent to the elisp project? "load-path" alone does not
state whether it should be ignored; the user must provide additional
information.

-- 
-- Stephe

Re: find-file-project

[Stephen Leake]

Stephen Leake <stephen_leake@stephe-leake.org> writes:

> I don't see
> any uses of project-ignores in the current code, but it could be used
> with find-grep in a project-aware xref.

I take it back; xref.el xref-find-regexp does exactly that.

-- 
-- Stephe

Re: find-file-project

[Stephen Leake]

Stefan Monnier <monnier@IRO.UMontreal.CA> writes:

>>>> I don't think each implementation should do its own completing-read.
>>>> Rather, they should just return the completion table from a generic
>>>> method. E.g. (cl-defmethod project-file-completion-table ...).
>
> 100% agreement.

I agree it would be nice, but it requires that completing-read return
meta-information in addition to the completion string. I've started a
separate thread on that issue.

-- 
-- Stephe

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/16/2015 04:40 PM, Stephen Leake wrote:
>
>> Ok, I'll add a brief mention of package.el under "New Modes and Packages
>> in Emacs 25.1", so we don't forget it.
>
> Please also mention project.el in there somewhere. Thanks. ;-)

Yes, I don't know how that spelling change happened. Sorry.

-- 
-- Stephe

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 04:31 PM, Stephen Leake wrote:

> I found it convenient for building a project object, and for testing.
> But it is otherwise not used in this patch.

Please build the project objects using the "recursive" approach.

> Except that 'completing-read' does require exactly that. I did try to
> prepend directories for uniquification; it doesn't work. Or at least it
> would require major surgery on completing-read to make it work.

You might have to use a different `category' and set up a 
`completion-category-overrides' entry that would only use 
`partial-completion' as its style.

You'll be able input "/file", and it'll match "foo/bar/file.el". The 
completion table will have to handle the `boundaries' action properly, 
though.

Later, we can even add a new, more permissive style. We could use it in 
other places as well.

>> On the other hand, if I understand it right, I can't type "test unit
>> foo", to see the unit test for foo.
>
> It's not google search.

I'm sure you agree that web search is very handy.

> Can you be more specific? What is the file name you are trying to
> complete to?

Suppose I have ./test/unit/foo_test.rb and ./test/functional/foo_test.rb.

Typing 'unit/foo' or 'functional/foo' would return one or the other. 
Note that trying to uniquify the paths automatically might create 
abbreviations where the user would try to input a path segment fully 
(not in this example, though).

> One thing it should handle but doesn't yet; when presented with:
>
> foo.el<bar> foo.el<baz> foo.el<car>
>
> The user should be able to type <b to narrow to the first two.

Typing bar/foo is more natural than foo.el<bar. Also quicker.

As an added bonus, if you see an unfamiliar file, you'll see its full 
relative path in the completion interface without having to open it.

> There's already a project-find-file.
>
> There are two similarly named functions; one interactive, the other
> dispatching. EDE uses -impl suffix for the latter; that would work here.

If I'm not mistaken, project-find-file, as currently implemented, will 
go away after you use completion tables as described above.

But if not, find-file-in-project is also an adequate name.

>>> More importantly, the result of the completion is treated differently; the
>>> default instance calls locate-file after rearranging the uniquification,
>>> while the global instance calls ceded-gnu-global-call.
>>
>> Why? If we've identified the requested file, let's just open it.
>
> `completing-read' does _not_ return an absolute or unique path. See below.

Well, it could. I wonder if we shouldn't go at it this way:

project-file-completion-table will return a table which returns absolute 
file (and directory) names.

find-file-in-project calls completing-read with that table. However, we 
could have a completing-read-function (now, or add it at some later 
time) which would display completions is some abbreviated fashion 
(shortening the roots, basically).

As a drawback - it might clash with completing-read-function already 
customized by the user. Not sure how to compose or prioritize them.

Alternatively, find-file-in-project could wrap the returned completion 
table in some "abbreviating" one that abbreviates the project roots in 
file paths when displaying them (and possibly abbreviates some file name 
segments as well). Then, after the selection is made, 
find-file-in-project simply expands the root abbreviation. Or, if some 
segments can be abbreviated too, it polls the wrapping table to get the 
original file path (so the table should maintain some correspondence cache).

Not sure if find-file-in-project should looks inside the whole 
search-path; if so, replace "project roots" with "search dirs" above.

Yet another option is simply to make find-file-in-project search only 
within the current project root.

>> This seems to be a good argument against using base file names, and in
>> favor of using full relative paths against project roots, combined
>> with abbreviated roots.
>
> That would add significant clutter to the UI when filenames collide; you
> often don't need the full path to a root to distinguish them.

You probably mean "don't collide". Yes, there will be some visual 
overhead, but this approach is probably the most flexible WRT different 
project structures, and the user can understand easily what's required 
of them. Less so with foo.el<bar.

> And in the global case, the elisp code has no idea what path global is
> using; it must always call global to get an absolute path.

I don't understand. Can't Global return absolute file paths?

> It would make sense to modify completing-read to be able to return
> meta-information with the completion. I looked at that briefly, and I
> have no idea how to go about it.

It must be doable, but we can manage without it.

> You keep ignoring the fact that in some use cases, the available project
> path information is already flat, and has to be converted to
> recursive/ignores. I presented the AdaCore gpr file example.

Maybe it has to, but not in the API.

> In addition, a mix of flat and recursive/ignore is convenient for users
> constructing a path.

Please, let's avoid that kind of complexity in the API. The recursive 
format is more powerful, so we should use just it.

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 07:59 PM, Stephen Leake wrote:

> I take it back; xref.el xref-find-regexp does exactly that.

Indeed.

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 07:41 PM, Stephen Leake wrote:

> By "root" I guess you mean the argument "dir", since that is an element
> of the recursive project-search-path.

Yes. A root, or a search-path element.

> project-search-path returns (".../dir1" ".../dir2" ".../dir3"); thus

As documented, project-search-path should return absolute file names. 
Though it's not terribly important for this example, since the code 
below could call file-relative-name first.

> project-ignores is only called with one of these three values. (I'm
> ignoring project-roots, because I don't understand when it would be
> different from project-search-path).

Indeed, the logic should be pretty similar. But the external 
project-search-path entries usually doesn't have any ignored 
directories, whereas there are often ignored dirs inside project-roots, 
usually configured via .gitignore or the project file.

> The user wants to ignore "*/*.elc", "*/*.dvi", "*/*.exe",
> "dir2/dir21/obj/*", "dir2/dir21/*.text", "dir3/dir31/*.dvi".
>
> So one implementation of project-ignores could be:
>
> (cond
>   ((string-equal dir ".../dir1")
>    '("*.elc" "*.dvi" "*.exe"))
>
>   ((string-equal dir ".../dir2")
>    '("*.elc" "*.dvi" "*.exe" "./dir2/dir21/obj/" "dir2/dir21/*.text"))
>
>   ((string-equal dir ".../dir3")
>    '("*.elc" "*.dvi" "*.exe" "./dir3/dir31/*.dvi"))
>
> )

I suppose. Or you could simply return the whole list each time, and 
leave it to 'find' to apply the ignores appropriately. Not sure what the 
performance hit will be. There might be false positives as well.

> Then find-file-path-completion-table could take a recursive
> project-search-path as the "path" argument, and recurse thru the
> directory tree, calling project-ignores at each directory, with a parent
> directory from project-search-path as the argument, and comparing the
> path of each file - relative to the parent directory - to the glob
> patterns.

I would just call 'find' for each "root" directory, pass it the 
appropriate ignores each time, and then parse its output.

If you must traverse the trees in Lisp, holding on to the ignores list 
of each tree root seems like a trivial optimization.

> For example, when processing dir2/dir21, it calls (project-ignores
> "dir2"), and compares "./dir2/dir21/obj/" to "dir2/dir21/obj", and
> "*.exe" to "dir2/dir21/foo.exe", both yeilding "ignore".
>
> Is that right?

It could, though the paths will be absolute.

> I gather this was designed to match the semantics of find?

Yes, it's to be compatible with both 'find' and '.gitignore' formats. 
The notion of "anchoring to root" comes from .gitignore.

> Since the compare for directories is different from that for files, it
> would simplify the use of project-ignores (at least for this use case)
> if it was split into project-ignore-files and project-ignore-dirs. Or
> took an arg to specify that.

Well... an entry that doesn't end with a slash, in .gitignore, will 
ignore both files and directories with that name.

So for full compatibility, we'd have to include all ignore-files entries 
into the ignore-dirs entries. That's pretty awkward.

> I was stuck on using a flat path for find-file-path-completion-table
> because I started out trying to reuse locate-file-completion-table. But
> now that I'm implementing the completion function from scratch, and it
> is reading the disk and ignoring some files anyway, it does make more
> sense start with a recursive path.

Thank you. I was hoping you'd come to that conclusion.

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/16/2015 07:41 PM, Stephen Leake wrote:
>
>> By "root" I guess you mean the argument "dir", since that is an element
>> of the recursive project-search-path.
>
> Yes. A root, or a search-path element.
>
>> project-search-path returns (".../dir1" ".../dir2" ".../dir3"); thus
>
> As documented, project-search-path should return absolute file names.

That's what the "..." is supposed to mean. I guess I was not being
"really clear". Sorry.

In the following, let's assume that "..." is "c:/project1".

>> project-ignores is only called with one of these three values. (I'm
>> ignoring project-roots, because I don't understand when it would be
>> different from project-search-path).
>
> Indeed, the logic should be pretty similar. But the external
> project-search-path entries usually doesn't have any ignored
> directories, whereas there are often ignored dirs inside
> project-roots, usually configured via .gitignore or the project file.

You've lost me here.

There is an ignored dir in this example; c:/project1/dir2/dir21/obj/.

Is that supposed to be in the result of project-ignores or not?

Does this mean file name completion should search on project-roots, not
project-search-path, for this use case?

xref-find-regexp searches on both, which is redundant. For example, for
a file in Emacs core, using only project-try-vc in project-find-files,
project-roots returns ("c:/Projects/emacs/master/"), and
project-search-path includes that directory along with others.

As we have discussed, in general a recursive path requires ignored
directories in order to be accurate. So since project-search-path is
recursive, it must always be used together with ignores; that means
project-ignores must return c:/project1/dir2/dir21/obj/ in this example.

>> The user wants to ignore "*/*.elc", "*/*.dvi", "*/*.exe",
>> "dir2/dir21/obj/*", "dir2/dir21/*.text", "dir3/dir31/*.dvi".
>>
>> So one implementation of project-ignores could be:
>>
>> (cond
>>   ((string-equal dir ".../dir1")
>>    '("*.elc" "*.dvi" "*.exe"))
>>
>>   ((string-equal dir ".../dir2")
>>    '("*.elc" "*.dvi" "*.exe" "./dir2/dir21/obj/" "dir2/dir21/*.text"))
>>
>>   ((string-equal dir ".../dir3")
>>    '("*.elc" "*.dvi" "*.exe" "./dir3/dir31/*.dvi"))
>>
>> )
>
> I suppose. 

Sorry, I need "yes" or "no"; this either meets the spec (and intended
semantics) of project-ignores, or it doesn't.

> Or you could simply return the whole list each time, and
> leave it to 'find' to apply the ignores appropriately. Not sure what
> the performance hit will be. There might be false positives as well.

I'm guessing this means "yes, it meets the intended semantics". I did
say "one implementation could be". Optimization can be left for later,
if performance is at all an issue.

>> Then find-file-path-completion-table could take a recursive
>> project-search-path as the "path" argument, and recurse thru the
>> directory tree, calling project-ignores at each directory, with a parent
>> directory from project-search-path as the argument, and comparing the
>> path of each file - relative to the parent directory - to the glob
>> patterns.
>
> I would just call 'find' for each "root" directory, pass it the
> appropriate ignores each time, and then parse its output.

Hmm. I had not considered that approach. It could well be that spawning
"find" is faster than iterating over the disk in elisp.

However, I don't want to rely on "find.exe" on Windows boxes, so I'll
stick with an all elisp solution.

>> For example, when processing dir2/dir21, it calls (project-ignores
>> "dir2"), and compares "./dir2/dir21/obj/" to "dir2/dir21/obj", and
>> "*.exe" to "dir2/dir21/foo.exe", both yeilding "ignore".
>>
>> Is that right?
>
> It could, though the paths will be absolute.

The argument to project-ignores will be "c:/project1/dir2"; I got that
wrong.

I guess you are also saying that the elisp functions for getting a list of
files in a directory will return absolute paths, so this will actually
be comparing "./dir2/dir21/obj/"
to "c:/project1/dir2/dir21/obj". Which can be accomplished by first
expanding the glob pattern relative to the current root.

That would be one way to implement it.

>> Since the compare for directories is different from that for files, it
>> would simplify the use of project-ignores (at least for this use case)
>> if it was split into project-ignore-files and project-ignore-dirs. Or
>> took an arg to specify that.
>
> Well... an entry that doesn't end with a slash, in .gitignore, will
> ignore both files and directories with that name.

Ah, that is what the doc string says; I was misreading it.

>> I was stuck on using a flat path for find-file-path-completion-table
>> because I started out trying to reuse locate-file-completion-table. But
>> now that I'm implementing the completion function from scratch, and it
>> is reading the disk and ignoring some files anyway, it does make more
>> sense start with a recursive path.
>
> Thank you. I was hoping you'd come to that conclusion.

You could have been more constructive in getting there.

-- 
-- Stephe

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/16/2015 04:31 PM, Stephen Leake wrote:
>
>> Except that 'completing-read' does require exactly that. I did try to
>> prepend directories for uniquification; it doesn't work. Or at least it
>> would require major surgery on completing-read to make it work.
>
> You might have to use a different `category' and set up a
> `completion-category-overrides' entry that would only use
> `partial-completion' as its style.
>
> You'll be able input "/file", and it'll match "foo/bar/file.el". The
> completion table will have to handle the `boundaries' action properly,
> though.

Is there documentation on this? The doc strings and elisp manual are not
helpful.

Or a working example?

>> Can you be more specific? What is the file name you are trying to
>> complete to?
>
> Suppose I have ./test/unit/foo_test.rb and ./test/functional/foo_test.rb.
>
> Typing 'unit/foo' or 'functional/foo' would return one or the other.
> Note that trying to uniquify the paths automatically might create
> abbreviations where the user would try to input a path segment fully
> (not in this example, though).

Yes, that's a good use case. I'll add it to the file completion tests.

I'll leave discussion of how to deal with the completing-read result to
the other thread.

>> And in the global case, the elisp code has no idea what path global is
>> using; it must always call global to get an absolute path.
>
> I don't understand. Can't Global return absolute file paths?

Yes, it can. But they don't get returned from completing-read.

You deleted too much context here, so I don't know what you are really
asking. 

>> It would make sense to modify completing-read to be able to return
>> meta-information with the completion. I looked at that briefly, and I
>> have no idea how to go about it.
>
> It must be doable, but we can manage without it.

Please provide code that shows how; I have no idea.

-- 
-- Stephe

Re: find-file-project

[Stefan Monnier]

> You might have to use a different `category' and set
> up a `completion-category-overrides' entry that would only use
> `partial-completion' as its style.

Indeed, the kind of completions you're suggesting will require
such tweaks.
But I also think they'll require a new completion style.

> You'll be able input "/file", and it'll match "foo/bar/file.el".

I don't see how `partial-completion' would do that.
[ I did have some earlier version of partial completion match "**/file"
  to "foo/bar/file.el", but the current code doesn't have that, and even
  less so without a "**".  ]

> Later, we can even add a new, more permissive style. We could use it in
> other places as well.

I'd love to have new completion styles, indeed.

Suggestions:

- combine substring and partial-completion,
  e.g. "a*l/b" can find "cable/rob.el".

- foo/bar should be able to find "tv/football/field/barberis.txt".
  I.e. like `partial-completion' but if "foo" doesn't match anything,
  look for it recursively.

- Same thing but where the order doesn't matter: bar/foo should also
  find "tv/football/field/barberis.txt".  This one should probably
  automatically recurse, even if there is a match in the current directory.

- maybe "bar/foo" should also find "tv/myfootball/field/umbertobarberis.txt".


        Stefan

Re: find-file-project

[Dmitry Gutov]

On 09/16/2015 07:58 PM, Stephen Leake wrote:

>> It's not an Ada project.
>
> That's not a useful definition, and therefore not helpful in moving this
> discussion towards closure.

It's fairly workable, though, as I think we've established.

We could call a project "flat" if, to create a source file in a new 
directory to the program, it's always necessary to add the new directory 
to a build file as well, or to some "directory paths" variable in the 
program's environment.

That doesn't hold true for Emacs Lisp because we can add a file in a 
foo/bar, when only foo is in load-path, and then (require 'foo/bar) 
somewhere.

By that measure, many Makefile-based projects could be called flat. So 
if we take just the part of Emacs written in C, if we add a new dir 
under src/, we'll have to modify Makefile with its settings.

But no one really expects to ever have directories under src/ containing 
irrelevant files. And since we want to be able to visit src/bitmaps/* 
anyway (don't we?), we might as well treat src/ in a recursive fashion 
anyway.

So I rather call a project "flat" if, in addition to the condition 
above, treating its directory list in a recursive fashion leads to 
results surprising to the user.

>> search-path may have entries where one is a subdirectory of another,
>> but each such entry is "recursive" anyway (we have to consider all
>> files inside each respective tree).
>
> That's clearly suboptimal; the subdirectory will be processed twice in
> every use of project-search-path. Why would that ever be a good idea?

Currently, the default implementation of project-search-path calls 
project-prune-directories. We could require it of all implementations, 
so that there's no duplication of this kind.

But then we won't be able to search for "qualified file names", like 
"semantic/symref/cscope.el" (or would do it less accurately). Which 
might be important in "jump to an import" usages. Not in Elisp (we have 
a better implementation of that already), but in other languages.

So maybe a better approach is to allow project-search-path to include 
both "emacs/lisp" and "emacs/lisp/progmodes", and either ask callers to 
use project-prune-directories as needed, or provide an accessor that 
returns the path already pruned (project-search-path-roots?).

> It clearly complicates completion; we have to delete lots of actual
> duplicates, rather than only worrying about files in different
> directories having the same name.

project-prune-directories is a fairly quick function. Compared to a 
serious operation like listing files, it has no performance hit. 
(project-prune-directories load-path) takes 2ms here.

What's so complicated?

> When would such a definition of project-search-path actually be useful?
>
> You are going a long way merely to justify calling "load-path" recursive.

Whether I justify it or not, semantic/symref/cscope.el isn't going 
anywhere. *And* it's even more useful in other languages, like Ruby or Java.

>> I don't see that. If you need this complexity, just use the
>> "recursive" format (it's more powerful),
>
> It is not "more powerful"; we have already agreed that one format can be
> converted accurately to the other; the code in the patch implements those
> conversions.

In theory, probably. But the format of "flat" ignores you've proposed is 
less powerful than the current project-ignores.

Or is project-ignore-files-globs supposed to contain items like 
"foo/*.c"? Or "foo/*/", or "foo/*/bar"?

> You are missing my main point; the user must specify information in
> addition to "load-path" in order to determine the correct search path
> for an elisp project.

I don't think we really can avoid that. Your definition of the Elisp 
project will fail in some project we don't know about that also uses 
nesting and the format (require 'foo/bar/tee).

> Whether that additional information is specified as "extra includes" or
> "extra ignores" is a secondary issue.

One also has to look at the actual language they're working with.

> Yes, that is the user intent. The question is, how is the user supposed
> to specify that intent to the elisp project? "load-path" alone does not
> state whether it should be ignored; the user must provide additional
> information.

And now you've hardcoded the same information in the project definition. 
Which will work most of the time, and then it'll fail for someone, for 
the reason described above.

You could argue that "most of the time" is good enough, but note that 
the cogre problem you're trying to solve is pretty niche as well. I may 
never encounter any similar problems with Elisp.

And adding a single ignore is pretty easy.

Re: find-file-project

[Dmitry Gutov]

On 09/17/2015 12:01 AM, Stephen Leake wrote:

> That's what the "..." is supposed to mean. I guess I was not being
> "really clear". Sorry.

In see. In hindsight, it should've been obvious to me, though.

> In the following, let's assume that "..." is "c:/project1".

Ok.

>> Indeed, the logic should be pretty similar. But the external
>> project-search-path entries usually doesn't have any ignored
>> directories, whereas there are often ignored dirs inside
>> project-roots, usually configured via .gitignore or the project file.
>
> You've lost me here.

You've asked about the difference between project-roots and 
project-search-path. Anyway, it's not too important here.

> There is an ignored dir in this example; c:/project1/dir2/dir21/obj/.
>
> Is that supposed to be in the result of project-ignores or not?

Sure.

> Does this mean file name completion should search on project-roots, not
> project-search-path, for this use case?

Err, that looks like a non-sequitur to me. project-find-file should 
search in where it's defined to search (up to you). It just occurs to me 
that some users might expect it to only list files in the currently 
opened project(s), whereas other users would like to jump to system 
include files as well.

> xref-find-regexp searches on both, which is redundant. For example, for
> a file in Emacs core, using only project-try-vc in project-find-files,
> project-roots returns ("c:/Projects/emacs/master/"), and
> project-search-path includes that directory along with others.

It's kinda redundant, but as long we define project-search-path as a 
list of directories with *source* files, it may omit the root 
directories if they contain only, say, documentation. But we want 
xref-find-regexp to search in documentation as well.

> As we have discussed, in general a recursive path requires ignored
> directories in order to be accurate. So since project-search-path is
> recursive, it must always be used together with ignores; that means

I don't see your point. Yes, ignores should be used, but only when there 
are ignores. "Flat" search path will have to use ignores as well 
("dir3/dir31/*.dvi" is a good example).

> project-ignores must return c:/project1/dir2/dir21/obj/ in this example.

You can have it return the absolute files names, but it's quicker to 
relative, "rooted" ignores.

> Sorry, I need "yes" or "no"; this either meets the spec (and intended
> semantics) of project-ignores, or it doesn't.

I would like to amend my answer: no, it's incorrect.

(project-ignores ".../dir3") should return ignores relative to that 
directory. So "./dir3/dir31/*.dvi" would be "./dir31/*.dvi" instead.

Or return absolute file names, they would work as well.

>> Or you could simply return the whole list each time, and
>> leave it to 'find' to apply the ignores appropriately. Not sure what
>> the performance hit will be. There might be false positives as well.
>
> I'm guessing this means "yes, it meets the intended semantics".

Aside from the detail above, yes.

> Hmm. I had not considered that approach. It could well be that spawning
> "find" is faster than iterating over the disk in elisp.

It's been tried, and it's definitely faster, especially for larger projects.

> However, I don't want to rely on "find.exe" on Windows boxes, so I'll
> stick with an all elisp solution.

I would urge you to reconsider: we assume the presence of 'find' 
already. M-x rgrep is probably the most common command that relies on 
it. Or M-x find-dired.

> I guess you are also saying that the elisp functions for getting a list of
> files in a directory will return absolute paths, so this will actually
> be comparing "./dir2/dir21/obj/"
> to "c:/project1/dir2/dir21/obj". Which can be accomplished by first
> expanding the glob pattern relative to the current root.
>
> That would be one way to implement it.

Yes, if you insist on processing it in Lisp.

>> Thank you. I was hoping you'd come to that conclusion.
>
> You could have been more constructive in getting there.

If you mean the small jab about Ada, I grew impatient about having to 
reiterate what we've already established about Emacs Lisp. Sorry?

Or do you mean something else? I've suggested to use completion-at-point 
from the outset.

Re: find-file-project

[Dmitry Gutov]

On 09/17/2015 12:11 AM, Stephen Leake wrote:

> Is there documentation on this? The doc strings and elisp manual are not
> helpful.

It's unfortunately under-documented, so one would have to work from the 
existing examples in minibuffer.el

>> I don't understand. Can't Global return absolute file paths?
>
> Yes, it can. But they don't get returned from completing-read.
>
> You deleted too much context here, so I don't know what you are really
> asking.

The issue was about why the Global project type has to define its own 
find-file generic function.

>>> It would make sense to modify completing-read to be able to return
>>> meta-information with the completion. I looked at that briefly, and I
>>> have no idea how to go about it.
>>
>> It must be doable, but we can manage without it.
>
> Please provide code that shows how; I have no idea.

I haven't actually written any completion styles myself, so Stefan would 
be a better source on the whys and hows of it. But using 
completion-at-point has well-known advantages, so we really want to go 
this way.

If you get stuck, please push the WIP to the feature branch, and I'll 
try to work it out as well.

Re: find-file-project

[Dmitry Gutov]

On 09/17/2015 04:26 AM, Stefan Monnier wrote:

> Indeed, the kind of completions you're suggesting will require
> such tweaks.

It brings up a related issue: `category' is a metadata element. Do we 
really want to require every project-file-completion-table 
implementation to include that piece of metadata?

Being able to assign it to the returned table in find-file-in-project 
would be handy.

>> You'll be able input "/file", and it'll match "foo/bar/file.el".
>
> I don't see how `partial-completion' would do that.

The same way as "-firefox" expands to "browse-url-firefox"?

It doesn't work with 'find-file', but maybe only because 
completion-file-name-table doesn't interpret "//" as "/**/". It doesn't 
interpret "/**/" properly either (only treats it as "/*/").

> [ I did have some earlier version of partial completion match "**/file"
>    to "foo/bar/file.el", but the current code doesn't have that, and even
>    less so without a "**".  ]

Why not? Anyway, I was talking about implementing a new completion 
table, nor necessarily reusing completion-file-name-table.

> Suggestions:
>
> - combine substring and partial-completion,
>    e.g. "a*l/b" can find "cable/rob.el".

If globs weren't trivial to implement here, honestly, I would leave the 
support for them altogether.

> - foo/bar should be able to find "tv/football/field/barberis.txt".
>    I.e. like `partial-completion' but if "foo" doesn't match anything,
>    look for it recursively.

Yes.

> - Same thing but where the order doesn't matter: bar/foo should also
>    find "tv/football/field/barberis.txt".  This one should probably
>    automatically recurse, even if there is a match in the current directory.

I rather leave that for a separate style, too permissive for my taste.

> - maybe "bar/foo" should also find "tv/myfootball/field/umbertobarberis.txt".

Why not. Although maybe I'd rather only match 'foo/bar', but not 
'bar/foo'. We might want a knob here.

project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/17/2015 12:01 AM, Stephen Leake wrote:
>
>> Does this mean file name completion should search on project-roots, not
>> project-search-path, for this use case?
>
> Err, that looks like a non-sequitur to me. project-find-file should
> search in where it's defined to search (up to you). It just occurs to
> me that some users might expect it to only list files in the currently
> opened project(s), whereas other users would like to jump to system
> include files as well.
>
>> xref-find-regexp searches on both, which is redundant. For example, for
>> a file in Emacs core, using only project-try-vc in project-find-files,
>> project-roots returns ("c:/Projects/emacs/master/"), and
>> project-search-path includes that directory along with others.
>
> It's kinda redundant, but as long we define project-search-path as a
> list of directories with *source* files, it may omit the root
> directories if they contain only, say, documentation. But we want
> xref-find-regexp to search in documentation as well.

Both find-file-in-project and xref-find-regexp are client code using the
project API.

In order to use the API correctly, the semantics of each call must be
well-defined.

That means it must be possible to tell from the API as defined by
project.el (and some future project.texi) what functions to call for
these two uses. They cannot rely on vague guesses about what some
backend might do, or what the user might want.

That means project-search-path should be clearly defined to be either
purely recursive or not, so client code and project backends know
whether to weed out duplicates or not.

It also means the distinction between project-search-path and
project-roots must be more clearly defined.

If there are use cases where different search paths are desired for
different purposes (such as "source" vs "documentation" above), the
the project API must provide a clear mechanism to distinguish those
cases, so the appropriate search path can be returned.

The current doc strings for project-search-path and project-roots do not
make that clear, and the way they are used and implemented in current
code makes project-roots purely redundant (no code would be harmed by
simply deleting it, and some would be improved thru reduced redundancy).

-- 
-- Stephe

Re: find-file-project

[Stefan Monnier]

>> Indeed, the kind of completions you're suggesting will require
>> such tweaks.
> It brings up a related issue: `category' is a metadata element.  Do we
> really want to require every project-file-completion-table
> implementation to include that piece of metadata?
> Being able to assign it to the returned table in find-file-in-project
> would be handy.

Right.

Of course, project-find-file could wrap the backend's completion table
so as to add its own `category' info to the meatadata, but
it's cumbersome.

We should try and unify the metadata returned by the completion-tables
and the metadata returned by the completion-at-point-functions (or
provided via completion-extra-properties).

>>> You'll be able input "/file", and it'll match "foo/bar/file.el".
>> I don't see how `partial-completion' would do that.
> The same way as "-firefox" expands to "browse-url-firefox"?

Ah, I see.  But that means all the files are returned as one flat list
(without using completion-boundaries).  Might be a good idea.

>> [ I did have some earlier version of partial completion match "**/file"
>> to "foo/bar/file.el", but the current code doesn't have that, and even
>> less so without a "**".  ]
> Why not?

Couldn't find that 25th hour!

>> - Same thing but where the order doesn't matter: bar/foo should also
>> find "tv/football/field/barberis.txt".  This one should probably
>> automatically recurse, even if there is a match in the current directory.
> I rather leave that for a separate style, too permissive for my taste.

Agreed, but occasionally I need such things, and that's where the *
globs come in handy ;-)


        Stefan

project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> Currently, the default implementation of project-search-path calls
> project-prune-directories. We could require it of all implementations,
> so that there's no duplication of this kind.
>
> But then we won't be able to search for "qualified file names", like
> "semantic/symref/cscope.el" (or would do it less accurately). Which
> might be important in "jump to an import" usages. Not in Elisp (we
> have a better implementation of that already), but in other languages.

I'm guessing you are talking about something like a new function
project-find-file-at-point that does the following:

    In a C file, if point is in an #include line, goto the beginning of
    a buffer containing the corresponding file.

And similarly for other languages.

In that case, the string in the #include could be something like
"dir1/dir2/file1.h", or just "file2.h".

If project-search-path is a superset of the C include path, we can
iterate thru project-search-path, checking if (expand-file-name
include-string dir) exists.

However, if the C include path is not purely recursive (which is
likely), then a purely recursive project-search-path will not be a
superset, and this will fail (so it will fail with all of the current
implementations of project-search-path).

That's an argument for providing project-flat-search-path (overkill for
this use case, but not harmfull), or project-c-include-search-path
(precisely what this use case wants), or an argument to
project-search-path telling it what sort of path to return.

On the other hand, find-file-in-project and xref-search-regexp want
purely recursive search paths.

EDE defines at least two kinds of paths; include and link, which have
different content.

Different use cases have different requirements for the semantics of
"search path". It will not be possible for a single function to meet all
the required semantics.

You suggest that we should define project-search-path loosely ("let the
backend do whatever it wants"), and let the client code cope ("just call
project-prune-directories").

But the API is the interface between the clients and the backend; it
would be much better to have clearer semantics, and options in the API
to choose among the possible semantics. That way client code can be
written that is truly independent of the project backends, and new
backends won't break existing client code.

I believe all the use cases we've discussed so far can be met with
reasonable efficency by providing both purely recursive and purely flat
search paths.

On the other hand, if we define project-search-path to return
mixed-recursive (to match load-path or C include exactly), some client
functions may be able to use it as is (they will have to make
assumptions about what the backend is including). Those that want pure
recursive can call project-prune-directories on the result, and those
that want flat can call project-recursive-ignores-to-flat.

We should start a file that documents the use cases we've discussed, so
we don't forget them.

-- 
-- Stephe

Re: find-file-project

[Dmitry Gutov]

On 09/18/2015 08:07 PM, Stefan Monnier wrote:

> Of course, project-find-file could wrap the backend's completion table
> so as to add its own `category' info to the meatadata, but
> it's cumbersome.

Indeed.

> We should try and unify the metadata returned by the completion-tables
> and the metadata returned by the completion-at-point-functions (or
> provided via completion-extra-properties).

Yup. We should probably do that sooner rather than later.

> Ah, I see.  But that means all the files are returned as one flat list
> (without using completion-boundaries).  Might be a good idea.

Ouch. That means I really misunderstand how completion-boundaries work.

>>> [ I did have some earlier version of partial completion match "**/file"
>>> to "foo/bar/file.el", but the current code doesn't have that, and even
>>> less so without a "**".  ]
>> Why not?
>
> Couldn't find that 25th hour!

Well, I meant that if earlier code already did it, you had to remove it. 
But I guess there was more work to be done, to make it good enough for 
public.

Re: project.el semantics

[Dmitry Gutov]

On 09/18/2015 08:14 PM, Stephen Leake wrote:

> I'm guessing you are talking about something like a new function
> project-find-file-at-point that does the following:
>
>      In a C file, if point is in an #include line, goto the beginning of
>      a buffer containing the corresponding file.

Pretty much, yes. But it would not necessarily use the content at point. 
Input with completion is also possible, very much in line with 
find-file-in-project discussed currently, with small differences.

> And similarly for other languages.
>
> In that case, the string in the #include could be something like
> "dir1/dir2/file1.h", or just "file2.h".

<dir1/dir2/file1.h>, actually. #include with double-quotes has a 
different meaning (those are relative includes, much easier to handle in 
a different way).

> If project-search-path is a superset of the C include path, we can
> iterate thru project-search-path, checking if (expand-file-name
> include-string dir) exists.

Right.

> However, if the C include path is not purely recursive (which is
> likely),

What is this guessing? I'm pretty sure it *is* "purely recursive", and 
if not, please quote some documentation.

And by "purely recursive" here, I mean that if /foo/bar is in 
C_INCLUDE_PATH, and foo.c contains "#include <tee/qux.h>", and 
/foo/bar/tee/qux.h exists, the compiler (or preprocessor, etc) *will* 
try to load it when compiling foo.c.

Hence, any file inside subtrees of C_INCLUDE_PATH is include-able, and 
jumping to any of them makes sense.

> then a purely recursive project-search-path will not be a
> superset, and this will fail (so it will fail with all of the current
> implementations of project-search-path).

I'm pretty sure that even if that were the case, it could be a 
good-enough feature, just one with undesirable quirks.

> That's an argument for providing project-flat-search-path (overkill for
> this use case, but not harmfull),

I don't think it is. Please, let's cease the flat-search-path discussion 
until we encounter a use case that's *really* hard to handle without it.

So far, I'm sorry, it simply looks like you're pushing your own 
preference. Which is fine to a certain extent, but really goes against 
my intent of having an easy-to-understand API.

> or project-c-include-search-path
> (precisely what this use case wants), or an argument to
> project-search-path telling it what sort of path to return.

Maybe we indeed should have two different generic functions, one for 
"source roots" (that doesn't return any two dirs where one is inside the 
other), and another that must return a list of directories exactly along 
the lines of C_INCLUDE_PATH. But let's not call it c-something, other 
languages have similar concepts (class path in Java, load path in Ruby).

Opinions welcome.

> On the other hand, find-file-in-project and xref-search-regexp want
> purely recursive search paths.

Yes. Although I wouldn't call C_INCLUDE_PATH "flat". It's just redundant 
for the above commands.

> EDE defines at least two kinds of paths; include and link, which have
> different content.

Could you point me to the documentation for this? There's no occurrences 
of "link" in ede/base.el.

> Different use cases have different requirements for the semantics of
> "search path". It will not be possible for a single function to meet all
> the required semantics.

You might be right.

> You suggest that we should define project-search-path loosely ("let the
> backend do whatever it wants"), and let the client code cope ("just call
> project-prune-directories").

Not what it wants, just within the documented leeway. I admit returning 
redundant elements is not documented, but for now it's just an idea, not 
something set in stone. Eventually it'll need to be documented, 
whichever way the decision comes out.

> But the API is the interface between the clients and the backend; it
> would be much better to have clearer semantics, and options in the API
> to choose among the possible semantics. That way client code can be
> written that is truly independent of the project backends, and new
> backends won't break existing client code.

100% agreement, except for "options in the API to choose among the 
different semantics". It's better to only have to deal with one 
semantics, as long as the result can be just as powerful.

> On the other hand, if we define project-search-path to return
> mixed-recursive (to match load-path or C include exactly), some client
> functions may be able to use it as is (they will have to make
> assumptions about what the backend is including).

The statement in parens above seems like a recipe for disaster. How 
would a function know what the backend is including? Will it have to 
provide different implementations for different backends? Remember, 
we're striving for "backend-agnostic" here.

> We should start a file that documents the use cases we've discussed, so
> we don't forget them.

Any suggestions for the collaboration platform?

Re: project.el semantics

[Dmitry Gutov]

On 09/18/2015 07:08 PM, Stephen Leake wrote:

> In order to use the API correctly, the semantics of each call must be
> well-defined.

No argument here.

> That means it must be possible to tell from the API as defined by
> project.el (and some future project.texi) what functions to call for
> these two uses. They cannot rely on vague guesses about what some
> backend might do, or what the user might want.

Yes, we need more documentation. Please simply consider the grand-parent 
email as bringing up an example of another use-case we might want to 
support.

> That means project-search-path should be clearly defined to be either
> purely recursive or not, so client code and project backends know
> whether to weed out duplicates or not.

Let's call it "redundant", or "non-redundant". The issue seems different 
from the "flat" search-path we've discussed previously.

> It also means the distinction between project-search-path and
> project-roots must be more clearly defined.

Sure. We need more clarity WRT to redundancy in the search path, maybe 
better docstrings (although it's not immediately clear to me how to 
improve the current wording), and some good examples in the manual.

> If there are use cases where different search paths are desired for
> different purposes (such as "source" vs "documentation" above), the
> the project API must provide a clear mechanism to distinguish those
> cases, so the appropriate search path can be returned.

But what are those different purposes? So far, the distinction as I 
understand it is that search-path contains actual source files, and 
project-roots can contain anything else. So it's just source files vs 
any files.

> The current doc strings for project-search-path and project-roots do not
> make that clear, and the way they are used and implemented in current
> code makes project-roots purely redundant (no code would be harmed by
> simply deleting it, and some would be improved thru reduced redundancy).

The default implementations don't have enough information to do better. 
Would you delete the default project-search-path implementation? Then 
what will the VC-backend implementation of it look like?

Re: find-file-project

[Stefan Monnier]

>> We should try and unify the metadata returned by the completion-tables
>> and the metadata returned by the completion-at-point-functions (or
>> provided via completion-extra-properties).
> Yup.  We should probably do that sooner rather than later.

Yes.

>>>> [ I did have some earlier version of partial completion match "**/file"
>>>> to "foo/bar/file.el", but the current code doesn't have that, and even
>>>> less so without a "**".  ]
>>> Why not?
>> Couldn't find that 25th hour!
> Well, I meant that if earlier code already did it, you had to remove it. But
> I guess there was more work to be done, to make it good enough for public.

The current code is a complete rewrite and works differently (in the
previous code, the "boundaries" part didn't exist and the * and ** was
implemented inside the completion-table).


        Stefan

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 09/18/2015 08:14 PM, Stephen Leake wrote:
>
>> However, if the C include path is not purely recursive (which is
>> likely),
>
> What is this guessing? I'm pretty sure it *is* "purely recursive", and
> if not, please quote some documentation.

"purely recursive" means no element of the path is a subdirectory of
another element.

It is quite possible for a project to have an include path like
"root/include/dir1 root/include/dir1/dir2".

>> then a purely recursive project-search-path will not be a
>> superset, and this will fail (so it will fail with all of the current
>> implementations of project-search-path).
>
> I'm pretty sure that even if that were the case, it could be a
> good-enough feature, just one with undesirable quirks.

Not a good advertisement for the project API. If I saw that statement in
the project user guide, I'd stop reading, and go learn about EDE.

Failure to go to an include file would certainly be "undesireable". I
don't think "quirk" is the right term, though; "bug" and "bad design"
are more appropriate.

>> That's an argument for providing project-flat-search-path (overkill for
>> this use case, but not harmfull),
>
> I don't think it is. 

Interesting. First you admit that your approach is wrong, then you deny
that another approach would work, without any proof.

> Please, let's cease the flat-search-path
> discussion until we encounter a use case that's *really* hard to
> handle without it.

We already have, at least twice. But you refuse to listen, so yes, I'll
stop trying.

> I admit
> returning redundant elements is not documented, but for now it's just
> an idea, not something set in stone. Eventually it'll need to be
> documented, whichever way the decision comes out.

What's wrong with picking one now, and documenting that? That at least
gives us a better defined base to start from.

>> But the API is the interface between the clients and the backend; it
>> would be much better to have clearer semantics, and options in the API
>> to choose among the possible semantics. That way client code can be
>> written that is truly independent of the project backends, and new
>> backends won't break existing client code.
>
> 100% agreement, except for "options in the API to choose among the
> different semantics". It's better to only have to deal with one
> semantics, as long as the result can be just as powerful.

Interesting; how to say "yes" and "no" in the same paragraph.

>> On the other hand, if we define project-search-path to return
>> mixed-recursive (to match load-path or C include exactly), some client
>> functions may be able to use it as is (they will have to make
>> assumptions about what the backend is including).
>
> The statement in parens above seems like a recipe for disaster. How
> would a function know what the backend is including? Will it have to
> provide different implementations for different backends? Remember,
> we're striving for "backend-agnostic" here.

Precisely my point; I'm glad you agree. So why do you refuse to fix the
documentation to be clear?

>> We should start a file that documents the use cases we've discussed, so
>> we don't forget them.
>
> Any suggestions for the collaboration platform?

savannah Emacs git.

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 09/19/2015 03:07 PM, Stephen Leake wrote:

> "purely recursive" means no element of the path is a subdirectory of
> another element.

Let's call it "non-redundant".

> It is quite possible for a project to have an include path like
> "root/include/dir1 root/include/dir1/dir2".

Yes.

>> I'm pretty sure that even if that were the case, it could be a
>> good-enough feature, just one with undesirable quirks.
>
> Not a good advertisement for the project API. If I saw that statement in
> the project user guide, I'd stop reading, and go learn about EDE.

Sure, EDE has no quirks at all.

> Failure to go to an include file would certainly be "undesireable". I
> don't think "quirk" is the right term, though; "bug" and "bad design"
> are more appropriate.

Why do you think "failure to go to an include file" would be among those 
quirks?

Rather, I would imagine that if #include didn't allow files in 
subdirectories, the main problem with the find-file-in-includes function 
that was implemented as if it did, would be that it would allow to go to 
*too* many files, not too few. So, false positives, not false negatives.

>>> That's an argument for providing project-flat-search-path (overkill for
>>> this use case, but not harmfull),
>>
>> I don't think it is.
>
> Interesting. First you admit that your approach is wrong, then you deny
> that another approach would work, without any proof.

Hence the word "think". But how would it help? Redundant or not, 
C_INCLUDE_PATH is "recursive" like I've described in the previous 
message. How would a "project-flat-search-path" relate to it?

>> I admit
>> returning redundant elements is not documented, but for now it's just
>> an idea, not something set in stone. Eventually it'll need to be
>> documented, whichever way the decision comes out.
>
> What's wrong with picking one now, and documenting that? That at least
> gives us a better defined base to start from.

On the one hand, project-search-path being non-redundant would be more 
useful for the currently implemented commands, as well as 
find-file-in-project.

One the other hand, if we add a second generic in the future, 
project-search-path seems like a better name for the *redundant* one of 
the two. Don't you think?

But changing this generic's semantics later might not be an option anymore.

>> 100% agreement, except for "options in the API to choose among the
>> different semantics". It's better to only have to deal with one
>> semantics, as long as the result can be just as powerful.
>
> Interesting; how to say "yes" and "no" in the same paragraph.

That's how I read that paragraph: "Sure, naturally, yeah... waaait a 
minute!"

>> The statement in parens above seems like a recipe for disaster. How
>> would a function know what the backend is including? Will it have to
>> provide different implementations for different backends? Remember,
>> we're striving for "backend-agnostic" here.
>
> Precisely my point;

...OK?

> I'm glad you agree. So why do you refuse to fix the
> documentation to be clear?

Because if we "fix" it that way now, we might decide that the matter is 
closed. Whereas a proper fix shouldn't be too hard, and with 
find-file-in-project implemented, find-file-in-includes would be just 
one step away.

> savannah Emacs git.

Fair enough. Somewhere in etc/?

Re: project.el semantics

[Dmitry Gutov]

Hi Stephen,

On 09/18/2015 07:08 PM, Stephen Leake wrote:

> That means project-search-path should be clearly defined to be either
> purely recursive or not, so client code and project backends know
> whether to weed out duplicates or not.
>
> It also means the distinction between project-search-path and
> project-roots must be more clearly defined.

Please see the commit 9776972 inside the branch project-next. I intend 
to merge it into master in a day or two.

Sorry to stir things around with the rename, but I think it's justified.

> If there are use cases where different search paths are desired for
> different purposes (such as "source" vs "documentation" above), the
> the project API must provide a clear mechanism to distinguish those
> cases, so the appropriate search path can be returned.

Also see the FIXME commentary above project-library-roots-function; it's 
waiting for the public opinion. Though it's not really about source vs 
documentation as much as about different kinds of sources.

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> Please see the commit 9776972 inside the branch project-next. I intend
> to merge it into master in a day or two.

The new project.el has:

(defvar project-library-roots-function 'etags-library-roots
  "Function that returns a list of library roots.

It should return a list of directories that contain source files
related to the current buffer.  Depending on the language, it
should include the headers search path, load path, class path,
and so on.

And later:

(cl-defgeneric project-library-roots (project)
  "Return the list of source root directories.

It's the list of directories outside of the current project that
contain related source files.

Project-specific version of `project-library-roots-function',
which see.  Unless it knows better, a specialized implementation
should use the value returned by that function."


These are inconsistent.

It would be best for the project-library-roots-function doc string to
_only_ refer to cl-defgeneric (ie, "default implementation of
`project-libary-roots'"), so you don't have duplicate documentation that
gets out of sync.


And then:

(cl-defgeneric project-roots (project)
  "Return the list of directory roots belonging to the current project.

Most often it's just one directory, which contains the project
file and everything else in the project.  But in more advanced
configurations, a project can span multiple directories.

The rule of tumb for whether to include a directory here, and not
in `project-library-roots', is whether its contents are meant to
be edited together with the rest of the project.

The directory names should be absolute.")


You seem to be trying to establish a difference between "a single
project" and "the libraries used by a project". Better terminology would
be "top level project" and "dependencies"; not all dependencies are
libraries. It would be helpful to make this distinction more explicit.

The default implementation of project-library-roots makes the lists
disjoint, so the doc strings should say that. In particular,
"load path" and "class path" contain _both_ the top level project and
the dependencies, so project-library-roots should _not_ be the same as
"load path" or "class path".

The advice in project-roots on "include here but not there" should
mention top-level vs dependencies.

On the other hand, `project-library-roots ((project (head vc))' does
_not_ make them disjoint, so there is a bug here somewhere.


The new function `project-find-regexp' is not consistent with other
usage; other project- search functions search both top level and
dependencies.


> Also see the FIXME commentary above project-library-roots-function;
> it's waiting for the public opinion. Though it's not really about
> source vs documentation as much as about different kinds of sources.

That FIXME: says, in part:

;; FIXME: Using the current approach, we don't have access to the
;; "library roots" of language A from buffers of language B, which
;; seems desirable in multi-language projects, at least for some
;; potential uses, like "jump to a file in project or library".

emacs-lisp-mode makes project-library-roots-function buffer-local,
which makes it language-specific. That's an argument for overriding the
default implementation of project-library-roots to do something more
useful. Which is what `project-library-roots ((project (head vc))' does,
for example. Which means that the behavior of projects in an elisp file
that happens to be in a vc directory is different from one that is not. 

No other language mode sets project-library-roots-function.

The root cause of this problem is trying to infer a project in an elisp
file. This makes sense in general, because an elisp file implies the use
of load-path, which is the main part of defining a project. A better way
is to provide a project-find-function that returns an elisp-project
object in elisp buffers; then elisp-project can override
project-library-roots to return load-path; it could also add the emacs C
sources if they are available.

That leaves the default behavior in non-elisp files that are also not in
a vc directory; they currently use 'etags-library-roots. You could
change the default definition of project-library-roots to use
etags-library-roots directly, but it seems better to have a variable for
this.

I would delete the advice that overriding functions should use
project-library-roots-function; the main reason to override is to do
something different.


I have no problems merging this branch to master; it makes things better
in most places, and no worse in any. Howver, it would be best to clean
up the inconsistencies above first.

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/08/2015 09:11 AM, Stephen Leake wrote:

> These are inconsistent.
>
> It would be best for the project-library-roots-function doc string to
> _only_ refer to cl-defgeneric (ie, "default implementation of
> `project-libary-roots'"), so you don't have duplicate documentation that
> gets out of sync.

It would be impossible for project-library-roots-function to implement 
the "outside of the current project" part of semantics, unless we pass 
it the current project.

And then, what's the point? It's just as easy to call 
project-subtract-directories from the default project-library-roots impl.

Any other inconsistencies? (I've pushed a minor update now, but it's 
probably unrelated to your concerns).

> You seem to be trying to establish a difference between "a single
> project" and "the libraries used by a project". Better terminology would
> be "top level project" and "dependencies"; not all dependencies are
> libraries. It would be helpful to make this distinction more explicit.

I think we might want to introduce the notion of project dependencies 
later, and then we'd have to call some of library-roots, again, library 
roots, to distinguish from the dependencies.

Better to defer that change.

Note that you currently have the freedom to include the checked out 
dependent projects in the project-roots (if you edit these projects 
together).

> The default implementation of project-library-roots makes the lists
> disjoint, so the doc strings should say that.

Doesn't it? It says "outside".

> In particular,
> "load path" and "class path" contain _both_ the top level project and
> the dependencies, so project-library-roots should _not_ be the same as
> "load path" or "class path".

I think "list of directories outside of the project" is pretty clear.

> The advice in project-roots on "include here but not there" should
> mention top-level vs dependencies.

Could you rephrase that? I'm not sure I follow.

> On the other hand, `project-library-roots ((project (head vc))' does
> _not_ make them disjoint, so there is a bug here somewhere.

Fixed, thanks for pointing that out.

> The new function `project-find-regexp' is not consistent with other
> usage; other project- search functions search both top level and
> dependencies.

Sorry, what other project- search functions? Do you mean 
xref-find-references?

Note that we also have project-or-libraries-find-regexp. IME, when doing 
a regexp search across the project, you aren't interested in the library 
code, most of the time.

>> Also see the FIXME commentary above project-library-roots-function;
>> it's waiting for the public opinion. Though it's not really about
>> source vs documentation as much as about different kinds of sources.
>
> That FIXME: says, in part:
>
> ;; FIXME: Using the current approach, we don't have access to the
> ;; "library roots" of language A from buffers of language B, which
> ;; seems desirable in multi-language projects, at least for some
> ;; potential uses, like "jump to a file in project or library".
>
> emacs-lisp-mode makes project-library-roots-function buffer-local,
> which makes it language-specific. That's an argument for overriding the
> default implementation of project-library-roots to do something more
> useful. Which is what `project-library-roots ((project (head vc))' does,
> for example. Which means that the behavior of projects in an elisp file
> that happens to be in a vc directory is different from one that is not.

`project-library-roots ((project (head vc))' knows nothing of library 
roots, however, aside from the local variable that's supposed to be set 
by the user.

> No other language mode sets project-library-roots-function.

There is etags-library-roots, however. It's not language-specific, but, 
again, vc-project doesn't know about it.

There is also ruby-library-roots already brewing in my head.

> The root cause of this problem is trying to infer a project in an elisp
> file. This makes sense in general, because an elisp file implies the use
> of load-path, which is the main part of defining a project. A better way
> is to provide a project-find-function that returns an elisp-project
> object in elisp buffers;

That implies having to create a separate project implementation for 
every language, making vc-project utterly useless.

You'd also have to create yet-another kind of project implementations, 
for multi-language projects.

Note that determining whether the a given directory tree has Elisp files 
inside is not 100% reliable, outside of traversing it whole. But if we 
pick some detection mechanism, (and we'd have to, for Elisp-project 
detection), we can just as well use it to call elisp-library-roots 
globally and make it auto-detect whether the current project is 
applicable. Like described in the FIXME.

> then elisp-project can override
> project-library-roots to return load-path; it could also add the emacs C
> sources if they are available.

At the moment, I feel that this approach would be a cop-out. But it's 
definitely an option.

> That leaves the default behavior in non-elisp files that are also not in
> a vc directory; they currently use 'etags-library-roots. You could
> change the default definition of project-library-roots to use
> etags-library-roots directly, but it seems better to have a variable for
> this.

I don't think we ever call project-library-roots-function from anywhere 
but project-library-roots (*). And that requires the presence of the 
current project.

> I would delete the advice that overriding functions should use
> project-library-roots-function; the main reason to override is to do
> something different.

(*) Hence, if we go your way, there seems to be no need for 
project-library-roots-function.

> I have no problems merging this branch to master; it makes things better
> in most places, and no worse in any. Howver, it would be best to clean
> up the inconsistencies above first.

Thanks, I'll merge after your next reply.

Re: project.el semantics

[Dmitry Gutov]

On 11/08/2015 03:07 PM, Dmitry Gutov wrote:
> On 11/08/2015 09:11 AM, Stephen Leake wrote:
>> emacs-lisp-mode makes project-library-roots-function buffer-local,
>> which makes it language-specific. That's an argument for overriding the
>> default implementation of project-library-roots to do something more
>> useful. Which is what `project-library-roots ((project (head vc))' does,
>> for example. Which means that the behavior of projects in an elisp file
>> that happens to be in a vc directory is different from one that is not.
>
> `project-library-roots ((project (head vc))' knows nothing of library
> roots, however, aside from the local variable that's supposed to be set
> by the user.
> ...
>> The root cause of this problem is trying to infer a project in an elisp
>> file. This makes sense in general, because an elisp file implies the use
>> of load-path, which is the main part of defining a project. A better way
>> is to provide a project-find-function that returns an elisp-project
>> object in elisp buffers;
>
> That implies having to create a separate project implementation for
> every language, making vc-project utterly useless.

I can also propose another change: if project-library-roots-function, in 
your opinion, complicates the project API unnecessarily, we can fold it 
into project-vc-library-roots, for now (and make that variable a hook).

In practice, other project implementation would still be able to use it, 
and if we see that happen, we can promote it formally to a variable 
independent from vc-project (with a rename). We'd have to figure out 
which of the roots to visit to read .dir-locals.el, though.

In this scenario, project-vc-library-roots would be allowed to contain 
both strings and functions, and the functions will normally be put into 
the global value.

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/08/2015 09:11 AM, Stephen Leake wrote:

>> You seem to be trying to establish a difference between "a single
>> project" and "the libraries used by a project". Better terminology would
>> be "top level project" and "dependencies"; not all dependencies are
>> libraries. It would be helpful to make this distinction more explicit.
>
> I think we might want to introduce the notion of project dependencies
> later, and then we'd have to call some of library-roots, again,
> library roots, to distinguish from the dependencies.

I don't understand; you seem to be saying a "library" is _not_ a
"dependency". That's not consistent with common usage.

"Dependency" includes system libraries, other managed projects, and
anything else the user wants to search.

Why do you insist on the less general term?

> Note that you currently have the freedom to include the checked out
> dependent projects in the project-roots (if you edit these projects
> together).

The user has the freedom to do whatever they want. The point is to have
clearly defined semantics, so the user at least knows what the intent of
each of these functions is.

>
>
>> The default implementation of project-library-roots makes the lists
>> disjoint, so the doc strings should say that.
>
> Doesn't it? It says "outside".

True; but I did not understand what you meant by that; it's not common
project terminology. I think "dependency" is much more widely
understood. 

>> In particular,
>> "load path" and "class path" contain _both_ the top level project and
>> the dependencies, so project-library-roots should _not_ be the same as
>> "load path" or "class path".
>
> I think "list of directories outside of the project" is pretty clear.

You have two data points on this; insisting that yours is the only one
that matters is simply not helpful.

It would be helpful if others would contribute here.

>> The advice in project-roots on "include here but not there" should
>> mention top-level vs dependencies.
>
> Could you rephrase that? I'm not sure I follow.

Suggested rewording:

  "Return the list of directory roots belonging to the current project.
This excludes project dependencies.

Most often it's just one directory, which contains the project
file and everything else in the project.  But in more advanced
configurations, a project can span multiple directories.

One rule of thumb for whether to include a directory here, and not
in `project-library-roots', is whether its contents are meant to
be edited together with the rest of the project.

If a directory contains a separate project, it belongs in
`project-library-roots'.

The directory names should be absolute.")

>> The new function `project-find-regexp' is not consistent with other
>> usage; other project- search functions search both top level and
>> dependencies.
>
> Sorry, what other project- search functions? Do you mean
> xref-find-references?

Functions that use project-roots and/or project-library-roots:

elisp--xref-find-references         both
etags--xref-find-references         both
project-find-regexp                 only project-roots
project-or-libraries-find-regexp    both

(I thought there were more; I guess I was thinking of my own code)

It is reasonable to provide a way to control what paths are searched,
but this is cumbersome and inconsistent.

Hmm. There's a "TODO" in etags--xref-find-references that says it
should be merged with elisp--xref-find-references; it's the same code.
So one solution would be to delete those two, and provide a new
function:

(defun project-find-references (symbol &optional paths)
"...
PATHS is one of:
'top-level    - only the top-level project
'dependencies - only the dependencies
'both         - both (the default)"

and change project-find-regexp to match.

Or provide project-find-references and
project-or-libraries-find-references.

>> The root cause of this problem is trying to infer a project in an elisp
>> file. This makes sense in general, because an elisp file implies the use
>> of load-path, which is the main part of defining a project. A better way
>> is to provide a project-find-function that returns an elisp-project
>> object in elisp buffers;
>
> That implies having to create a separate project implementation for
> every language, making vc-project utterly useless.

Only for languages that imply a path of some sort. In which case that
path is more useful than using "path = vc root".

vc projects are useful in situations where there is neither a language
defined path nor a build system project file that defines a path. That's
pretty rare for me, but they are not "utterly useless".

> You'd also have to create yet-another kind of project implementations,
> for multi-language projects.

That's true anyway.

> Note that determining whether the a given directory tree has Elisp
> files inside is not 100% reliable, outside of traversing it whole. 

I didn't say anything about "directories with elisp files"; just about
elisp-mode.

> But if we pick some detection mechanism, (and we'd have to, for
> Elisp-project detection), we can just as well use it to call
> elisp-library-roots globally and make it auto-detect whether the
> current project is applicable. 

That's what EDE does; we should improve that rather than duplicate it.

I am proposing something much simpler; a project-find-file that uses the
mode of the current buffer to help pick the project.

>> then elisp-project can override
>> project-library-roots to return load-path; it could also add the emacs C
>> sources if they are available.
>
> At the moment, I feel that this approach would be a cop-out. 

How is it worse than the current cop-out of setting
project-library-roots-function in elisp-mode?

>> I would delete the advice that overriding functions should use
>> project-library-roots-function; the main reason to override is to do
>> something different.
>
> (*) Hence, if we go your way, there seems to be no need for
> project-library-roots-function.

No, it's needed for projects that don't have any better way to define a
project path.

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/09/2015 11:10 AM, Stephen Leake wrote:

> I don't understand; you seem to be saying a "library" is _not_ a
> "dependency". That's not consistent with common usage.

I'm saying we'd probably want to treat "managed project dependencies" 
differently from "library dependencies".

And if we were to introduce a "dependencies" accessor, I think it would 
return a list of projects, not directories.

> "Dependency" includes system libraries, other managed projects, and
> anything else the user wants to search.
>
> Why do you insist on the less general term?

Because it's a less loaded one. 
http://www.jetbrains.org/intellij/sdk/docs/reference_guide/project_model.html 
also uses the term "library" in a similar fashion, so I think it will be 
clear enough.

> The user has the freedom to do whatever they want. The point is to have
> clearly defined semantics, so the user at least knows what the intent of
> each of these functions is.

For now, the distinction is "you want to edit this" vs "you don't want 
to edit this".

>>> The default implementation of project-library-roots makes the lists
>>> disjoint, so the doc strings should say that.
>>
>> Doesn't it? It says "outside".
>
> True; but I did not understand what you meant by that; it's not common
> project terminology. I think "dependency" is much more widely
> understood.

I'm using "outside" as "directory terminology". We seem to approach this 
from different directions: I think that the user first decides what 
their "project roots" will be, and *then* designates a certain set of 
directories that also contain related files, as "library roots".

>> I think "list of directories outside of the project" is pretty clear.
>
> You have two data points on this; insisting that yours is the only one
> that matters is simply not helpful.

Yes, sorry. You repeated the complaint, so I just repeated the answer.

> It would be helpful if others would contribute here.

Absolutely. I keep waiting for more people to join the discussion at 
this level of detail.

John, would you like to weigh in? This is one of the issues we'd might 
want to decide on before the freeze.

> Suggested rewording:
>
>    "Return the list of directory roots belonging to the current project.
> This excludes project dependencies.

But do we want to exclude project dependencies, always? This might 
contradict the third paragraph: sometimes, you edit certain dependencies 
together with the project.

> Most often it's just one directory, which contains the project
> file and everything else in the project.  But in more advanced
> configurations, a project can span multiple directories.
>
> One rule of thumb for whether to include a directory here, and not
> in `project-library-roots', is whether its contents are meant to
> be edited together with the rest of the project.
>
> If a directory contains a separate project, it belongs in
> `project-library-roots'.

As stated, I'm not sure about this.

> Functions that use project-roots and/or project-library-roots:
>
> elisp--xref-find-references         both
> etags--xref-find-references         both

These are different implementations for the same command: 
xref-find-references.

> project-find-regexp                 only project-roots
> project-or-libraries-find-regexp    both
>
> (I thought there were more; I guess I was thinking of my own code)
>
> It is reasonable to provide a way to control what paths are searched,
> but this is cumbersome and inconsistent.

Inconsistent, yes. Somewhat. If we're walking about the last two, how 
would you change them? Rename to project-without-libtaries-find-regexp 
and project-find-regexp?

IME, the former will be the one that the user will prefer most of the time.

> Hmm. There's a "TODO" in etags--xref-find-references that says it
> should be merged with elisp--xref-find-references; it's the same code.
> So one solution would be to delete those two, and provide a new
> function:
>
> (defun project-find-references (symbol &optional paths)

I'll get around to it in a short while, but the idea is that the 
xref-find-references implementation based on project+grep is just the 
default one (and it will be one function). There will also be different, 
more specialized implementations possible, and the entry point for all 
of these is still going to be called xref-find-references.

> "...
> PATHS is one of:
> 'top-level    - only the top-level project
> 'dependencies - only the dependencies
> 'both         - both (the default)"
>
> and change project-find-regexp to match.
>
> Or provide project-find-references and
> project-or-libraries-find-references.

This kind of goes against the idea of providing an implementation for 
the predefined interface.

So, yes, etags--xref-find-references searches both the project roots and 
the libraries, and ideally the user would choose, but I'm not sure where 
to ask them. Should we expose these semantics in the 
xref-find-references interface somehow?

>> That implies having to create a separate project implementation for
>> every language, making vc-project utterly useless.
>
> Only for languages that imply a path of some sort. In which case that
> path is more useful than using "path = vc root".

Most languages do, I think.

> vc projects are useful in situations where there is neither a language
> defined path nor a build system project file that defines a path. That's
> pretty rare for me, but they are not "utterly useless".

All right, I can understand this viewpoint.

>> You'd also have to create yet-another kind of project implementations,
>> for multi-language projects.
>
> That's true anyway.

Not really. If the is a hook that will return library roots for 
different languages, a given project implementation can use them all. 
Should I write a proof-of-concept?

Where I come from, it's common enough to have several "project files", 
so to speak, at the top of a project, coming from different languages. 
For instance, having both Gemfile and package.json.

>> Note that determining whether the a given directory tree has Elisp
>> files inside is not 100% reliable, outside of traversing it whole.
>
> I didn't say anything about "directories with elisp files"; just about
> elisp-mode.

I don't think we should have "current project" depend on the current 
buffer. Only on the current directory. Or be picked explicitly by the 
user (not implemented), and be "current" for the whole Emacs session.

>> But if we pick some detection mechanism, (and we'd have to, for
>> Elisp-project detection), we can just as well use it to call
>> elisp-library-roots globally and make it auto-detect whether the
>> current project is applicable.
>
> That's what EDE does; we should improve that rather than duplicate it.

Not really. EDE chooses from a list of applicable project 
implementations. It has no hooks for "libraries", AFAIK.

> I am proposing something much simpler; a project-find-file that uses the
> mode of the current buffer to help pick the project.

I dislike that. The project should be the same, whether we're visiting 
an .el file, or the README file beside it.

>> At the moment, I feel that this approach would be a cop-out.
>
> How is it worse than the current cop-out of setting
> project-library-roots-function in elisp-mode?

Not worse, it's just bad on its own.

project-library-roots-function being buffer-local is a problem. Hence 
that FIXME. And hence the proposals to improve it that I've written in 
the previous two messages (one of which you haven't replied to yet).

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/09/2015 11:10 AM, Stephen Leake wrote:
>
>> I don't understand; you seem to be saying a "library" is _not_ a
>> "dependency". That's not consistent with common usage.
>
> I'm saying we'd probably want to treat "managed project dependencies"
> differently from "library dependencies".

In general? for the search path? why?

Personally, I've only ever used "project + dependencies" in an Emacs
search path.

What are the use cases for having so many choices of search path:

top level project
libraries
other dependencies

?

> And if we were to introduce a "dependencies" accessor, I think it
> would return a list of projects, not directories.

You can easily get the project from the directory, and you can easily
search the directory. So the directory is more useful.

>> "Dependency" includes system libraries, other managed projects, and
>> anything else the user wants to search.
>>
>> Why do you insist on the less general term?
>
> Because it's a less loaded one.
> http://www.jetbrains.org/intellij/sdk/docs/reference_guide/project_model.html
> also uses the term "library" in a similar fashion, so I think it will
> be clear enough.

Well, For my projects, I'll just put all dependencies in the "library"
path, and move on.

>
>> The user has the freedom to do whatever they want. The point is to have
>> clearly defined semantics, so the user at least knows what the intent of
>> each of these functions is.
>
> For now, the distinction is "you want to edit this" vs "you don't want
> to edit this".

Which implies that managed project dependencies (which can be edited) go
in project-roots, so it is not just top-level.

I don't see the point of making that distinction here; the files in
non-editable places will be read-only. Non-edit does not imply
non-search.

>>>> The default implementation of project-library-roots makes the lists
>>>> disjoint, so the doc strings should say that.
>>>
>>> Doesn't it? It says "outside".
>>
>> True; but I did not understand what you meant by that; it's not common
>> project terminology. I think "dependency" is much more widely
>> understood.
>
> I'm using "outside" as "directory terminology". 

So what would be wrong with including both ways of describing it, so
more people understand what is wanted?


> We seem to approach
> this from different directions: I think that the user first decides
> what their "project roots" will be, and *then* designates a certain
> set of directories that also contain related files, as "library
> roots".

I don't see how that affects anything.

If I don't know what "project roots" are supposed to be (because the doc
string is not clear to me), I can't do this.

>> Suggested rewording:
>>
>>    "Return the list of directory roots belonging to the current project.
>> This excludes project dependencies.
>
> But do we want to exclude project dependencies, always? This might
> contradict the third paragraph: sometimes, you edit certain
> dependencies together with the project.

Fine; mention explicitly that some project dependencies should go here,
if you want to edit them.

The point is that some people will read "library" as "dependency", and
then do The Wrong Thing.

Not that it will matter very much; most uses
will always search both project-roots and project-library-roots.


>> Most often it's just one directory, which contains the project
>> file and everything else in the project.  But in more advanced
>> configurations, a project can span multiple directories.
>>
>> One rule of thumb for whether to include a directory here, and not
>> in `project-library-roots', is whether its contents are meant to
>> be edited together with the rest of the project.
>>
>> If a directory contains a separate project, it belongs in
>> `project-library-roots'.
>
> As stated, I'm not sure about this.

So provide alternate wording that uses the concepts "separate projects"
and "dependencies", and says what you want it to say.

People will want to know.

>> Functions that use project-roots and/or project-library-roots:
>>
>> elisp--xref-find-references         both
>> etags--xref-find-references         both
>
> These are different implementations for the same command:
> xref-find-references.
>
>> project-find-regexp                 only project-roots
>> project-or-libraries-find-regexp    both
>>
>> (I thought there were more; I guess I was thinking of my own code)
>>
>> It is reasonable to provide a way to control what paths are searched,
>> but this is cumbersome and inconsistent.
>
> Inconsistent, yes. Somewhat. If we're walking about the last two, how
> would you change them? Rename to project-without-libtaries-find-regexp
> and project-find-regexp?
>
> IME, the former will be the one that the user will prefer most of the time.
>
>> Hmm. There's a "TODO" in etags--xref-find-references that says it
>> should be merged with elisp--xref-find-references; it's the same code.
>> So one solution would be to delete those two, and provide a new
>> function:
>>
>> (defun project-find-references (symbol &optional paths)
>
> I'll get around to it in a short while, but the idea is that the
> xref-find-references implementation based on project+grep is just the
> default one (and it will be one function). There will also be
> different, more specialized implementations possible, and the entry
> point for all of these is still going to be called
> xref-find-references.

Right; so project-find-references should be a cl-defgeneric.

xref-collect-references would be better named
xref-semantic-find-references; it is a semantic implementation of
xref-find-references.


>
>
>> "...
>> PATHS is one of:
>> 'top-level    - only the top-level project
>> 'dependencies - only the dependencies
>> 'both         - both (the default)"
>>
>> and change project-find-regexp to match.
>>
>> Or provide project-find-references and
>> project-or-libraries-find-references.
>
> This kind of goes against the idea of providing an implementation for
> the predefined interface.

And points out that the predefined interface doesn't allow choice of
path. So providing a choice of path via project-roots and
project-library-roots is moot; there is no way for the user to take
advantage of that.

> So, yes, etags--xref-find-references searches both the project roots
> and the libraries, and ideally the user would choose, but I'm not sure
> where to ask them. Should we expose these semantics in the
> xref-find-references interface somehow?

If you think people will actually want to have a choice of paths, yes,
that is where it needs to be.

It's simpler to avoid the whole issue and merge project-roots and
project-library-roots, at least in this first iteration of the project
API. Then we can find out if people actually want a choice of paths.

 >>> You'd also have to create yet-another kind of project implementations,
>>> for multi-language projects.
>>
>> That's true anyway.
>
> Not really. If the is a hook that will return library roots for
> different languages, a given project implementation can use them all.

That's changing the API, which is a different approach than taking
advantage of the current API.

> Should I write a proof-of-concept?

Sure, but not before this merge, and probably not before the feature freeze.

> Where I come from, it's common enough to have several "project files",
> so to speak, at the top of a project, coming from different languages.
> For instance, having both Gemfile and package.json.

I would merge multiple project files into one Emacs project, not have
different Emacs projects for each language.

project.el can support both styles, of course.

>>> Note that determining whether the a given directory tree has Elisp
>>> files inside is not 100% reliable, outside of traversing it whole.
>>
>> I didn't say anything about "directories with elisp files"; just about
>> elisp-mode.
>
> I don't think we should have "current project" depend on the current
> buffer. 

And yet you are fine having it depend on the "current language"; where
does that come from?

> Only on the current directory. 

I include multiple language source files in one directory.

> Or be picked explicitly by the user (not implemented), and be
> "current" for the whole Emacs session.

Always a good choice ;).

>>> At the moment, I feel that this approach would be a cop-out.
>>
>> How is it worse than the current cop-out of setting
>> project-library-roots-function in elisp-mode?
>
> Not worse, it's just bad on its own.
>
> project-library-roots-function being buffer-local is a problem. Hence
> that FIXME. 

Ok; enhance the FIXME: to say "because some modes make this
buffer-local" (project.el does _not_ declare it buffer local).

-- 
-- Stephe

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> Absolutely. I keep waiting for more people to join the discussion at this
> level of detail.
> 
> John, would you like to weigh in? This is one of the issues we'd might want
> to decide on before the freeze.

Sure, although I haven't been following the history of this thread. Could
someone please summarize where we're at now? Also, how does this library
relate to something like Projectile, which is what I presently use for
project-related commands.

John

Re: project.el semantics

[Dmitry Gutov]

On 11/10/2015 12:16 AM, John Wiegley wrote:

> Sure, although I haven't been following the history of this thread. Could
> someone please summarize where we're at now?

The current state of affairs is in lisp/progmodes/project.el (200+ 
lines), and you can read the current bout of the discussion starting 
with http://lists.gnu.org/archive/html/emacs-devel/2015-11/msg00525.html

I hope that's not too much.

> Also, how does this library
> relate to something like Projectile, which is what I presently use for
> project-related commands.

Does its Commentary answer this question?

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> The current state of affairs is in lisp/progmodes/project.el (200+ lines),
> and you can read the current bout of the discussion starting with
> http://lists.gnu.org/archive/html/emacs-devel/2015-11/msg00525.html

I did read that discussion, but I didn't walk away with any clear notions of
what's being discussed now.

>> Also, how does this library
>> relate to something like Projectile, which is what I presently use for
>> project-related commands.

> Does its Commentary answer this question?

So is it an API that Projectile could make use of? Is it as simple as "find
the root of the current project"?

John

Re: project.el semantics

[Dmitry Gutov]

On 11/10/2015 03:07 AM, John Wiegley wrote:

> I did read that discussion, but I didn't walk away with any clear notions of
> what's being discussed now.

Have you looked at the code?

- Some bikeshedding about method names, their semantics and how to 
better document them. Does that sound important?

- The problem of project-library-roots-function being a variable, and 
being set in emacs-lisp-mode to a buffer-local value. There's a FIXME 
above its definition, please have a read.

> So is it an API that Projectile could make use of? Is it as simple as "find
> the root of the current project"?

It's an API that Projectile can write an adapter for, to allow 
third-party code not to depend on Projectile, or EDE, or another project 
system, explicitly. The current set of methods deals with directories 
related to the current project, but can be extended.

Re: project.el semantics

[Dmitry Gutov]

On 11/09/2015 08:15 PM, Stephen Leake wrote:

>> Or be picked explicitly by the user (not implemented), and be
>> "current" for the whole Emacs session.
>
> Always a good choice ;).

Yeah, I've been thinking about it, and we probably should implement 
something like Eclipse's workspace (a set of open projects), maybe like 
a optional mode of operation.

Only having the current project depend on the current directory is 
limiting, e.g. if we want to execute some project-related operation 
(like project-or-libraries-find-regexp) from some file inside a library.

That issue is not specific to Ada, or complex projects, and can come up 
in any nontrivial project.

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

>> I did read that discussion, but I didn't walk away with any clear notions
>> of what's being discussed now.
> Have you looked at the code?

Ah, a code review. I will add it to Org.

>> So is it an API that Projectile could make use of? Is it as simple as "find
>> the root of the current project"?

> It's an API that Projectile can write an adapter for, to allow third-party
> code not to depend on Projectile, or EDE, or another project system,
> explicitly. The current set of methods deals with directories related to the
> current project, but can be extended.

OK, that sounds like good starting functionality to me. I've often wanted a
simple, general way to reference the "project root".

John

Re: project.el semantics

[Dmitry Gutov]

On 11/09/2015 08:15 PM, Stephen Leake wrote:

>> I'm saying we'd probably want to treat "managed project dependencies"
>> differently from "library dependencies".
>
> In general? for the search path? why?

Not necessarily for the search path. But it would be a different notion, 
so we might as well reserve the name.

> Personally, I've only ever used "project + dependencies" in an Emacs
> search path.

That's totally fine.

> What are the use cases for having so many choices of search path:
>
> top level project
> libraries
> other dependencies

We'll have to see, really.

>> And if we were to introduce a "dependencies" accessor, I think it
>> would return a list of projects, not directories.
>
> You can easily get the project from the directory, and you can easily
> search the directory. So the directory is more useful.

I don't know about that. If you only know the directory, you're at the 
mercy of project-find-functions. If (project-dependencies project) 
returns project instances, however, you're free to choose the right 
project implementation to use for them, set up "parent project" 
references, use some other information from the current project file, 
and so on.

Although again, what will the project dependencies be used for, is still 
an open question for me.

> Well, For my projects, I'll just put all dependencies in the "library"
> path, and move on.

Not a problem.

>> For now, the distinction is "you want to edit this" vs "you don't want
>> to edit this".
>
> Which implies that managed project dependencies (which can be edited) go
> in project-roots, so it is not just top-level.

Oh, so that's what "managed" means. All right; then that means yes, in 
my opinion.

> I don't see the point of making that distinction here; the files in
> non-editable places will be read-only. Non-edit does not imply
> non-search.

In dynamic languages that I've used, the library files are often 
perfectly editable. You just don't want to do that, most of the time.

Another thing you might want to do - replace some term across the 
project (using M-x project-find-regexp and then M-x xref-query-replace). 
If we were to show its occurrences in library files as well, that would 
be a waste of user's and computer's time.

>> I'm using "outside" as "directory terminology".
>
> So what would be wrong with including both ways of describing it, so
> more people understand what is wanted?

I generally try to get away with using as little words as possible when 
describing something, because words can be misunderstood, especially new 
ones, which haven't been used before in the given context (such as 
"dependency").

Since we're discussing project-library-roots here, I'm not sure which 
improvement you mean. How about simply replacing "the list of 
directories outside of the project" with "the list of directories 
outside of the project roots"?

Because if we describe it in terms of dependencies, a dependency might 
be checked out inside a project root subdirectory in certain 
configurations (maybe with an appropriate .gitignore entry, maybe not). 
In that case, though, we probably don't want to have 
project-library-roots include it.

>> We seem to approach
>> this from different directions: I think that the user first decides
>> what their "project roots" will be, and *then* designates a certain
>> set of directories that also contain related files, as "library
>> roots".
>
> I don't see how that affects anything.
>
> If I don't know what "project roots" are supposed to be (because the doc
> string is not clear to me), I can't do this.

You said: "The default implementation of project-library-roots makes the 
lists disjoint, so the doc strings should say that."

And my point is that if author knows what the "project roots" are, they 
should understand what "project library roots" are, easily. Can we agree 
that the project-library-roots docstring is more or less fine?

> Fine; mention explicitly that some project dependencies should go here,
> if you want to edit them.
>
> The point is that some people will read "library" as "dependency", and
> then do The Wrong Thing.

I do think that occasional misreadings of the spec aren't going to be 
much of a problem, because there's not going to be *too* many project 
implementations.

We can add "So if the project has dependencies which should be edited 
together with it, they should go here as well." to the third paragraph 
of the project-roots docstring. But doesn't it look redundant?

> So provide alternate wording that uses the concepts "separate projects"
> and "dependencies", and says what you want it to say.
>
> People will want to know.

Guess that might go into the manual instead. I've seen someone state 
that it's okay for the manual to have redundancies.

> Right; so project-find-references should be a cl-defgeneric.

xref-find-references will be a cl-defgeneric. Would there be a good 
reason to have project-find-references as well?

Keep in mind that the current Grep-based "find references" are sloppy, 
best-effort implementations. It's not like we expect the users to use 
them when there's any other alternative.

> xref-collect-references would be better named
> xref-semantic-find-references; it is a semantic implementation of
> xref-find-references.

It's a... semantic-symref-tool implementation of it, actually. Which has 
no relation to Semantic grammars, its tags, or etc.

And I'm seriously considering to get away from it and always use Grep, 
because the other tools, while they can be faster, they're also entirely 
opaque to the user, can have outdated databases, non-indexed relevant 
directories, unsupported languages, and so on.

> It's simpler to avoid the whole issue and merge project-roots and
> project-library-roots, at least in this first iteration of the project
> API. Then we can find out if people actually want a choice of paths.

IMO, it's more important to have the distinction between "project roots" 
and "project library roots" in "find regexp" than in "find references".

>> Not really. If the is a hook that will return library roots for
>> different languages, a given project implementation can use them all.
>
> That's changing the API, which is a different approach than taking
> advantage of the current API.

I don't understand. Which API?

>> Should I write a proof-of-concept?
>
> Sure, but not before this merge,

I've merged project-next.

 > and probably not before the feature freeze.

Why not? These aren't particularly big changes that I'm talking about.

They're not backward-compatible, however, and I wouldn't want to freeze 
the API in the state that we'll want to change later.

>> Where I come from, it's common enough to have several "project files",
>> so to speak, at the top of a project, coming from different languages.
>> For instance, having both Gemfile and package.json.
>
> I would merge multiple project files into one Emacs project, not have
> different Emacs projects for each language.

And to do that, vc-project needs to know how to get library-roots from 
each of those project files. From a hook. Right?

>> I don't think we should have "current project" depend on the current
>> buffer.
>
> And yet you are fine having it depend on the "current language"; where
> does that come from?

No, I'm not. The "current project" value cannot depend on the "current 
language" already: the `project-current' contract forbids it.

The only part that can depend on the current language is 
`project-library-roots', and here I am, again, discussing the ways to 
improve that.

>> Only on the current directory.
>
> I include multiple language source files in one directory.

Yessss. Did I mention that I have multiple project files in one 
directory? Naturally, they're from different languages.

> Ok; enhance the FIXME: to say "because some modes make this
> buffer-local" (project.el does _not_ declare it buffer local).

I've pushed a change along these lines.

Re: project.el semantics

[Dmitry Gutov]

On 11/10/2015 03:40 AM, John Wiegley wrote:

> Ah, a code review.

If you like. I just meant it's hard to discuss code without having seen it.

> OK, that sounds like good starting functionality to me. I've often wanted a
> simple, general way to reference the "project root".

Except we have "project roots", so it's a little bit more complicated. 
But yes. And we also have a few commands now that take advantage of this 
API.

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

>> Ah, a code review.
> If you like. I just meant it's hard to discuss code without having seen it.

I guess I'm just asking exactly what you'd like to discuss about the code. If
you want a review, then I know what's being asked for. Otherwise, is there an
open question that needs my comment? It seems to be developing along fairly
well.

John

Re: project.el semantics

[Dmitry Gutov]

On 11/10/2015 08:00 AM, John Wiegley wrote:

> I guess I'm just asking exactly what you'd like to discuss about the code. If
> you want a review, then I know what's being asked for. Otherwise, is there an
> open question that needs my comment? It seems to be developing along fairly
> well.

You can see we disagree about some things with Stephen, from the 
discussion. These issues relate to method names, docstrings and the 
FIXME that's in the file.

So I be happy if you or anyone else, really, had anything to offer on 
that. But to do that, I think one has to read the file.

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> You can see we disagree about some things with Stephen, from the discussion.
> These issues relate to method names, docstrings and the FIXME that's in the
> file.
>
> So I be happy if you or anyone else, really, had anything to offer on that.
> But to do that, I think one has to read the file.

Ah, OK, then I'll get back to you shortly with some comments. Thank you for
the clarification.

John

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

>> You can easily get the project from the directory, and you can easily
>> search the directory. So the directory is more useful.
>
> I don't know about that. If you only know the directory, you're at the
> mercy of project-find-functions. If (project-dependencies project)
> returns project instances, however, you're free to choose the right
> project implementation to use for them, set up "parent project"
> references, use some other information from the current project file,
> and so on.
>
> Although again, what will the project dependencies be used for, is
> still an open question for me.

The general rule is: if we don't have a good use case, don't add code.

>> I don't see the point of making that distinction here; the files in
>> non-editable places will be read-only. Non-edit does not imply
>> non-search.
>
> In dynamic languages that I've used, the library files are often
> perfectly editable. You just don't want to do that, most of the time.
>
> Another thing you might want to do - replace some term across the
> project (using M-x project-find-regexp and then M-x
> xref-query-replace). If we were to show its occurrences in library
> files as well, that would be a waste of user's and computer's time.

You can't allow for all possible user desires by providing different
path functions at this level.

Instead, there should just be one path, that includes all files that the
user might want to do anything with. Then, when the user actually
invokes a function to do something, they can provide a predicate if they
want a subset of that.

>>> I'm using "outside" as "directory terminology".
>>
>> So what would be wrong with including both ways of describing it, so
>> more people understand what is wanted?
>
> I generally try to get away with using as little words as possible
> when describing something, because words can be misunderstood,
> especially new ones, which haven't been used before in the given
> context (such as "dependency").

"outside" is a new word in that sense.

You have to rely on common definitions; "dependency" is one (although
apparently not for you, sigh).

"outside" is a common English word, but "outside the project" is very
fuzzy. 

> Since we're discussing project-library-roots here, I'm not sure which
> improvement you mean. How about simply replacing "the list of
> directories outside of the project" with "the list of directories
> outside of the project roots"?

I suggest "not included in" rather than "outside".

> Because if we describe it in terms of dependencies, a dependency might
> be checked out inside a project root subdirectory in certain
> configurations (maybe with an appropriate .gitignore entry, maybe
> not). In that case, though, we probably don't want to have
> project-library-roots include it.

Why not? We allow subdirectories in the same path in general (ie
`load-path'); that's what project-prune-directories is for.

> We can add "So if the project has dependencies which should be edited
> together with it, they should go here as well." to the third paragraph
> of the project-roots docstring. 

That would be good.

>> So provide alternate wording that uses the concepts "separate projects"
>> and "dependencies", and says what you want it to say.
>>
>> People will want to know.
>
> Guess that might go into the manual instead. I've seen someone state
> that it's okay for the manual to have redundancies.

Ok, please start the manual. That would help a lot.

>> Right; so project-find-references should be a cl-defgeneric.
>
> Keep in mind that the current Grep-based "find references" are sloppy,
> best-effort implementations. It's not like we expect the users to use
> them when there's any other alternative.

Right; that's what default implementations of generic functions are for.

>> xref-collect-references would be better named
>> xref-semantic-find-references; it is a semantic implementation of
>> xref-find-references.
>
> It's a... semantic-symref-tool implementation of it, actually. Which
> has no relation to Semantic grammars, its tags, or etc.

Ok, so xref-semantic-symref-find-references, or
xref-symref-find-references. 

> And I'm seriously considering to get away from it and always use Grep,
> because the other tools, while they can be faster, they're also
> entirely opaque to the user, can have outdated databases, non-indexed
> relevant directories, unsupported languages, and so on.

But if I normally use global, I want xref to use it. And I know how to
deal with all those issues. Although some details in the manual would
help. 

I could write a new global xref backend, but it's better to reuse
existing code.

Unless you are proposing to rewrite symref to be more consistent with
xref. 

>> It's simpler to avoid the whole issue and merge project-roots and
>> project-library-roots, at least in this first iteration of the project
>> API. Then we can find out if people actually want a choice of paths.
>
> IMO, it's more important to have the distinction between "project
> roots" and "project library roots" in "find regexp" than in "find
> references".

But you don't provide any code that allows the user to take advantage of
that distinction. So it how important can it be?

>>> Where I come from, it's common enough to have several "project files",
>>> so to speak, at the top of a project, coming from different languages.
>>> For instance, having both Gemfile and package.json.
>>
>> I would merge multiple project files into one Emacs project, not have
>> different Emacs projects for each language.
>
> And to do that, vc-project needs to know how to get library-roots from
> each of those project files. From a hook. Right?

No, I would not expect vc-project to know anything about project files.

I would write a new project backend that knows about some set of project
files. 

-- 
-- Stephe

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/10/2015 03:40 AM, John Wiegley wrote:
>
>> Ah, a code review.
>
> If you like. I just meant it's hard to discuss code without having seen it.
>
>> OK, that sounds like good starting functionality to me. I've often wanted a
>> simple, general way to reference the "project root".
>
> Except we have "project roots", so it's a little bit more complicated.

And also "project library roots".

I'm trying to clarify what they mean, with little success.

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/10/2015 07:38 PM, Stephen Leake wrote:

> And also "project library roots".
>
> I'm trying to clarify what they mean, with little success.

That's a rather strong complaint. No doubt the current wording can be 
improved, but I think it's at least intelligible, and unambiguous enough.

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/10/2015 08:00 AM, John Wiegley wrote:
>
>> I guess I'm just asking exactly what you'd like to discuss about the code. If
>> you want a review, then I know what's being asked for. Otherwise, is there an
>> open question that needs my comment? It seems to be developing along fairly
>> well.
>
> You can see we disagree about some things with Stephen, from the
> discussion. These issues relate to method names, docstrings and the
> FIXME that's in the file.

More precisely, the only hot issue is the semantics of
"package-library-roots" and "package-roots".

They provide the list of directories that are related to the user's
project in some way (contain code, or documentation, or dependencies
(other packages or libraries)).

One use for these is to provide the search path for grep, or for
locate-file (with some glue code).

I don't understand why we have both; one would be sufficient. I have yet
to see an actual use case that requires both.

Since we have both, I don't understand what directories go in one vs the
other.

The rationale seems to change each time I propose a different use case.

One way to move forward would be to start a manual section for
package.el, and capture use cases in it, so we could refer to them. Or
capture use cases in a design document, if we don't want them in the
user manual.

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/10/2015 07:36 PM, Stephen Leake wrote:

>> Although again, what will the project dependencies be used for, is
>> still an open question for me.
>
> The general rule is: if we don't have a good use case, don't add code.

So I'm not adding it. Just keeping that possibility in mind, and hence 
keeping the word "dependency" free.

>> Another thing you might want to do - replace some term across the
>> project (using M-x project-find-regexp and then M-x
>> xref-query-replace). If we were to show its occurrences in library
>> files as well, that would be a waste of user's and computer's time.
>
> You can't allow for all possible user desires by providing different
> path functions at this level.

Indeed, I can't account for every variation. So I service just the ones 
that seem most important to me at this point.

> Instead, there should just be one path, that includes all files that the
> user might want to do anything with. Then, when the user actually
> invokes a function to do something, they can provide a predicate if they
> want a subset of that.

How would they provide the predicate when calling the command 
interactively? Especially a predicate that would distinguish between 
project roots and library roots.

> You have to rely on common definitions; "dependency" is one (although
> apparently not for you, sigh).

"outside" is definitely a more common word for me than "dependency". *shrug*

> "outside" is a common English word, but "outside the project" is very
> fuzzy.

You're free to suggest another word that means "not inside of any of the 
project roots".

>> Since we're discussing project-library-roots here, I'm not sure which
>> improvement you mean. How about simply replacing "the list of
>> directories outside of the project" with "the list of directories
>> outside of the project roots"?
>
> I suggest "not included in" rather than "outside".

That implies the (not (member ...)) semantics, rather than (not 
(file-in-directory-p ...)), which is what `project-subtract-directories' 
actually does.

And "not member" semantics seems pointless to me here; we might as well 
allow project-library-roots to include project-roots element, if the 
consumer will have to call `project-combine-directories' on them together.

Having "library roots" entirely outside the project roots, however, 
makes a lot of sense to me intuitively: if they're inside, they are a 
part of the project anyway. What kind of use would we have for such 
elements?

>> Because if we describe it in terms of dependencies, a dependency might
>> be checked out inside a project root subdirectory in certain
>> configurations (maybe with an appropriate .gitignore entry, maybe
>> not). In that case, though, we probably don't want to have
>> project-library-roots include it.
>
> Why not? We allow subdirectories in the same path in general (ie
> `load-path'); that's what project-prune-directories is for.

Note that neither `elisp--xref-find-references', not 
`etags--xref-find-references' call `project-prune-directories' (or 
`project-combine-directories', as it's called now).

>> We can add "So if the project has dependencies which should be edited
>> together with it, they should go here as well." to the third paragraph
>> of the project-roots docstring.
>
> That would be good.

Would you be able to give an example of a project file that lists 
dependencies like that, that "should be edited together" with the 
current project? It would fit the manual well.

>> Guess that might go into the manual instead. I've seen someone state
>> that it's okay for the manual to have redundancies.
>
> Ok, please start the manual. That would help a lot.

If you're waiting for me to start, you might have to wait a while. I 
haven't written a manual in my life (not in Texinfo, or in English).

If you or someone else add the file with structure outline some 
placeholders a la <write XX here>, that would speed the process 
considerably.

>> It's a... semantic-symref-tool implementation of it, actually. Which
>> has no relation to Semantic grammars, its tags, or etc.
>
> Ok, so xref-semantic-symref-find-references, or
> xref-symref-find-references.

semantic-symref is a command that displays references in a tree-like UI, 
and it uses Semantic on the way to do that. xref-collect-references doesn't.

I don't really know a good name for it. 
xref-collect-references-using-any-cedet-indexing-tool-we-can-find?

It's not a user-level command, though, so we can get away with just a 
mediocre name.

> But if I normally use global, I want xref to use it. And I know how to
> deal with all those issues. Although some details in the manual would
> help.

Yes, the manual could be the answer.

> I could write a new global xref backend, but it's better to reuse
> existing code.

Does lisp/cedet/cedet-global.el have code that will help us with 
implementing "find definitions" using Global?

> Unless you are proposing to rewrite symref to be more consistent with
> xref.

Maybe. Depending on what you exactly mean by that. I'd rather rewrite 
the "tools" CEDET code to be more flexible, for different uses.

>> IMO, it's more important to have the distinction between "project
>> roots" and "project library roots" in "find regexp" than in "find
>> references".
>
> But you don't provide any code that allows the user to take advantage of
> that distinction. So it how important can it be?

What code?

We have both project-find-regexp and project-or-libraries-find-regexp 
already, and I find the difference between them useful in practice.

> No, I would not expect vc-project to know anything about project files.
>
> I would write a new project backend that knows about some set of project
> files.

Again, how many project backends will that be? One for every distinct 
language combination?

Basically, you position eliminates the need for 
project-library-roots-function, as it's used now. I think that we should 
create a better use for it, and do teach vc-project about library-roots 
for different project files, through it.

Re: project.el semantics

[Dmitry Gutov]

On 11/11/2015 01:41 AM, Stephen Leake wrote:

> More precisely, the only hot issue is the semantics of
> "package-library-roots" and "package-roots".
> ...
> I don't understand why we have both; one would be sufficient. I have yet
> to see an actual use case that requires both.

How could I explain that better? I want to:

- Search inside the project root(s). But not inside libraries.

- But sometimes inside libraries as well.

Because usually libraries will only bring false positives. But for some 
kinda of searches, searching in libraries proves insightful as well.

That's missing in this description?

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> How could I explain that better? I want to:
> - Search inside the project root(s). But not inside libraries.
> - But sometimes inside libraries as well.

These distinctions are not strictly directory-based. For example, with Helm I
search in the list of files generated by "git ls-files". It's not about where
the files are within the project, but whether they have been registered with
Git.

So before we talk too much about different kinds of "roots", we may be asking
the wrong question. What is it that we want to distinguish, exactly?

John

Re: project.el semantics

[Dmitry Gutov]

On 11/11/2015 03:17 AM, John Wiegley wrote:

> These distinctions are not strictly directory-based. For example, with Helm I
> search in the list of files generated by "git ls-files". It's not about where
> the files are within the project, but whether they have been registered with
> Git.

We define "library roots" as always being outside the project. Because 
that's a simpler model, and I can't exactly come up with an example of a 
"library root" inside the project root being useful.

> So before we talk too much about different kinds of "roots", we may be asking
> the wrong question. What is it that we want to distinguish, exactly?

project vs libraries. Or, "files we own and edit" vs "files we sometimes 
want to look at" (and maybe edit during debugging).

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/11/2015 01:41 AM, Stephen Leake wrote:
>
>> More precisely, the only hot issue is the semantics of
>> "package-library-roots" and "package-roots".
>> ...
>> I don't understand why we have both; one would be sufficient. I have yet
>> to see an actual use case that requires both.
>
> How could I explain that better? I want to:
>
> - Search inside the project root(s). But not inside libraries.
>
> - But sometimes inside libraries as well.
>
> Because usually libraries will only bring false positives. But for
> some kinda of searches, searching in libraries proves insightful as
> well.
>
> That's missing in this description?

There is also the notion of "editable" vs "non-editable", in the doc
string for project-roots.

This doesn't tell me whether a dependency that is not a "library"
belongs in project-root or project-library-root.

But my main problem with this is that it is far too limiting.

There are many other possible use cases for restricting a search over
the list of directories related to a project:

- Search in all dependencies that come from Google (or some other
  vendor).

- Search only in directories that are not read-only.

- Search only in directories that are not marked "passed unit testing".

- Search only in directories that are not marked "refactoring for
  feature 'foo' finished".

- etc.

We cannot possibly anticipate the set of restrictions some user might
want to put on a search. So why should we assume "search on libraries"
is so important that it needs this level of attention?

I think the correct approach is to add a "predicate" argument to
functions that use the project directory path; similar to the
"predicate" argument in completion tables.

Then the user can supply a predicate function that implements each of the
above use cases. 

-- 
-- Stephe

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> project vs libraries. Or, "files we own and edit" vs "files we
> sometimes want to look at" (and maybe edit during debugging).

Those are conflicting criteria, so you are proving my point.

As an example, you define project-library-roots for elisp to include
_all_ of load-path (see the recently added function
`elisp-library-roots').

On a user machine (not a developer machine), a lot of load-path is
treated as read-only. But some is not. So why is it in
project-library-roots?

The obvious answer is "because there is no way to tell which
subset of load-path is treated as user-editable".

Note that "owned by the user" is not enough; ELPA packages are not editable,
but they are in ~/.emacs.d/elpa, owned by the user. And a user may have
installed Emacs in their own directory tree.

Which is precisely my point; there is no good way to decide whether a
directory belongs in project-roots or project-library-roots.

-- 
-- Stephe

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/10/2015 07:36 PM, Stephen Leake wrote:
>
>>> Although again, what will the project dependencies be used for, is
>>> still an open question for me.
>>
>> The general rule is: if we don't have a good use case, don't add code.
>
> So I'm not adding it. Just keeping that possibility in mind, and hence
> keeping the word "dependency" free.

My point is you don't have a good use case for "library", either. At
least, the use case for "search only on libraries" is no more important
than other similar "search only on ..." use cases.

>> Instead, there should just be one path, that includes all files that the
>> user might want to do anything with. Then, when the user actually
>> invokes a function to do something, they can provide a predicate if they
>> want a subset of that.
>
> How would they provide the predicate when calling the command
> interactively? Especially a predicate that would distinguish between
> project roots and library roots.

In a lot of cases, they will simply search the entire directory list,
and filter the results manually as they go thru them.

But they can take the time to write a predicate function, and invoke the
command via M-; to supply it as an argument in the call.

Or they could write a wrapper that provides a list of common predicates.

In general, project.el is not about the final user interface, it is
about providing core functionality.

>>> Since we're discussing project-library-roots here, I'm not sure which
>>> improvement you mean. How about simply replacing "the list of
>>> directories outside of the project" with "the list of directories
>>> outside of the project roots"?
>>
>> I suggest "not included in" rather than "outside".
>
> That implies the (not (member ...)) semantics, rather than (not
> (file-in-directory-p ...)), which is what
> `project-subtract-directories' actually does.

Since project-roots is treated recursively, "not included in" means (not
(file-in-directory-p ...)) to me.

> And "not member" semantics seems pointless to me here; we might as
> well allow project-library-roots to include project-roots element, if
> the consumer will have to call `project-combine-directories' on them
> together.

I'm confused. With the current definitions, project-roots and
project-library-roots are disjoint, so the user does have to call
project-combine-directories if they want to search on both.

You now seem to be agreeing with me; it would be better to have only
one project-roots function.

> Having "library roots" entirely outside the project roots, however,
> makes a lot of sense to me intuitively: if they're inside, they are a
> part of the project anyway. What kind of use would we have for such
> elements?

I'm not sure what you are asking here.

You seem to be asking "why would we want to search on
project-library-roots". But obviously you _do_ want the ability to
search on that; that's why it's there. So I must be missing something.

>>> We can add "So if the project has dependencies which should be edited
>>> together with it, they should go here as well." to the third paragraph
>>> of the project-roots docstring.
>>
>> That would be good.
>
> Would you be able to give an example of a project file that lists
> dependencies like that, that "should be edited together" with the
> current project? It would fit the manual well.

We have not defined an Emacs project file yet. I can easily provide an
Ada example, but I don't think you want that actual file in the Emacs
manual.

Any case where a large project is divided into subprojects, all of which
are editable. For example, my NASA GDS projects looked like this:

GDS
    sal

    common

    map

    dscovr

All four of these are projects. "sal" and "common" are lower level; they
are used by "map" and "dscovr", which are the projects used by the MAP
and DSCOVR missions respectively (you can Google those ...). "sal" and
"common" are listed as dependencies in the "map" and "dscovr" Ada
project files. Nothing in "map" depends on "dscovr", and vice-versa.

All of them depend on the Ada runtime library, which is also in the
project search path.

All of them are maintained by the GDS team. There is one GDS team member
primarily responsible for each mission-level project; I was mostly
responsible for the common projects. Any GDS team member can edit any
file in any project.

The only thing that is a "library" is the Ada runtime library, installed
with the Ada compiler.

So for the dscovr project, what goes in project-roots, and what goes in
project-library-roots, and why?

Note that the Ada project tool provides only one "source search path";
it does not distinguish between "libraries" and "non-libraries". For the
dscovr project, "dscovr", "sal", "common", and the runtime library are
in the search path.

>> I could write a new global xref backend, but it's better to reuse
>> existing code.
>
> Does lisp/cedet/cedet-global.el have code that will help us with
> implementing "find definitions" using Global?

Yes. I've started an implementation that does that:

(defun semantic-symref--xref-find-definitions (symbol &optional symref-tool)
  "Implement `xref-find-function' for 'definitions for the semantic/symref backend.
SYMREF-TOOL determines which symref tool to use (default
'detect); see `semantic-symref-tool-alist'."
  (require 'semantic/symref)
  (defvar semantic-symref-tool)
  (let* ((semantic-symref-tool (or symref-tool 'detect))
         (res (semantic-symref-find-tags-by-name symbol 'project))
         (hits (and res (oref res hit-lines)))
	 xrefs)

    (dolist (hit hits)
      (push
       (xref-file-location
	:file (cdr hit)
	:line (car hit)
	:column 0)
       ;; FIXME: add :hint (match-string 3))
       xrefs))

    xrefs))

>>> IMO, it's more important to have the distinction between "project
>>> roots" and "project library roots" in "find regexp" than in "find
>>> references".
>>
>> But you don't provide any code that allows the user to take advantage of
>> that distinction. So it how important can it be?
>
> We have both project-find-regexp and project-or-libraries-find-regexp
> already, and I find the difference between them useful in practice.

Yes, sorry, I temporarily forgot about those; they are new.

xref-find-references needs similar treatment to be consistent. But I
think that's the wrong approach; adding a more general user-specified
predicate function is much better.

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/11/2015 12:27 PM, Stephen Leake wrote:

> My point is you don't have a good use case for "library", either. At
> least, the use case for "search only on libraries" is no more important
> than other similar "search only on ..." use cases.

You can't say I don't have a use case when I do use it. Just take it on 
faith.

"search only on libraries" doesn't seems particularly important, but 
"search in project only" and "in project + libraries" are.

>> How would they provide the predicate when calling the command
>> interactively? Especially a predicate that would distinguish between
>> project roots and library roots.
>
> In a lot of cases, they will simply search the entire directory list,
> and filter the results manually as they go thru them.

That's not an answer to the question.

> But they can take the time to write a predicate function, and invoke the
> command via M-; to supply it as an argument in the call.

That's not how we usually imagine a user interface.

> Or they could write a wrapper that provides a list of common predicates.

A user, conceivably, could. Because they probably know the layout of a 
given project; even though they might have to create a new wrapper for 
each project, or a kind of project, they work on.

But third-party code that might use project.el, doesn't know anything 
about the project, except what project.el tells it.

> In general, project.el is not about the final user interface, it is
> about providing core functionality.

Providing core functionality for third-party code that's not, usually, 
written by the user either. So it's exactly the business of project.el 
to include the kind of metadata like "these are the directories the user 
might be interested in, but the user doesn't own the code inside".

> Since project-roots is treated recursively, "not included in" means (not
> (file-in-directory-p ...)) to me.

Not to me. We need a third opinion here. I might be biased, of course:

http://apidock.com/ruby/v1_9_3_392/Enumerable/include%3F

>> And "not member" semantics seems pointless to me here; we might as
>> well allow project-library-roots to include project-roots element, if
>> the consumer will have to call `project-combine-directories' on them
>> together.
>
> I'm confused. With the current definitions, project-roots and
> project-library-roots are disjoint, so the user does have to call
> project-combine-directories if they want to search on both.

Why are you confused? Look at the definitions: they use `append'.

> You now seem to be agreeing with me; it would be better to have only
> one project-roots function.

Nope.

>> Having "library roots" entirely outside the project roots, however,
>> makes a lot of sense to me intuitively: if they're inside, they are a
>> part of the project anyway. What kind of use would we have for such
>> elements?
>
> I'm not sure what you are asking here.

I was continuing the line of thought that started with the assumption 
that you want the (not (member ...)) semantics.

> You seem to be asking "why would we want to search on
> project-library-roots".

No.

>> Would you be able to give an example of a project file that lists
>> dependencies like that, that "should be edited together" with the
>> current project? It would fit the manual well.
>
> We have not defined an Emacs project file yet. I can easily provide an
> Ada example, but I don't think you want that actual file in the Emacs
> manual.

Not the contents of the file, of course. I just want to construct a 
realistic-sounding sentence, maybe mentioning Maven modules or whatever.

> Any case where a large project is divided into subprojects, all of which
> are editable. For example, my NASA GDS projects looked like this:
>
> GDS
>      sal
>
>      common
>
>      map
>
>      dscovr
>
> All four of these are projects. "sal" and "common" are lower level; they
> are used by "map" and "dscovr", which are the projects used by the MAP
> and DSCOVR missions respectively (you can Google those ...). "sal" and

You work on very cool projects, no question about it. :)

(And MAP is apparently WMAP).

> "common" are listed as dependencies in the "map" and "dscovr" Ada
> project files. Nothing in "map" depends on "dscovr", and vice-versa.

There is a choice here, would you always use the same project 
instance/configuration when working on any of them (option 1, see 
below), or would you use different project instances when working on map 
and dscovr (option 2).

> ...
 > Any GDS team member can edit any
> file in any project.

I'm going to assume that they not only can, but also do that, with some 
regularity.

> The only thing that is a "library" is the Ada runtime library, installed
> with the Ada compiler.

That's fine. In the general case, though, you could have some 
third-party libraries installed as well, right?

> So for the dscovr project, what goes in project-roots, and what goes in
> project-library-roots, and why?

Apparently you're going with option 2.

Then project-roots will contain dscovr, sal and common. And 
library-roots will have what's left in the search path (Ada runtime 
library).

Personally, I expect that to be useful (e.g. limiting searches to the 
projects that you control). But if not, you an also disregard that 
distinction. Do you see any downsides to it?

> Note that the Ada project tool provides only one "source search path";
> it does not distinguish between "libraries" and "non-libraries". For the
> dscovr project, "dscovr", "sal", "common", and the runtime library are
> in the search path.

I'm hoping that you can still distinguish sal and common from the 
standard library somehow, programmatically.

>>> I could write a new global xref backend, but it's better to reuse
>>> existing code.
>>
>> Does lisp/cedet/cedet-global.el have code that will help us with
>> implementing "find definitions" using Global?
>
> Yes. I've started an implementation that does that:

That's great. I've also been thinking of the "tag by name" 
implementation for Grep, to use as a slow fallback. So far, it involves 
piggybacking on font-lock.

>> We have both project-find-regexp and project-or-libraries-find-regexp
>> already, and I find the difference between them useful in practice.
>
> Yes, sorry, I temporarily forgot about those; they are new.
>
> xref-find-references needs similar treatment to be consistent. But I

Thus far xref-find-references doesn't depend on the presence of "current 
project" (some implementations might use, some might not).

So creating two variations of xref-find-references would be incongruous, 
even if maybe useful.

> think that's the wrong approach; adding a more general user-specified
> predicate function is much better.

If you only accept user-specified predicates, you're assuming that the 
project backend can't provide any useful information to classify the 
directories. Whereas I think it pretty handy if the user doesn't have to 
write any code to start taking advantage of some basic distinctions 
(like "project contents" vs "libraries").

I'm open to the ideas in this direction, but the UI must be better than M-:.

Re: project.el semantics

[Dmitry Gutov]

On 11/11/2015 11:55 AM, Stephen Leake wrote:
> Dmitry Gutov <dgutov@yandex.ru> writes:
>
>> project vs libraries. Or, "files we own and edit" vs "files we
>> sometimes want to look at" (and maybe edit during debugging).
>
> Those are conflicting criteria, so you are proving my point.

I don't see that.

> As an example, you define project-library-roots for elisp to include
> _all_ of load-path (see the recently added function
> `elisp-library-roots').
>
> On a user machine (not a developer machine), a lot of load-path is
> treated as read-only. But some is not. So why is it in
> project-library-roots?

a) They might be editable (and even authored by the user), but it's 
pretty likely that they are not the part of the current project. Which 
can be Emacs (which doesn't depend on any load-path directories outside 
of its tree; it just helps them work).

b) We have no project files to determine the actual dependencies of the 
little Elisp package you're working on, and figure out which projects 
it's tightly coupled to.

> The obvious answer is "because there is no way to tell which
> subset of load-path is treated as user-editable".
>
> Note that "owned by the user" is not enough; ELPA packages are not editable,
> but they are in ~/.emacs.d/elpa, owned by the user. And a user may have
> installed Emacs in their own directory tree.

"editable" and "owned by the user" is irrelevant. The docstring doesn't 
mention either of those conditions.

> Which is precisely my point; there is no good way to decide whether a
> directory belongs in project-roots or project-library-roots.

Not if you only take one word from the whole docstring. It says 
"directory ... contents meant to be edited together".

Which, yes, if impossible to determine for load-path, which is why it 
goes into library-roots.

But it's possible to answer that question in other languages and 
configurations.

Re: project.el semantics

[Dmitry Gutov]

On 11/11/2015 11:44 AM, Stephen Leake wrote:

> There is also the notion of "editable" vs "non-editable", in the doc
> string for project-roots.

Not "editable". "Edited together". If that phrase looks confusing from 
the outset, please suggest a synonym.

> This doesn't tell me whether a dependency that is not a "library"
> belongs in project-root or project-library-root.
>
> But my main problem with this is that it is far too limiting.
>
> There are many other possible use cases for restricting a search over
> the list of directories related to a project:

Yes. I also want the user to be able to do whatever and wherever they 
want, but we must have a decent UI, too.

Maybe add the ability to tag directories with some metadata? Like vendor 
or... I don't know what else. :) The rest of your list deals with 
transient properties. They could also work if some third-party tools 
were able to modify the directory metadata.

> - Search in all dependencies that come from Google (or some other
>    vendor).
>
> - Search only in directories that are not read-only.
>
> - Search only in directories that are not marked "passed unit testing".
>
> - Search only in directories that are not marked "refactoring for
>    feature 'foo' finished".
>
> - etc.
>
> We cannot possibly anticipate the set of restrictions some user might
> want to put on a search. So why should we assume "search on libraries"
> is so important that it needs this level of attention?

Because it's easy to implement, and because that's the distinction that 
I wanted to have first. Which is like 50% of our userbase at this point.

And also because it correlates with "code ownership", which is a fairly 
common notion.

Re: project.el semantics

[John Wiegley]

The among of discussion on project.el semantics makes it pretty clear that
there is a lack of both clarity and consensus among us. We should take a step
back to define what we want project.el to actually "do", in exact terms. We
can then look at the implementation through that lens, and make changes
accordingly.

Having read this thread, and looked at the code, I'm still not clear on what a
"project root" is, what a "project library root" is, why it's different, and
what "outside the project" means.

For this to be ready for 25.1, I would like a dead simple definition of what
project.el provides, something that takes up maybe a page in the Emacs manual.
I *think* it's a core API for identifying directories and/or files within some
notion of a "project", so that tooling can build on top of it to search source
files, ignore build products in dired, create TAGS, etc.

John

Re: project.el semantics

[Dmitry Gutov]

On 11/11/2015 06:48 PM, John Wiegley wrote:
> The among of discussion on project.el semantics makes it pretty clear that
> there is a lack of both clarity and consensus among us. We should take a step
> back to define what we want project.el to actually "do", in exact terms. We
> can then look at the implementation through that lens, and make changes
> accordingly.

It's supposed to be a generic replacement for the top-level EDE 
ede-project class, more or less.

> Having read this thread, and looked at the code, I'm still not clear on what a
> "project root" is, what a "project library root" is, why it's different, and
> what "outside the project" means.

There are places in the filesystem on the user machine that are not 
"inside" the project.

Those are outside. Does "external to the project" sound better?

> For this to be ready for 25.1, I would like a dead simple definition of what
> project.el provides, something that takes up maybe a page in the Emacs manual.

Since you both don't understand the few sentences that seem clear to me, 
I'm apparently an utter failure of a technical writer. Which is not 
terribly surprising, considering it's my second language.

Volunteers to write the manual are welcome.

> I *think* it's a core API for identifying directories and/or files within some
> notion of a "project", so that tooling can build on top of it to search source
> files, ignore build products in dired, create TAGS, etc.

Pretty much. But:

Not "within the project", but related to the project. Does the term 
"library" sound familiar?

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> It's supposed to be a generic replacement for the top-level EDE ede-project
> class, more or less.

Except I don't know what that is. :(

> Those are outside. Does "external to the project" sound better?

Ah, I think so, yes. "Ancillary roots" might be even better.

> Since you both don't understand the few sentences that seem clear to me, I'm
> apparently an utter failure of a technical writer. Which is not terribly
> surprising, considering it's my second language.

I don't mean to criticize your writing. It's hard to speak at the level of
precise specification.

> Not "within the project", but related to the project. Does the term
> "library" sound familiar?

OK, this is starting to make more sense now. So, you're saying that *within* a
project there are both distinguished directories, and file subsets with common
meaning; and *outside* the project (say, in "/usr/include"), that are likewise
distinguished directories and file subsets with common meaning.

What we may want to do is avoid the concept "root" entirely, and talk instead
about general "filesets": That is, every "project" would have 0 or more
associated filesets, and a way to identify which project (perhaps multiple!)
that a given buffer is associated with.

A fileset could be defined as:

    Everything within a directory
    Everything within a directory tree
    Everything within a directory or tree matching a predicate

In the most general case, a fileset is determined by whatever function the
user provides in .dir-locals.el.

Filesets in turn have an extensible set of attributes:

    Is the file part of the project, or external to it?
    Is it under version control?
    Is it a build product?
    Should it be searched, tagged, presented in various contexts, etc.?
      (This itself might be an extensible sublist)

This way we can talk in general terms, and not about concrete roots or
directories. In the most abstract case, the notion of "project" should be
entirely definable by the user, and may look nothing like what we presently
have in mind. It's even possible that a buffer without an associated file, say
a *Compilation* buffer, could momentarily be considered part of a project, and
as a candidate for cross-buffer searching within that project.

John

Re: project.el semantics

[Dmitry Gutov]

On 11/11/2015 07:22 PM, John Wiegley wrote:

>> It's supposed to be a generic replacement for the top-level EDE ede-project
>> class, more or less.
>
> Except I don't know what that is. :(

ede-project is defined in lisp/cedet/ede/base.el. I think its defclass 
declaration is all that you need to know for this discussion.

It's the top parent of any EDE project. If someone wants to write a 
command acting on the current project, and they want it to work on any 
kind of EDE project, they would have to use the info that ede-project 
provides.

>> Those are outside. Does "external to the project" sound better?
>
> Ah, I think so, yes. "Ancillary roots" might be even better.

I meant "external to the project" instead of "outside the project", of 
course. In the project-library-roots docstring. But you seem to mean a 
rename, right?

Do you really think "project-ancillary-roots" is better than 
"project-library-roots"? The word "ancillary" doesn't mean anything (or 
it can mean basically anything). "Library" at least evokes some 
associations.

>> Since you both don't understand the few sentences that seem clear to me, I'm
>> apparently an utter failure of a technical writer. Which is not terribly
>> surprising, considering it's my second language.
>
> I don't mean to criticize your writing. It's hard to speak at the level of
> precise specification.

Yet by now I've spent a gargantuan amount of time writing these emails, 
as well as comments and docstrings in project.el. If reading them 
doesn't make things clearer to other people, maybe I should take up 
another, more productive hobby.

I honestly expected an "I see what you mean", or at least some targeted 
questions about the sentences I've already written. Not "I still don't 
understand what X and Y means".

>> Not "within the project", but related to the project. Does the term
>> "library" sound familiar?
>
> OK, this is starting to make more sense now. So, you're saying that *within* a
> project there are both distinguished directories, and file subsets with common
> meaning;

A project consists of directories. Most often, it consists of just one 
directory, but there are odd configurations where it's spread over 
several directories (Stephen gave some examples). It's not hard to 
support that case, so it's "project roots" now, instead of "project root".

We've not reached the stage where we want to group different kinds of 
files within the project somehow, yet. But we have project-ignores, so 
you might say it splits files into two groups (ignored and not).

> and *outside* the project (say, in "/usr/include"), that are likewise
> distinguished directories and file subsets with common meaning.

Yes. And `project-library-roots-function' docstring mentions "headers 
search path, load path, class path".

> What we may want to do is avoid the concept "root" entirely, and talk instead
> about general "filesets": That is, every "project" would have 0 or more
> associated filesets, and a way to identify which project (perhaps multiple!)
> that a given buffer is associated with.

That suggestion seems orthogonal. Does "project library filesets" sound 
more clear to you than "project library roots" or "project library root 
directories"? Does the difference between "project library filesets" and 
"project filesets" become more apparent?

The names ending with "-roots" mean the elements of the returned lists 
are treated in a recursive fashion (everything within each of project 
roots, including subdirectories, belongs to the project).

> A fileset could be defined as:
>
>      Everything within a directory
>      Everything within a directory tree
>      Everything within a directory or tree matching a predicate
>
> In the most general case, a fileset is determined by whatever function the
> user provides in .dir-locals.el.

What function? Each user will have to write a function now?

I'm happy to discuss patches along these lines, but please note that 
you're proposing to drastically increase the API's complexity, and 
you're not solving any of what I see as the two immediate problems (and 
the "feature freeze" is the day after tomorrow):

- The disagreement around what "library roots" means (or library 
filesets, etc). And whether we should have that in the API.

- Dealing with "library roots" for projects with multiple source 
languages inside, in an automatic fashion, without a combinatoric 
explosion of project backends. This is the first FIXME in project.el 
talks about. Would you like to comment on it?

> Filesets in turn have an extensible set of attributes:
>
>      Is the file part of the project, or external to it?
>      Is it under version control?
>      Is it a build product?
>      Should it be searched, tagged, presented in various contexts, etc.?
>        (This itself might be an extensible sublist)

I'd expect to see a justification for each of these features.

"Under version control" - this question is answered by VC. Why would we 
have that in the project API?

"build product" - why not simply ignore those using project-ignores?

"Should it be searched, tagged, presented in various contexts" - that 
description sounds like a pony.

> This way we can talk in general terms, and not about concrete roots or
> directories. In the most abstract case, the notion of "project" should be
> entirely definable by the user, and may look nothing like what we presently
> have in mind.

The more flexible the API allows every project to me, the more trouble, 
in general, the consumers of the API are going to have with it.

Example one:

You want to have "filesets" instead of simply directories. We'll have a 
new data structure to decide on, and the API consumers will have figure 
out what filesets are, and how to work with them. Directories are 
trivial in comparison.

Two:

You want filesets to be allowed to include only immediate directory 
contents, or the contents of the whole directory tree.

That means that any consumer will have to handle both of these cases. 
E.g. xref-collect-matches will have to use two different code paths to 
handle both recursive and non-recursive traversals.

As we've discussed with Stephen previously, a "directory to traverse 
non-recursively" is equivalent to "directory to traverse recursively, 
but ignoring all its subdirectories", and the second part can be 
implemented via project-ignores (that lists all subdirectories of the 
directory in question). That results in just one code path in all consumers.

Stephen still wanted to support both recursive and non-recursive 
traversals explicitly, so if you both insist on that, I guess I will 
give in.

Three:

Allowing "Everything within a directory or tree matching a predicate" 
means, I believe, that searching through the fileset won't be possible 
to implement with just one find+grep call. If that involves calling the 
predicate at each directory level, that is.

> It's even possible that a buffer without an associated file, say
> a *Compilation* buffer, could momentarily be considered part of a project, and
> as a candidate for cross-buffer searching within that project.

And that feature would mean that one can't implement searching through 
the project with find+grep at all. Do you consider it valuable enough, 
to give up on that simplicity?

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

>> In general, project.el is not about the final user interface, it is
>> about providing core functionality.
>
> Providing core functionality for third-party code that's not, usually,
> written by the user either. So it's exactly the business of project.el
> to include the kind of metadata like "these are the directories the
> user might be interested in, but the user doesn't own the code
> inside".

Ok, I can see the need for some sort of metadata. But that suggests:

(cl-defgeneric project-metadata (project dir)
"Return interesting metadata for DIR in PROJECT."
...)

We need more examples of useful metadata before we can begin to design a
format for the return result.

>> Any case where a large project is divided into subprojects, all of which
>> are editable. For example, my NASA GDS projects looked like this:
>>
>> GDS
>>      sal
>>
>>      common
>>
>>      map
>>
>>      dscovr
>>
>> All four of these are projects. "sal" and "common" are lower level; they
>> are used by "map" and "dscovr", which are the projects used by the MAP
>> and DSCOVR missions respectively (you can Google those ...). "sal" and
>
> You work on very cool projects, no question about it. :)

Used to; I'm retired now. I worked on robots before that; even more fun.

>> "common" are listed as dependencies in the "map" and "dscovr" Ada
>> project files. Nothing in "map" depends on "dscovr", and vice-versa.
>
>> So for the dscovr project, what goes in project-roots, and what goes in
>> project-library-roots, and why?
>
> Then project-roots will contain dscovr, sal and common. And
> library-roots will have what's left in the search path (Ada runtime
> library).

You did not answer the most important part of the question: "why?".

I'm left with "in order to decide which directories go in project-root
vs project-library-root, ask Dmitry".

> Personally, I expect that to be useful (e.g. limiting searches to the
> projects that you control). But if not, you an also disregard that
> distinction. Do you see any downsides to it?

Yes; it's not flexible enough. There were times when I wanted to search
only in map/, or only in sal/. I just used grep-find.

I could only dream about the other use cases I described, but they did
come up. For refactoring, I would make a low-level change, that would
generate lots of compiler errors (Ada is a big help here). Then I would
maintain a manual list of which ones were fixed in a text file.

Some more structured system would be nice.

>> Note that the Ada project tool provides only one "source search path";
>> it does not distinguish between "libraries" and "non-libraries". For the
>> dscovr project, "dscovr", "sal", "common", and the runtime library are
>> in the search path.
>
> I'm hoping that you can still distinguish sal and common from the
> standard library somehow, programmatically.

Only by knowing the directory layout of the disk; the Ada project
doesn't care, so it doesn't know.

>> xref-find-references needs similar treatment to be consistent. But I
>
> Thus far xref-find-references doesn't depend on the presence of
> "current project" (some implementations might use, some might not).
>
> So creating two variations of xref-find-references would be
> incongruous, even if maybe useful.

Right. On the other hand, a user-defined predicate would be a
reasonable feature for xref-find-references.

> If you only accept user-specified predicates, you're assuming that the
> project backend can't provide any useful information to classify the
> directories. 

The predicate can query the metadata, via project-metadata.

> I'm open to the ideas in this direction, but the UI must be better than M-:.

We have to agree that the UI is a separate discussion, or we will get nowhere.

-- 
-- Stephe

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> I meant "external to the project" instead of "outside the project", of
> course.

What is the difference between these two? To my ears, they mean exactly the
same thing.

> Do you really think "project-ancillary-roots" is better than
> "project-library-roots"?

Neither sounds good, actually.

I think we're too focused on "projects" having to do with code, where code
uses libraries. A "library-root" presumes things about the content of the
project which I suggest we don't need to presume.

The filesets idea was not meant to complicate the API, but to step back and
reassess the API we really want. I understand now why you chose "root" as a
suffix, and that much makes sense.

Here's where I'm coming from:

When I'm sitting at a buffer in Emacs, that buffer is usually related to
something. It could be just a buffer (like *scratch*) with no relation at all,
but that's pretty rare.

Typically, the only relationship it has is to a file. But it may have other
relationships:

  - To other buffers
  - To other files
  - To other directories
  - To other directories trees

The selection of these "other" things can be based on many, many different
criteria: same file type, same directory, same "project" in the version
control sense, same modification day, etc., etc.

Why confine ourselves to thinking about code projects that use libraries, when
we can imagine a larger picture, where the original idea is just a particular
set of choices?

I'd love to have a programmatic API for associating files and buffers, and a
set of commands able to take these associations into account. I'd use them for
lots of things that have nothing to do with source code. When I'm editing a
Markdown file, for example, I might have a preview buffer showing me a PNG of
the rendered result. These buffers are all related to my current "project",
and I'd like a way to "save all files in project", or "close all buffers in
project".

Projectile gives me some of that today, but it's very much tied to Git and
filesystem searching -- which are great default backends for this type of
functionality, mind you; they just shouldn't be the limit of our thinking.

This is one of the "IDE features" I mentioned several weeks ago, which is why
I want us to spend more time thinking about it. Why start out with a smaller
API that does a very focused thing, when we can imagine something much more
capable?

John

Re: project.el semantics

[Stephen Leake]

John Wiegley <jwiegley@gmail.com> writes:

>>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:
>
>> It's supposed to be a generic replacement for the top-level EDE ede-project
>> class, more or less.
>
> Except I don't know what that is. :(

Previouslyl, we have said that a project.el object is not a replacement
for EDE projects. It makes sense for project.el to be used as a wrapper
for EDE projects.

The project.el header says it fairly well:

;; This file contains generic infrastructure for dealing with
;; projects, and a number of public functions: finding the current
;; root, related project directories, search path, etc.
;;
;; The goal is to make it easy for Lisp programs to operate on the
;; current project, without having to know which package handles
;; detection of that project type, parsing its config files, etc.

EDE can be used as a project.el backend, to do the "detection of project
type, parsing its config files, etc".

projectile could be another backend.

elisp packages that interface with build tools like ant or maven could
be other backends.

Then higher level code, like locate-file or xref-find-definition, can
use the project.el API to access project features. Mostly search-path,
at the moment; the intent is that there be other project features in the
future.

> OK, this is starting to make more sense now. So, you're saying that *within* a
> project there are both distinguished directories, and file subsets with common
> meaning; and *outside* the project (say, in "/usr/include"), that are likewise
> distinguished directories and file subsets with common meaning.
>
> What we may want to do is avoid the concept "root" entirely, and talk instead
> about general "filesets": That is, every "project" would have 0 or more
> associated filesets, 

Yes, I think roots (more precisely, a list of roots and ignores, or an
explicit list of directories like `load-path') is a proxy for the
project fileset.

> and a way to identify which project (perhaps multiple!)
> that a given buffer is associated with.

Yes; that's project-current.

> A fileset could be defined as:
>
>     Everything within a directory
>     Everything within a directory tree
>     Everything within a directory or tree matching a predicate

or:

      Everything within the directory trees on a path

> In the most general case, a fileset is determined by whatever function the
> user provides in .dir-locals.el.

That's not the most general case; that's just one particular project
backend; the vc backend.

Other project backends, for EDE or gradle, will have other ways of
defining the filesets.

I'm working on implementing a couple of these, but I don't seem to be
moving very fast on it.

> Filesets in turn have an extensible set of attributes:
>
>     Is the file part of the project, or external to it?

There are actually three categories:

- member of the top level project

- member of a dependency of the top level project

- everything else

So, if "the project" is defined as "all elisp files on load-path", then
foo.c is in "everything else", even if it is in a load-path directory.

>     Is it under version control?

Emacs already has a standard API for version control, but it might make
sense to wrap it in a project metadata structure.

>     Is it a build product?

That would be useful.

>     Should it be searched, tagged, presented in various contexts, etc.?
>       (This itself might be an extensible sublist)

Some of this can be a permanent attribute of the file, but it depends on
the context. As I pointed out in another message, some contexts (like
"refactor foo") are transient, and defined by the user, not the fileset.

> This way we can talk in general terms, and not about concrete roots or
> directories. In the most abstract case, the notion of "project" should be
> entirely definable by the user, 

Better, "definable by the backend". I reserve "user" in this case for
the person that is running the code that uses project.el. The project
backends define what attributes the project filesets can have.

But maybe you meant something like "the user writes the project files
that the build tool uses" and which the project backend uses to define
what the project is; that's true.

> and may look nothing like what we presently have in mind. It's even
> possible that a buffer without an associated file, say a *Compilation*
> buffer, could momentarily be considered part of a project, and as a
> candidate for cross-buffer searching within that project.

Right; clearly a compilation buffer should be associated with a project.
Currently, since project-current uses directory-based project detection,
this works well. EDE relies mostly on directory-based detection as well.

I prefer letting the user specify a single active project; that way the
same search path is used no matter what buffer a search is started from.
That's partly because that's what the ada-mode code does (I maintain
ada-mode, and used it extensively at NASA); I'm trying to see what
advantages there are to a more fragmented project approach.

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/12/2015 12:14 AM, Stephen Leake wrote:

> Ok, I can see the need for some sort of metadata. But that suggests:
>
> (cl-defgeneric project-metadata (project dir)
> "Return interesting metadata for DIR in PROJECT."
> ...)

Yes, more or less. Instead of directories, we could have complex objects 
like John suggests, with their own properties, but the above seems be 
simpler.

> We need more examples of useful metadata before we can begin to design a
> format for the return result.

Probably. I just have in mind one tag so far, e.g. let's call it 
"category". There might be none (in which case the directory is a part 
of the project), it might be "library", and/or something else.

> Used to; I'm retired now.

That's a pretty good job as well. :)

>> Then project-roots will contain dscovr, sal and common. And
>> library-roots will have what's left in the search path (Ada runtime
>> library).
>
> You did not answer the most important part of the question: "why?".

Because, from dscovr depends on sal and common, and according to your 
description, the person working on dscovr can make changes to sal and 
common as well, if needed. So a search across "current project, as well 
as code I edit together with it" would include dscovr, sal and common.

Which sounds useful to me, but I don't really know your workflows. If 
the dscovr developer doesn't need to edit sal and common 98% of their 
time, maybe they should indeed go into project-library-roots.

If they only need to edit sal or common for like 5-20% of their work on 
dscovr, maybe a finer distinction is needed, like three kinds of searches:

- Across project-roots and project-library-roots.

- project-roots only.

- The one project roots the current buffer is in.

Which complicates things a bit, but it's still simpler than having user 
predicates and arbitrary metadata.

> I'm left with "in order to decide which directories go in project-root
> vs project-library-root, ask Dmitry".

Everybody should Ask Dmitry more often.

>> Personally, I expect that to be useful (e.g. limiting searches to the
>> projects that you control). But if not, you an also disregard that
>> distinction. Do you see any downsides to it?
>
> Yes; it's not flexible enough. There were times when I wanted to search
> only in map/, or only in sal/. I just used grep-find.

You can also use C-u M-x project-find-regexp, and point it to map/ or 
sal/. Specifying a single directory is relatively easy anyway.

> I could only dream about the other use cases I described, but they did
> come up. For refactoring, I would make a low-level change, that would
> generate lots of compiler errors (Ada is a big help here). Then I would
> maintain a manual list of which ones were fixed in a text file.

I save test errors log from the Compilation buffer to file too, from 
time to time.

> Some more structured system would be nice.

I think a harder question would be, how do you want the list to be 
updated. Apparently you don't want to just recompile the whole project, 
or you wouldn't need the text file.

>> I'm hoping that you can still distinguish sal and common from the
>> standard library somehow, programmatically.
>
> Only by knowing the directory layout of the disk; the Ada project
> doesn't care, so it doesn't know.

If all else fails, they could be tagged with a directory-local variable, 
via .dir-locals.el. That would also give the choice of "is this library 
or not" to the user.

> Right. On the other hand, a user-defined predicate would be a
> reasonable feature for xref-find-references.

It might be.

> The predicate can query the metadata, via project-metadata.

I can display all available metadata to the user, and ask: directories 
with which pieces of metadata set do you want to search? The user enters 
the category names, separated by spaces, and the search proceeds.

I can't do the same with predicates. I can't even list them all, really.

> We have to agree that the UI is a separate discussion, or we will get nowhere.

The API should take possible UIs into account. Like described above, if 
we standardize on metadata, I can imagine at least one UI. If we 
standardize on arbitrary predicates, it's much harder.

This kind of change to the API should even come with a corresponding 
change proposal for xref-find-references, to demonstrate that it's viable.

Re: project.el semantics

[Dmitry Gutov]

On 11/12/2015 12:30 AM, John Wiegley wrote:

>> I meant "external to the project" instead of "outside the project", of
>> course.
>
> What is the difference between these two? To my ears, they mean exactly the
> same thing.

Not much. You didn't like one word, so I suggested another, to described 
the same thing.

> I think we're too focused on "projects" having to do with code, where code
> uses libraries. A "library-root" presumes things about the content of the
> project which I suggest we don't need to presume.

"Project libraries" paint a quick and simple picture. Which, I imagine, 
might be good for the first adopters of the API. I'm not going to hold 
on tight to it, but please keep in mind that simple is often good.

> The filesets idea was not meant to complicate the API, but to step back and
> reassess the API we really want. I understand now why you chose "root" as a
> suffix, and that much makes sense.
>
> Here's where I'm coming from:
>
> When I'm sitting at a buffer in Emacs, that buffer is usually related to
> something. It could be just a buffer (like *scratch*) with no relation at all,
> but that's pretty rare.

In the current concept, the file, or the non-file-visiting buffer, is 
related to its default-directory. Then `project-current' takes that 
directory and determines the current project based on it, using 
project-find-functions.

project-find-functions being explicitly documented to depend only on the 
directory, but not on the contents of the current buffer, is a conscious 
decision: I don't want to see foo.el belong to one project, and 
README.md lying nearby in the same dir, to another, from the standpoint 
of this API. Do you disagree?

> Typically, the only relationship it has is to a file. But it may have other
> relationships:
>
>    - To other buffers

Other buffers also often have a default-directory set. So we can extend 
to them the relations that the original buffer has to their directories.

>    - To other files
>    - To other directories
>    - To other directories trees

If you intend to keep the tree/no-tree distinction, please keep in mind 
my comment from the previous email.

> The selection of these "other" things can be based on many, many different
> criteria: same file type, same directory, same "project" in the version
> control sense, same modification day, etc., etc.
>
> Why confine ourselves to thinking about code projects that use libraries, when
> we can imagine a larger picture, where the original idea is just a particular
> set of choices?

The API clients need to have an idea of the kinds of relations the API 
is dealing with. Otherwise, how would they ask the API anything?

For instance, what will project-find-regexp do? Will it query the API 
for a list of directories related to the current buffer/file/directory? 
What kind of directories? What kind of relations will it ask for?

Will we have anything like project-or-libraries-find-regexp? What will 
it change to?

What will the project-find-file command (current not implemented) look like?

> I'd love to have a programmatic API for associating files and buffers, and a
> set of commands able to take these associations into account. I'd use them for
> lots of things that have nothing to do with source code.

Why don't we save altering operations for the project backends? The API 
should work with existing backends as well (such as Projectile and EDE), 
and I'm not sure if they can support what you're describing.

> When I'm editing a
> Markdown file, for example, I might have a preview buffer showing me a PNG of
> the rendered result. These buffers are all related to my current "project",
> and I'd like a way to "save all files in project", or "close all buffers in
> project".

That's a very simple example, which can be served by the current API 
just as well. Do you have any example that will show off the difference?

> This is one of the "IDE features" I mentioned several weeks ago, which is why
> I want us to spend more time thinking about it. Why start out with a smaller
> API that does a very focused thing, when we can imagine something much more
> capable?

More capable is better than less, provided everything else is equal. But 
carefully considered limitations are also valuable, because they protect 
from misuse and other surprises by both backends and consumers.

I don't really see the end result from your description. Judging from 
just that message, you either want to remove project-library-roots and 
add a bunch of some other accessors instead, or remove all current 
cl-defgeneric declarations and declare each project to be a hash-table, 
using some sets of keys, both predefined (?) and not, and documented to 
have files and directories as values. The first option is simply vague, 
the second one, well, writing code for it would definitely be more 
error-prone: currently, if you mistype the name of a generic function, 
at least you get a warning.

You'll have to give more specifics either way.

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/12/2015 12:14 AM, Stephen Leake wrote:
>
>> Ok, I can see the need for some sort of metadata. But that suggests:
>>
>> (cl-defgeneric project-metadata (project dir)
>> "Return interesting metadata for DIR in PROJECT."
>> ...)
>
> Yes, more or less. Instead of directories, we could have complex
> objects like John suggests, with their own properties, but the above
> seems be simpler.
>
>> We need more examples of useful metadata before we can begin to design a
>> format for the return result.
>
> Probably. I just have in mind one tag so far, e.g. let's call it
> "category". There might be none (in which case the directory is a part
> of the project), 

If I pass in a DIR that is _not_ related to the project, the category
should be something like "unknown". So we have at least four values for
category:

unknown
top-level (ie "the project")
library
other-dependency

But then there is also, orthogonal to category:

editable (or read-write)
read-only

>>> Then project-roots will contain dscovr, sal and common. And
>>> library-roots will have what's left in the search path (Ada runtime
>>> library).
>>
>> You did not answer the most important part of the question: "why?".
>
> Because, from dscovr depends on sal and common, and according to your
> description, the person working on dscovr can make changes to sal and
> common as well, if needed. 

So you are focusing on the "read-only" vs "read/write" aspect.

> So a search across "current project, as
> well as code I edit together with it" 

This is project-roots.

> would include dscovr, sal and common.
>
> Which sounds useful to me, but I don't really know your workflows. If
> the dscovr developer doesn't need to edit sal and common 98% of their
> time, maybe they should indeed go into project-library-roots.

Yes, the developer will want to search different subsets of the full
project at different times.

Which is why this needs to be more flexible.

The user should be able to specify a subset of the search path for each
search, at run-time. Having only project-roots and project-library-roots
available is too limiting.

> If they only need to edit sal or common for like 5-20% of their work
> on dscovr, maybe a finer distinction is needed, like three kinds of
> searches:
>
> - Across project-roots and project-library-roots.
>
> - project-roots only.
>
> - The one project roots the current buffer is in.
>
> Which complicates things a bit, but it's still simpler than having
> user predicates and arbitrary metadata.

The UI for a predicate is more complicated, yes. But the project.el API
is simpler; a single predicate arg is clearly simpler than the confusion
between project-roots and project-library-roots.

Which is why I keep emphasizing that project.el is _not_ about the UI.

>>> Personally, I expect that to be useful (e.g. limiting searches to the
>>> projects that you control). But if not, you an also disregard that
>>> distinction. Do you see any downsides to it?
>>
>> Yes; it's not flexible enough. There were times when I wanted to search
>> only in map/, or only in sal/. I just used grep-find.
>
> You can also use C-u M-x project-find-regexp, and point it to map/ or
> sal/. Specifying a single directory is relatively easy anyway.

Not with the current API:

(defun project-find-regexp (regexp)
...)

How would I specify a single root or directory?

>> The predicate can query the metadata, via project-metadata.
>
> I can display all available metadata to the user, and ask: directories
> with which pieces of metadata set do you want to search? The user
> enters the category names, separated by spaces, and the search
> proceeds.

That would be a dialog-box style UI, yes.

> I can't do the same with predicates. I can't even list them all,
> really.

The dialog box uses the user input to construct a predicate, which it
passes to the project.el UI. 

That's an example of what I had in mind; higher level code implementing
a nice UI to drive project.el functions.

The alternative to an arbitrary predicate function is to hard-code some
set of possible choices in the search code. Any such set is limiting.

It may be that we discover some commonly used predicates; they could be
standardized in project.el.

On the GDS project, one search we repeated often was for FIXME:
comments; we had some structure in them, identifying which release it
had to be fixed for, who was assigned to fix it, etc. So I could have
written a few predicates that take advantage of the structure, and used
a menu to select among them for a search.

>> We have to agree that the UI is a separate discussion, or we will get nowhere.
>
> The API should take possible UIs into account. Like described above,

Yes. Metadata query functions and search predicates work for a very
large variety of UIs (menus, dialog boxes, Siri), as well as a very
large variety of searches. "project-roots vs project-library-roots"
works for a similar set of UIs but a much smaller set of searches.

> if we standardize on metadata, I can imagine at least one UI. If we
> standardize on arbitrary predicates, it's much harder.

Metadata is provided by the backend to the client code. Predicates
are provided by the client code to the backend. They are complementary,
not mutually exclusive.

> This kind of change to the API should even come with a corresponding
> change proposal for xref-find-references, to demonstrate that it's
> viable.

Yes, I'll put it on my list.

-- 
-- Stephe

Re: project.el semantics

[Steinar Bang]

>>>>> Dmitry Gutov <dgutov@yandex.ru>:

> On 11/11/2015 11:55 AM, Stephen Leake wrote:
>> Dmitry Gutov <dgutov@yandex.ru> writes:
>> 
>>> project vs libraries. Or, "files we own and edit" vs "files we
>>> sometimes want to look at" (and maybe edit during debugging).

>> Those are conflicting criteria, so you are proving my point.

> I don't see that.

(FWIW I didn't see that either...? The distinction Dmitry made, made
perfect sense to me, with my Java and Python glasses on (...and Java and
Python is where I currently want better emacs support))

Re: project.el semantics

[Dmitry Gutov]

On 11/12/2015 08:44 AM, Stephen Leake wrote:

> If I pass in a DIR that is _not_ related to the project, the category
> should be something like "unknown". So we have at least four values for
> category:

Why? The API consumer can determine that the directory is unrelated if 
it's not in project-roots or project-library-roots. Or whatever 
differenly named accessor we will have.

> unknown
> top-level (ie "the project")
> library
> other-dependency
>
> But then there is also, orthogonal to category:
>
> editable (or read-write)
> read-only

Is it orthogonal, though? I'd think top-level or other-dependency means 
"read-write", and the rest is "read-only".

But if it is orthogonal in some cases, I doubt a project backend can 
choose which directories are read-write. Then we don't need to document 
these, and leave them as possibly user-defined categories, for later.

> So you are focusing on the "read-only" vs "read/write" aspect.

I'm focusing on the "edited together" relation. Not on whether you can 
write to a particular directory.

>> So a search across "current project, as
>> well as code I edit together with it"
>
> This is project-roots.

Indeed.

> The user should be able to specify a subset of the search path for each
> search, at run-time. Having only project-roots and project-library-roots
> available is too limiting.

Let's be realistic. I doubt most people are going to use arbitrary 
predicates and searches that only touch some of the library-roots, as 
well as some of project-roots but other others, often. So we can serve 
the simplest case first.

Let's just keep that in mind, at least.

> The UI for a predicate is more complicated, yes. But the project.el API
> is simpler; a single predicate arg is clearly simpler than the confusion
> between project-roots and project-library-roots.

That doesn't help much if the API consumer doesn't know how to get the 
predicate from the user. *That's* a complication.

> Which is why I keep emphasizing that project.el is _not_ about the UI.

I imagine most of API consumers to be user-level commands.

> Not with the current API:
>
> (defun project-find-regexp (regexp)
> ...)
>
> How would I specify a single root or directory?

Like I said: "C-u M-x". Use the prefix argument; it's documented in its 
docstring. The API doesn't factor into it.

>> I can display all available metadata to the user, and ask: directories
>> with which pieces of metadata set do you want to search? The user
>> enters the category names, separated by spaces, and the search
>> proceeds.
>
> That would be a dialog-box style UI, yes.

I was thinking more of completing-read.

> The dialog box uses the user input to construct a predicate, which it
> passes to the project.el UI.

What kind of input? How does in turn into a predicate? Does the user 
write a whole predicate function in there?

> The alternative to an arbitrary predicate function is to hard-code some
> set of possible choices in the search code. Any such set is limiting.

You see, the usage of predicate will not be coded into the API either 
way; there's no need for it, the API doesn't care, it just returns 
directories the consumer will call predicates on.

But unless we document the properties the predicates can rely on, it 
amounts to exposing no information at all.

> On the GDS project, one search we repeated often was for FIXME:
> comments; we had some structure in them, identifying which release it
> had to be fixed for, who was assigned to fix it, etc. So I could have
> written a few predicates that take advantage of the structure, and used
> a menu to select among them for a search.

To find all FIXMEs, you have to run the search on all directories first, 
don't you? And then, when you get the results, you can filter down the 
matches. That has nothing to do with which directories to search, I think.

> Yes. Metadata query functions and search predicates work for a very
> large variety of UIs (menus, dialog boxes, Siri), as well as a very
> large variety of searches. "project-roots vs project-library-roots"
> works for a similar set of UIs but a much smaller set of searches.

If we go the project-directory-metadata route, what other accessors will 
be left? You want to remove project-library-roots, right?

Will project-roots retain its name?

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> "Project libraries" paint a quick and simple picture. Which, I imagine,
> might be good for the first adopters of the API. I'm not going to hold on
> tight to it, but please keep in mind that simple is often good.

Simple in this case might also be misleading, since "libraries" has a very
particular connotation, whereas the functionality we're talking about could be
about so much more than libraries.

> project-find-functions being explicitly documented to depend only on the
> directory, but not on the contents of the current buffer, is a conscious
> decision: I don't want to see foo.el belong to one project, and README.md
> lying nearby in the same dir, to another, from the standpoint of this API.
> Do you disagree?

Yes, I do. This type of decision shouldn't be baked into the core API -- it
should be provided as a default configuration that *uses* that API.

I'm not opposed to what you want to achieve -- far from it. I just want to get
there a different way, which will leave open a wider road for future changes.
Right now, project.el is "baking in" a lot of decisions that I don't see as
necessary at that level. What I'm suggesting is a more general API, plus a set
of defaults, where the *end result* is likely to behave much like what you've
proposed.

> I don't really see the end result from your description. Judging from just
> that message, you either want to remove project-library-roots and add a
> bunch of some other accessors instead, or remove all current cl-defgeneric
> declarations and declare each project to be a hash-table, using some sets of
> keys, both predefined (?) and not, and documented to have files and
> directories as values. The first option is simply vague, the second one,
> well, writing code for it would definitely be more error-prone: currently,
> if you mistype the name of a generic function, at least you get a warning.
> 
> You'll have to give more specifics either way.

I think we've reached the point where it is clear that project.el, as stated,
is not ready for inclusion in 25.1. I _really do_ want a module like this, I
just want something broader. This sort of module is key to my IDE vision,
which is why I care so much about getting this right.

I would like to remove project.el from the source tree for now, and move it to
a feature branch for further discussion and development. I understand this is
disappointing, but I'm fairly certain you'll be happier with where we end up
down the road. I intend the very same functionality you're proposing now, but
in a way that will allow many more cool things besides.

John

Re: project.el semantics

[Dmitry Gutov]

On 11/12/2015 07:26 PM, John Wiegley wrote:

> Simple in this case might also be misleading, since "libraries" has a very
> particular connotation, whereas the functionality we're talking about could be
> about so much more than libraries.

That's true, too.

> Yes, I do. This type of decision shouldn't be baked into the core API -- it
> should be provided as a default configuration that *uses* that API.

In *each* project backend, separately? And then I, as a user, will have 
to hunt down every such setting in every implementation, and change it?

> I'm not opposed to what you want to achieve -- far from it. I just want to get
> there a different way, which will leave open a wider road for future changes.

Why don't you provide some justification for flexibility? I did provide 
a justification for the given restriction.

> Right now, project.el is "baking in" a lot of decisions that I don't see as
> necessary at that level. What I'm suggesting is a more general API, plus a set
> of defaults, where the *end result* is likely to behave much like what you've
> proposed.

Again, I disagree, on this particular issue.

Too bad you haven't presented any alternative design, or examples of 
features that won't work in the current design.

And I'd really like to see what that "set of defaults" is going to look 
like.

> I think we've reached the point where it is clear that project.el, as stated,
> is not ready for inclusion in 25.1. I _really do_ want a module like this, I
> just want something broader. This sort of module is key to my IDE vision,
> which is why I care so much about getting this right.

I've asked for outside input on two particular questions. You haven't 
addressed the more interesting one, but decided that you want to do 
everything different, in an unspecified way.

That's fine, as long as you're going to take up responsibility for the 
feature now. And good luck with your vision.

Also note that, as far as the discussion with Stephen is concerned, we 
may be reaching some consensus here. But hey, you're the maintainer.

> I would like to remove project.el from the source tree for now, and move it to
> a feature branch for further discussion and development. I understand this is
> disappointing, but I'm fairly certain you'll be happier with where we end up
> down the road. I intend the very same functionality you're proposing now, but
> in a way that will allow many more cool things besides.

I will agree to that, as long as you remove xref from the source tree, 
too. It has a number design questions unsolved, and I consider them much 
more important. While it works to an extent, the decisions in question 
will most likely severely change the data structures used, and I don't 
want to have to be tied by having to support the current data structures 
later.

So if you think we can't get project.el (a relatively small package) 
right in time for the release, we definitely won't be able to do that 
for xref.

Re: project.el semantics

[Dmitry Gutov]

On 11/12/2015 07:53 PM, Dmitry Gutov wrote:

> I will agree to that, as long as you remove xref from the source tree,

Or alternatively, we keep the new commands, as well as elisp and etags 
implementations, but document them briefly, and declare the extension 
interface (currently, xref-find-function), to be 
private/experimental/work-in-progress.

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> In *each* project backend, separately? And then I, as a user, will have to
> hunt down every such setting in every implementation, and change it?

I think the average user shouldn't even know this machinery exists, the
defaults should be so well chosen. We'll have a surface API for asking typical
questions like "What files should be ignored in this directory?" Only advanced
users should write any Lisp code; and they will be rewarded with the ability
to configure the system to their heart's content.

> Why don't you provide some justification for flexibility? I did provide a
> justification for the given restriction.

Because I want to use this system for things outside of those restrictions.

> Too bad you haven't presented any alternative design, or examples of
> features that won't work in the current design.
>
> And I'd really like to see what that "set of defaults" is going to look
> like.

This will take significantly more work than I can do in the time I have today.
Proving to you that the alternative design is better will require me to write
some of it, and that will need several days to work out at the very least.

> That's fine, as long as you're going to take up responsibility for the
> feature now. And good luck with your vision.

Thanks, Dmitry, although I sense more frustration than congratulation in your
wish... I certainly hope you will continue to work with me on this feature,
since I'd like you to be in charge of the defaults, since in that area, your
restrictions are most likely the right ones.

To be clear: I'm not saying I don't like your vision. I just want your vision
to be built on top of a broader foundation than it currently is. Your idea
addresses the needn of multi-directory projects relying on external supporting
libraries -- I want an API for talking about entity aggregation in general
(buffers, files, directories, and other resources). Within such an API,
projects and their supporting libraries should be a natural fit.

> I will agree to that, as long as you remove xref from the source tree, too.
> It has a number design questions unsolved, and I consider them much more
> important. While it works to an extent, the decisions in question will most
> likely severely change the data structures used, and I don't want to have to
> be tied by having to support the current data structures later.
>
> So if you think we can't get project.el (a relatively small package) right
> in time for the release, we definitely won't be able to do that for xref.

That's quite acceptable. I'm not in a hurry, and want to get these new APIs
right.

> Or alternatively, we keep the new commands, as well as elisp and etags
> implementations, but document them briefly, and declare the extension
> interface (currently, xref-find-function), to be
> private/experimental/work-in-progress.

Also perfectly fine, whichever you think is best for xref.el. I'm OK with
"trial features" appearing in a release, so long as we note that they may
change without notice and shouldn't be too much relied upon.

John

Re: project.el semantics

[John Mastro]

John Wiegley <jwiegley@gmail.com> wrote:
>> Or alternatively, we keep the new commands, as well as elisp and etags
>> implementations, but document them briefly, and declare the extension
>> interface (currently, xref-find-function), to be
>> private/experimental/work-in-progress.
>
> Also perfectly fine, whichever you think is best for xref.el. I'm OK with
> "trial features" appearing in a release, so long as we note that they may
> change without notice and shouldn't be too much relied upon.

Speaking as a user[1], I hope you'll take this route rather than
removing project.el and xref.el. Emacs 25.1 would be less useful to me
without them. I also hope Dmitry will continue to work on them, despite
the frustrating dynamics.

[1] Admittedly a somewhat unusual user in that I'm reading emacs-devel.

-- 
john

Re: project.el semantics

[Dmitry Gutov]

On 11/12/2015 08:17 PM, John Wiegley wrote:

> I think the average user shouldn't even know this machinery exists, the
> defaults should be so well chosen. We'll have a surface API for asking typical
> questions like "What files should be ignored in this directory?"

How can you ask the question about "this directory", if the code 
responding to the question doesn't know it's not allowed to use the 
contents of the current buffer? Aren't you bothered by the perspective 
of getting two different answers about the directory, depending on which 
file you're visiting?

> Only advanced
> users should write any Lisp code; and they will be rewarded with the ability
> to configure the system to their heart's content.

I meant more like having to M-x customize-variable for each backend I 
end up using.

>> Why don't you provide some justification for flexibility? I did provide a
>> justification for the given restriction.
>
> Because I want to use this system for things outside of those restrictions.

Well, I'm going to wait for the specifics.

But maybe you should consider using other APIs for those other things.

I wouldn't call the API a "system". From where I'm standing, it should 
just be a set of contracts tailored for a particular purpose. There's no 
"inheritance" anywhere, so you can't really reuse it for highly 
different things.

>> And I'd really like to see what that "set of defaults" is going to look
>> like.
>
> This will take significantly more work than I can do in the time I have today.
> Proving to you that the alternative design is better will require me to write
> some of it, and that will need several days to work out at the very least.

The 25.1 release is months away, judging for the feature freeze lengths 
of previous Emacs releases. So I think we could make an exception for 
project.el, considering it's basically just words, and not a lot of code 
that might need exhaustive testing.

But still, it's your decision.

> Thanks, Dmitry, although I sense more frustration than congratulation in your
> wish... I certainly hope you will continue to work with me on this feature,
> since I'd like you to be in charge of the defaults, since in that area, your
> restrictions are most likely the right ones.

Yes on first two statements, but I'll have to see what the "defaults" 
will look like.

> Also perfectly fine, whichever you think is best for xref.el. I'm OK with
> "trial features" appearing in a release, so long as we note that they may
> change without notice and shouldn't be too much relied upon.

Let's do that, then.

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/12/2015 08:44 AM, Stephen Leake wrote:
>
>> But then there is also, orthogonal to category:
>>
>> editable (or read-write)
>> read-only
>
> Is it orthogonal, though? I'd think top-level or other-dependency
> means "read-write", and the rest is "read-only".

Why define that in the API? Let the user do whatever they want.

Maybe the user is only allowed to review the code, not change it.

Maybe the user is responsible for one of the dependencies, and is 
reviewing how the top-level uses that dependency, so they are not
allowed to edit the top-level, but they can edit the dependency. 

> But if it is orthogonal in some cases, I doubt a project backend can
> choose which directories are read-write. Then we don't need to
> document these, and leave them as possibly user-defined categories,
> for later.

I don't follow. The point is to list metadata that a user might want to
know, that the backend might be able to provide.

Clearly, read/write is a useful item. One way the user can set it is via
.dir-local.

>> So you are focusing on the "read-only" vs "read/write" aspect.
>
> I'm focusing on the "edited together" relation. Not on whether you can
> write to a particular directory.

I'm assuming the "edited together" comment on `project-roots' implies
"don't edit" on `project-library-roots'. Thus project-roots is
read-write, and project-library-roots is read-only.

Note that I am not talking about OS level file permissions, but about
the logical use of those directories in the context of the project.

If that's not the case, then I'm back to being totally confused.

>> The user should be able to specify a subset of the search path for each
>> search, at run-time. Having only project-roots and project-library-roots
>> available is too limiting.
>
> Let's be realistic. I doubt most people are going to use arbitrary
> predicates and searches that only touch some of the library-roots, as
> well as some of project-roots but other others, often. So we can serve
> the simplest case first.

I agree. But the simplest case is to always search all the directories
(ie project-root and project-library-root). You are allowing that, plus
one other case.

In other words, I also doubt most people will want to search only
project-roots or only project-library-roots. But I do agree those are
reasonable use cases, among many other similar ones. So the API should
provide for them.

>> Not with the current API:
>>
>> (defun project-find-regexp (regexp)
>> ...)
>>
>> How would I specify a single root or directory?
>
> Like I said: "C-u M-x". Use the prefix argument; it's documented in
> its docstring. 

Ah, I missed that.

> The API doesn't factor into it.

That's bad; you should not be able to do something interactively that
you cannot do programmatically. This should be:

(defun project-find-regexp (regexp &optional dir)
  "Find all matches for REGEXP in the current project.
If DIR is non-nil (default prefix arg), search in that instead.
If DIR is \\[universal-argument], prompts for DIR."

I'll have to play with that to get the interactive behavior; I don't see
how to do it right now. It might be best to provide a (slightly) higher
level function that provides the interactive UI.

The prompt for dir should use completing-read on project-roots and
project-library-roots; project-ignores should not have to cope with dir
outside that list.

This can also be implemented via a predicate arg to project-find-regexp;
then project-find-regexp-dir can prompt for a directory and construct
the appropriate predicate.

>>> I can display all available metadata to the user, and ask: directories
>>> with which pieces of metadata set do you want to search? The user
>>> enters the category names, separated by spaces, and the search
>>> proceeds.
>>
>> That would be a dialog-box style UI, yes.
>
> I was thinking more of completing-read.

Ok, that's another UI.

>> The dialog box uses the user input to construct a predicate, which it
>> passes to the project.el UI.
>
> What kind of input? How does in turn into a predicate? Does the user
> write a whole predicate function in there?

The code writes the predicate function, not the user.

I'll just pick the "read-only" metadata for an example:

-*- lexical-binding: t; -*-

(let* ((read-only (completing-read "read-only?" '(t nil)))
       (predicate
         (lambda (dir)
           (eq read-only (metadata-read-only (project-metadata project dir))))))
   (project-find-regexp regexp predicate))

This could be extended to include the rest of the defined metadata.

>> The alternative to an arbitrary predicate function is to hard-code some
>> set of possible choices in the search code. Any such set is limiting.
>
> You see, the usage of predicate will not be coded into the API either
> way; there's no need for it, the API doesn't care, it just returns
> directories the consumer will call predicates on.

Well, that's one choice. But I'd prefer the predicate in the API, so I
don't have to implement the same predicate-calling code in each
higher-level function.

We can leave that for later, to see how often it really gets used.

I think completion tables is a good precedent here; they provide for an
arbitrary predicate, that could just as easily be implemented one level
up.

On the other hand, the user code can just as easily distinguish
project-roots from project-library-roots; so why is that in the API?

You can't have it both ways; project-roots and project-libarary-roots is
just one special predicate.

> But unless we document the properties the predicates can rely on, it
> amounts to exposing no information at all.

Yes, we want to document those properties. That's why I'm pushing so
hard to find out what "project-root" means; I'm trying to find out what
properties of the project you actually care about.

We seem to have settled on "read-only".

>> On the GDS project, one search we repeated often was for FIXME:
>> comments; we had some structure in them, identifying which release it
>> had to be fixed for, who was assigned to fix it, etc. So I could have
>> written a few predicates that take advantage of the structure, and used
>> a menu to select among them for a search.
>
> To find all FIXMEs, you have to run the search on all directories
> first, don't you? And then, when you get the results, you can filter
> down the matches. That has nothing to do with which directories to
> search, I think.

True, this is not a directory level filter; bad example.

Still, it illustrates the need for another predicate on the result;
that could also be in the API, or at the client level.

>> Yes. Metadata query functions and search predicates work for a very
>> large variety of UIs (menus, dialog boxes, Siri), as well as a very
>> large variety of searches. "project-roots vs project-library-roots"
>> works for a similar set of UIs but a much smaller set of searches.
>
> If we go the project-directory-metadata route, what other accessors
> will be left? You want to remove project-library-roots, right?

Yes, and merge its content into project-roots.

> Will project-roots retain its name?

Yes, that's fine.

-- 
-- Stephe

Re: project.el semantics

[Stephen Leake]

Steinar Bang <sb@dod.no> writes:

>>>>>> Dmitry Gutov <dgutov@yandex.ru>:
>
>> On 11/11/2015 11:55 AM, Stephen Leake wrote:
>>> Dmitry Gutov <dgutov@yandex.ru> writes:
>>> 
>>>> project vs libraries. Or, "files we own and edit" vs "files we
>>>> sometimes want to look at" (and maybe edit during debugging).
>
>>> Those are conflicting criteria, so you are proving my point.
>
>> I don't see that.
>
> (FWIW I didn't see that either...? The distinction Dmitry made, made
> perfect sense to me, with my Java and Python glasses on (...and Java and
> Python is where I currently want better emacs support))

Just because something is a "library" doesn't mean you can't edit it.

But I guess you are all saying that is exactly what it means; Ok.

There might also be other dependencies (not libraries) that you
should not edit, that are editable in other circumstances, via other
projects.

For example, you are on a team that maintains several top-level
projects, and a low-level utility project shared by all. You are not
allowed to change the utility project unless you are implementing a
change order. So in your everyday project, the utility dependecy is
read-only. You also have a change-order project, that makes the utility
dependency read-write.

So it would be best to have "project-read-write-roots" and
"project-read-only-roots". That would be much clearer.

Except I'd prefer a predicate, as discussed in the other emails.

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/12/2015 09:28 PM, Stephen Leake wrote:

> Why define that in the API? Let the user do whatever they want.

Or that. I'm just saying that a project backend probably wouldn't be 
able to tell.

> Clearly, read/write is a useful item. One way the user can set it is via
> .dir-local.

Yeah, maybe.

> I'm assuming the "edited together" comment on `project-roots' implies
> "don't edit" on `project-library-roots'. Thus project-roots is
> read-write, and project-library-roots is read-only.

"Don't edit when working on this project".

> Note that I am not talking about OS level file permissions, but about
> the logical use of those directories in the context of the project.

Very well.

>> Let's be realistic. I doubt most people are going to use arbitrary
>> predicates and searches that only touch some of the library-roots, as
>> well as some of project-roots but other others, often. So we can serve
>> the simplest case first.
>
> I agree. But the simplest case is to always search all the directories
> (ie project-root and project-library-root). You are allowing that, plus
> one other case.

I meant the simplest approach that still allows a certain distinctions, 
like the one we've discussed (read-only/read-write).

> In other words, I also doubt most people will want to search only
> project-roots or only project-library-roots.

I do think that many users would find only searching project-roots 
useful in practice. And that's what IDEs do: when you "search in 
project", you search only the project root(s). Maybe several projects, 
if they're open at the same time (in the same "workspace"), but not 
inside the libraries.

>> The API doesn't factor into it.
>
> That's bad; you should not be able to do something interactively that
> you cannot do programmatically. This should be:
>
> (defun project-find-regexp (regexp &optional dir)
>    "Find all matches for REGEXP in the current project.
> If DIR is non-nil (default prefix arg), search in that instead.
> If DIR is \\[universal-argument], prompts for DIR."

As you can see, you're still able to "do that programmatically". The 
modified project-find-regexp illustrates that, because it's an API 
consumer, not a part of it.

> The prompt for dir should use completing-read on project-roots and
> project-library-roots; project-ignores should not have to cope with dir
> outside that list.

I'd change this requirement as follows: search an arbitrary directory 
inside the project, but use the project-ignores value for the project 
root that it's inside.

There would be some technical issues, such as handling "rooted" ignores 
with find-grep.

> This can also be implemented via a predicate arg to project-find-regexp;
> then project-find-regexp-dir can prompt for a directory and construct
> the appropriate predicate.

I don't really see the need for the word "predicate" to appear anywhere 
near its definition, thus far.

> The code writes the predicate function, not the user.
>
> I'll just pick the "read-only" metadata for an example:
>
> -*- lexical-binding: t; -*-
>
> (let* ((read-only (completing-read "read-only?" '(t nil)))
>         (predicate
>           (lambda (dir)
>             (eq read-only (metadata-read-only (project-metadata project dir))))))
>     (project-find-regexp regexp predicate))
>
> This could be extended to include the rest of the defined metadata.

The consumer doesn't need to construct a "formal" predicate function: 
just ask the project API for a list of dirs, and filter out ones you 
don't like, maybe using cl-loop.

> Well, that's one choice. But I'd prefer the predicate in the API, so I
> don't have to implement the same predicate-calling code in each
> higher-level function.

You can use cl-remove-if-not.

> On the other hand, the user code can just as easily distinguish
> project-roots from project-library-roots; so why is that in the API?

What user code? Please remember that the user doesn't in general, write 
Elisp code.

How would project-find-regexp distinguish them?

> You can't have it both ways; project-roots and project-libarary-roots is
> just one special predicate.

You could say that.

> Yes, we want to document those properties. That's why I'm pushing so
> hard to find out what "project-root" means; I'm trying to find out what
> properties of the project you actually care about.
>
> We seem to have settled on "read-only".

Yes, but if we're going the more flexible route, I prefer the words like 
"libraries" and "dependencies". Using them, the user might make the 
choice about which directories are interesting for the current search.

> Still, it illustrates the need for another predicate on the result;
> that could also be in the API, or at the client level.

The consumer can just as well filter the results using cl-remove-if-not.

>> If we go the project-directory-metadata route, what other accessors
>> will be left? You want to remove project-library-roots, right?
>
> Yes, and merge its content into project-roots.
>
>> Will project-roots retain its name?
>
> Yes, that's fine.

Please feel free to submit a patch across these lines (adding 
project-metadata). It might be moot now, given that John decided to pull 
project.el from Emacs 25.1, but it could at least serve as a point of 
comparison for his project API proposal.

Re: project.el semantics

[Dmitry Gutov]

On 11/12/2015 08:26 PM, John Mastro wrote:

> Speaking as a user[1], I hope you'll take this route rather than
> removing project.el and xref.el. Emacs 25.1 would be less useful to me
> without them. I also hope Dmitry will continue to work on them, despite
> the frustrating dynamics.

I think xref.el stays but project.el will be pulled from Emacs 25.1. 
I'll replace the uses of project.el in xref-find-references, with direct 
calls to vc-root-dir or equivalent.

Re: project.el semantics

[Dmitry Gutov]

On 11/13/2015 12:04 AM, Dmitry Gutov wrote:

> Please feel free to submit a patch across these lines (adding
> project-metadata). It might be moot now, given that John decided to pull
> project.el from Emacs 25.1, but it could at least serve as a point of
> comparison for his project API proposal.

Scratch that. John and I had a chat, and it seems project.el has a 
chance in Emacs 25.1, provided it's changed along the lines we've been 
discussing here.

I've had some thoughts since.

Some minor things:

- We'll probably want to rename `project-roots' to 
`project-directories'. Even though they'll be treated like "roots" 
(traversed recursively) most of the time, a project backend can return 
different kinds of directories in this list, including certain 
subdirectories of the project root(s).

For instance, in can include the test roots in there. Or, I don't know, 
some other kinds of directories that aren't supposed to be traversed 
recursively.

- If we're only documenting the "categories" key inside 
project-metadata, why not have a project-directory-categories method 
instead? We can add the -metadata method, too, now or later.

- What's the category name for the directories that are the project 
roots in the current interpretation? Internal?

Bigger one:

- If each directory is supposed to be categorized, and the set of 
categories is flexible, what's going to be the counterpart to 
`project-library-roots-function'?

Do we call it project-roots-function, and allow it to return a list of 
uncategorized strings, which each project-directories impl must merge in?

Or do we allow a new variable which would return a buffer-local project 
(adhering to the same API), which the "real" project would delegate to?

Alternatively, I'm inclined to mix the first option with demoting 
project-roots-function to project-vc-directories (a mixed list of 
strings and functions). One fewer wart in the API, and the list would be 
directory-local, instead of buffer-local.

We can also mix 2 and 3 (project-vc-directories might contain 
language-specific backends, as well as strings), but that's probably too 
weird.

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> Alternatively, I'm inclined to mix the first option with demoting
> project-roots-function to project-vc-directories (a mixed list of strings
> and functions). One fewer wart in the API, and the list would be
> directory-local, instead of buffer-local.

That sounds like a better direction, from what I read. We want to avoid
"baking in" as many decisions about the *meaning* of content as possible. This
is also why I like your renaming from roots to directories.

John

Re: project.el semantics

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/13/2015 12:04 AM, Dmitry Gutov wrote:
>
>> Please feel free to submit a patch across these lines (adding
>> project-metadata). It might be moot now, given that John decided to pull
>> project.el from Emacs 25.1, but it could at least serve as a point of
>> comparison for his project API proposal.
>
> Scratch that. John and I had a chat, and it seems project.el has a
> chance in Emacs 25.1, provided it's changed along the lines we've been
> discussing here.
>
> I've had some thoughts since.
>
> Some minor things:
>
> - We'll probably want to rename `project-roots' to
> `project-directories'. Even though they'll be treated like "roots"
> (traversed recursively) most of the time, a project backend can return
> different kinds of directories in this list, including certain
> subdirectories of the project root(s).
>
> For instance, in can include the test roots in there. Or, I don't
> know, some other kinds of directories that aren't supposed to be
> traversed recursively.

Then it also needs to indicate which are recursive and which aren't. One
way is to return a cons (recursive-dirs . non-recursive-dirs).

What is the use case for this?

We have two major use cases for "the list of directories in a project":

1) call "find" in an external process - this wants only the roots,
   together with ignores.

2) iterate over each directory in elisp - this wants all the
   directories, one at a time.

1 is satisfied by project-all-roots (ie (append (project-roots prj)
(project-library-roots prj))) and project-ignores. Allowing
project-all-roots to return non-roots breaks this case.

2 is not supported directly. It is not simple to get the list of all
directories from project-all-roots and project-ignores.

For Emacs 25, we should keep this as simple as possible; provide two
functions, one that returns all recursive roots and ignores, another
that returns a non-recursive list of all directories. Backends can
provide whichever is convenient; project.el can convert between them.

> - If we're only documenting the "categories" key inside
> project-metadata, why not have a project-directory-categories method
> instead? We can add the -metadata method, too, now or later.

This should wait for Emacs 26; we don't have a solid proposal.

-- 
-- Stephe

Re: project.el semantics

[Stephen Leake]

Stephen Leake <stephen_leake@stephe-leake.org> writes:

> For Emacs 25, we should keep this as simple as possible; provide two
> functions, one that returns all recursive roots and ignores, another
> that returns a non-recursive list of all directories. Backends can
> provide whichever is convenient; project.el can convert between them.

An alternative is to provide an iterator that returns one directory at a
time; this computes the non-recursive directory list lazily.

-- 
-- Stephe

Re: project.el semantics

[John Wiegley]

>>>>> Stephen Leake <stephen_leake@stephe-leake.org> writes:

> Then it also needs to indicate which are recursive and which aren't. One way
> is to return a cons (recursive-dirs . non-recursive-dirs).

I think Dmitry and I talked about a more general way of yielding metadata
associated with project-related "entities" (mainly directories and files in
this version), and that this metadata could indicate recursion, or even how to
perform the recursion. For example, it could be recursion with a file name, or
i-node testing, predicate.

John

Re: project.el semantics

[Dmitry Gutov]

On 11/21/2015 12:03 PM, Stephen Leake wrote:

>> For instance, in can include the test roots in there. Or, I don't
>> know, some other kinds of directories that aren't supposed to be
>> traversed recursively.
>
> Then it also needs to indicate which are recursive and which aren't. One
> way is to return a cons (recursive-dirs . non-recursive-dirs).

Those will be directories of different kinds, and the "non-recursive" 
(or, most-likely) "non-traversable" status would be reflected in their 
category name.

For instance, some directory might contain the key config files of the 
project; its category, then, will be "config-file-are-here".

I'm not sure it would be a good idea to have two directories belonging 
to the same set of categories, that the callers would have to treat 
differently WRT traversal.

> What is the use case for this?

For instance, in addition to the "project root" (let's imagine there's 
just one root), a Java project can have an src and test directories 
inside it. The backend could report that "src" belongs to the 
"source-roots" category, and "test" belongs to "test-roots" category.

Then, provided there is a stable test file naming convention, we could 
define commands that would allow the user to jump between the source 
file and its test, and perform other related operations.

> 1 is satisfied by project-all-roots (ie (append (project-roots prj)
> (project-library-roots prj))) and project-ignores. Allowing
> project-all-roots to return non-roots breaks this case.

Basically, the caller would have to pass the returned value through 
`project-combine-directory'. Alternatively, we can have a helper like this:

(defun project-directories-in-categories (project &rest categories)
   (project-combine-directories
    (cl-delete-if-not
     (lambda (dir)
       (cl-subsetp categories (project-categories project dir)))
     (project-directories project))))

Or, as yet another option, we can define `project-directories' to have 
this signature, and the project backend to implement it. But I'd prefer 
to have the helper.

> 2 is not supported directly. It is not simple to get the list of all
> directories from project-all-roots and project-ignores.

I'm not convinced it's actually needed. Like discussed previously, the 
capability encourages slower implementations. Instead, where possible, 
the API consumers should combine 'find' with other programs.

Even so, while it's "not simple", it perfectly possible to implement it 
on top of the API. So we can include it as a utility function, but 
there's no need to make it a method, for a backend to implement.

> For Emacs 25, we should keep this as simple as possible; provide two
> functions, one that returns all recursive roots and ignores, another
> that returns a non-recursive list of all directories. Backends can
> provide whichever is convenient; project.el can convert between them.

And I'm against that:

a) See above.

b) Having only these two functions won't make it possible to have both 
project-find-regexp and project-or-libraries-find-regexp (or whatever 
its name will be). I've explained why its beneficial to have both, and 
the distinction between them.

>> - If we're only documenting the "categories" key inside
>> project-metadata, why not have a project-directory-categories method
>> instead? We can add the -metadata method, too, now or later.
>
> This should wait for Emacs 26; we don't have a solid proposal.

I thought the idea is pretty simple. What would you like me to clarify?

Re: project.el semantics

[Dmitry Gutov]

On 11/22/2015 07:18 AM, John Wiegley wrote:

> I think Dmitry and I talked about a more general way of yielding metadata
> associated with project-related "entities" (mainly directories and files in
> this version), and that this metadata could indicate recursion, or even how to
> perform the recursion. For example, it could be recursion with a file name, or
> i-node testing, predicate.

Indeed, if we have the metadata for directories, we can have a key 
indicating the recursive-ness of traversal.

I don't particularly like that option, because it allows the consumers 
to target only specific kinds of project, and only handle one case (the 
recursive, or the non-recursive one).

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> On 11/22/2015 07:18 AM, John Wiegley wrote:
>> I think Dmitry and I talked about a more general way of yielding metadata
>> associated with project-related "entities" (mainly directories and files in
>> this version), and that this metadata could indicate recursion, or even how
>> to perform the recursion. For example, it could be recursion with a file
>> name, or i-node testing, predicate.

> I don't particularly like that option, because it allows the consumers to
> target only specific kinds of project, and only handle one case (the
> recursive, or the non-recursive one).

Could you clarify? I thought the metadata approach give us a freedom that
implied the exact opposite, compared to a recursive vs. non-recursive only
vector.

John

Re: project.el semantics

[Dmitry Gutov]

On 11/22/2015 07:43 AM, John Wiegley wrote:

> Could you clarify? I thought the metadata approach give us a freedom that
> implied the exact opposite, compared to a recursive vs. non-recursive only
> vector.

A sloppy consumer of the API can disregard the "recursiveness" metadata 
key, and traverse the directories in only one of the possible ways. For 
instance, Stephen is used to non-recursive traversal of directories, in 
Ada projects. So he could write commands that do not support recursive 
traversal (maybe left as a TODO), but would still work fine with his Ada 
project backend. As a result, their universal applicability would be 
limited.

Or, vice versa, some other third-party package can only expect the 
"recursive" kind of directories, and traverse them that way. That would 
work fine on most projects out there, but would lead to undesirable 
effects if someone tried to use it with the Ada backend.

IME, freedom rarely implies correctness.

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> Or, vice versa, some other third-party package can only expect the
> "recursive" kind of directories, and traverse them that way. That would work
> fine on most projects out there, but would lead to undesirable effects if
> someone tried to use it with the Ada backend.

I'm still not sure why this means we need to encode restrictions within
project.el, rather than, say, in Ada mode, or whichever other mode is
providing information to project.el for the traversal of its related elements.

> IME, freedom rarely implies correctness.

I want as much freedom of expression as we can get away with, so long as the
default and expected use cases remain easily within reach.

I feel, though, as if I've lost my grip on the vision for this project again.
As I understand it, it has two main components:

    1. Ways to determine members of a project.
    2. Ways to usefully apply this information.

I think our discussion so far have been in the area of #1 only, right? And is
that because we're stuck on how to define membership?

As long as, at the level of project.el, membership can optionally be a list of
"collection functions", this would allow near infinite expressiveness while
not detracting from the standard use cases of recursive and non-recursive
directory traversals.

I don't think we gain anything by "baking in" what membership means, and
disallowing flexible definitions. Such a road leads us to continuing to have
to extend and hack project.el's interface, as users run into the limits of
what it can express. Which Stefan already has done with ada-mode. We shouldn't
even be discussing ada-mode! There is a simple design path that can allow
Stefan to define his project contents however he wants. Such questions should
not even be an issue at this level of the API.

John

Re: project.el semantics

[Dmitry Gutov]

On 11/22/2015 06:55 PM, John Wiegley wrote:

> I'm still not sure why this means we need to encode restrictions within
> project.el, rather than, say, in Ada mode, or whichever other mode is
> providing information to project.el for the traversal of its related elements.

I don't know what restrictions for Ada mode you have in mind. And how we 
might be able to mandate them, if not though project.el API.

> I want as much freedom of expression as we can get away with, so long as the
> default and expected use cases remain easily within reach.

Freedom has a cost. In particular, if directories can be recursive or 
not, *each* API consumer will have to pay the price of supporting both 
ways. Like I already said, xref-collect-matches will have to grow 
another code path. And every other similar function will have to do that 
too.

I have mentioned this multiple times already. Why don't we stop going in 
circles?

> I feel, though, as if I've lost my grip on the vision for this project again.

Maybe that's because you're only thinking in generals, and not examining 
the practical applications. Like the function mentioned above, and how 
it will have to change to accommodate your freedom.

> As I understand it, it has two main components:
>
>      1. Ways to determine members of a project.
>      2. Ways to usefully apply this information.
>
> I think our discussion so far have been in the area of #1 only, right? And is
> that because we're stuck on how to define membership?

 From where I'm standing, it's because a certain person keeps derailing 
the discussion into making the API closer to how a "list of directories" 
usually works in Ada.

And not because it's necessary for functionality (as we've discussed, 
recursive dirs + ignores can adequately describe a non-recursive set of 
dirs), but simply because it's convenient for a particular minority use 
case.

> As long as, at the level of project.el, membership can optionally be a list of
> "collection functions", this would allow near infinite expressiveness while
> not detracting from the standard use cases of recursive and non-recursive
> directory traversals.

That's too vague. How will your "collection functions" mesh with using 
"find", so that most operations can stay performant in large projects?

> I don't think we gain anything by "baking in" what membership means, and
> disallowing flexible definitions.

On the subject of what we gain, see above.

> Such a road leads us to continuing to have
> to extend and hack project.el's interface, as users run into the limits of
> what it can express.

Again: recursive dirs + ignores can adequately describe a non-recursive 
set of dirs.

> Which Stefan already has done with ada-mode. We shouldn't
> even be discussing ada-mode! There is a simple design path that can allow
> Stefan to define his project contents however he wants. Such questions should
> not even be an issue at this level of the API.

If Stephen wants to define his project contents however he wants, 
without regard for any consumers, that cannot fit in any stable API 
definition.

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> Freedom has a cost. In particular, if directories can be recursive or not,
> *each* API consumer will have to pay the price of supporting both ways. Like
> I already said, xref-collect-matches will have to grow another code path.
> And every other similar function will have to do that too.
>
> I have mentioned this multiple times already. Why don't we stop going in
> circles?

Hi Dmitry,

I thank you for your nearly endless patience while we dance this dance. I will
write up a description of higher-level semantics later today, against which we
can look at the various API options.

Our API will live in Emacs for potentially decades; it's OK with me if it
takes us a few months to reach a satisfying consensus.

For the time being -- aka, Emacs 25.1 -- I think we should keep project.el in
core, so that xref.el can make use of it, but that we not document it until we
all agree on the API we want to commit to. Perhaps the preamble to project.el
should even state that it is presently for internal use only. I don't want us
to feel rushed to get this right.

John

Re: project.el semantics

[John Wiegley]

>>>>> John Wiegley <jwiegley@gmail.com> writes:

> I thank you for your nearly endless patience while we dance this dance. I
> will write up a description of higher-level semantics later today, against
> which we can look at the various API options.

What is a "project"?

A project, semantically, is a set of sets: {{A, ...}, {B, ...}, ...}, where
each member set defines a "view" of the entities of the project. Example
views:

  - All files related to the project, in any way
  - All files containing source code (i.e., no generated content)
  - All files under version control
  - All generated files and directories
  - The arguments one would provide to GNU find to produce a similar list of
    files to any of the above
  - The directories and files relating to the project, with some indication of
    whether those directories should be recursed (and how)
  - etc., etc.

There are potentially infinitely many ways to view a project, depending on
what the users wishes to do. Examples of such uses include:

  - Build a TAGS file
  - Run a query-replace operation
  - Populate a dired buffer with the project file paths
  - Run a find-grep on the project's contents
  - Provide data to a higher-level package like Projectile
  - Delete all generated files
  - Build an ignore list for dired-omit-files
  - etc., etc.

The main question is, how do we determine this "set of sets", since the fully
generated set is potentially infinite?

To achieve this, we take the "project set" to be intensionally defined, and
provide an API for querying along the multiple views. The data returned from
these views should be generative (using stream.el), so that element-by-element
actions do not require excess allocation up front.

This API should be sufficient to define the set of utility functions that
operate in terms of it, and of taking whatever actions we wish in relation to
a "project".

Having described it this way, it seems there at least three sub-languages
involved: (1) A description language to define projects; (2) a query language
to define the desired view of a project; and (3) a selection language to
determine how the results of the query should be returned. In SQL terms, it
might look like:

    SELECT (selection) FROM (definition) WHERE (query);

For convenience, the DEFINITION can often be determined from context, such as
by querying Git or VC.

For convenience, the SELECTION can often be reduced to small set of options,
such as "all file paths" or "files and directory roots".

For convenience, the QUERY can be reduced to a set of possibilities, like
"source code", or "ignored files", or "library files", etc.

While the surface API might encode these conveniences as short-hands for what
are likely to be common uses, the underlying API should allow the general form
of a rich query. The project is, in a sense, a database of entities, and it's
hard to pre-determine all possible useful queries of such data.

How these conveniences are implemented is completely open; `cl-defgeneric'
methods that encode the QUERY&SELECTION against an auto-determined DEFINITION
is just fine, and is how project.el is written today.

What is presently missing is the lower-level query API, and the freedom to
specify the DEFINITION, QUERY and SELECTION in more flexible ways.

*That said*, I'd be willing to postone such an API until a future version,
since, from the user's perpective, it could be seen as an implementation
detail behind the convenience APIs.

So maybe this has come full circle, but if so, I hope it adds some clarity.

John

Re: project.el semantics

[Stephen Leake]

John Wiegley <jwiegley@gmail.com> writes:

>>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:
>
>> Or, vice versa, some other third-party package can only expect the
>> "recursive" kind of directories, and traverse them that way. That would work
>> fine on most projects out there, but would lead to undesirable effects if
>> someone tried to use it with the Ada backend.
>
> I'm still not sure why this means we need to encode restrictions within
> project.el, rather than, say, in Ada mode, or whichever other mode is
> providing information to project.el for the traversal of its related elements.
>
>> IME, freedom rarely implies correctness.
>
> I want as much freedom of expression as we can get away with, so long as the
> default and expected use cases remain easily within reach.
>
> I feel, though, as if I've lost my grip on the vision for this project
> again.

Me too.

> As I understand it, it has two main components:
>
>     1. Ways to determine members of a project.
>     2. Ways to usefully apply this information.

Yes.

> I think our discussion so far have been in the area of #1 only, right? And is
> that because we're stuck on how to define membership?

My impression is Dmitry is stuck on not accepting other valid use cases
as requirements.

> As long as, at the level of project.el, membership can optionally be a list of
> "collection functions", this would allow near infinite expressiveness while
> not detracting from the standard use cases of recursive and non-recursive
> directory traversals.

Are we still talking about emacs 25? this would be a big change from the
current API.

> I don't think we gain anything by "baking in" what membership means, and
> disallowing flexible definitions. Such a road leads us to continuing to have
> to extend and hack project.el's interface, as users run into the limits of
> what it can express. Which Stefan already has done with ada-mode. 

I think you've confused "Stefan" with "Stephen".

On the other hand, I don't know what part of ada-mode you are refering
to, so maybe you are talking about Stefan?

> We shouldn't even be discussing ada-mode! 

I only bring it up as an example of what a project can be used for, and
therefore what project.el should support.

> There is a simple design path that can allow Stefan to define his
> project contents however he wants. Such questions should not even be
> an issue at this level of the API.

I don't follow.

My approach is this: I should be able to rewrite the user interface of
the project-related portions of the current ada-mode on top of the
project.el API, while also adapting the current ada-mode code to provide
a project.el backend. If I can do that, then those UI functions will be
generally useful with other project backends (I can reuse them on Java
projects, etc).

This imposes requirements on the project.el API. I agree the
requirements should not be written in terms of ada-mode; ada-mode is
just one use case.

Looking at this another way; you seem to be saying "a project backend
can implement `project-roots' (or any other project function) to return
anything it wants". I don't think that works. That would mean the
project definition for `project-roots' would be:

(cl-defgeneric project-roots (project)
"Return some backend-defined information.")

How would client code, that is supposed to be backend agnostic, use this
function?

I don't understand what you mean by "define project contents however he
wants".

-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/23/2015 12:04 AM, Stephen Leake wrote:

> My impression is Dmitry is stuck on not accepting other valid use cases
> as requirements.

"I want to specify things in this way, but not that way" is not a new, 
or a disregarded, requirement, as long as both ways are equal in 
expressiveness, and I think we've established that they are.

It's not like I'm happy to make things harder for you, but simply 
insisting on your side without addressing my objections won't improve 
anything.

> Are we still talking about emacs 25? this would be a big change from the
> current API.

Apparently we've decided to make both xref and project.el experimental 
in Emacs 25. It will be an opportunity to think on them more and collect 
more feedback, without committing to the current APIs.

> My approach is this: I should be able to rewrite the user interface of
> the project-related portions of the current ada-mode on top of the
> project.el API, while also adapting the current ada-mode code to provide
> a project.el backend. If I can do that, then those UI functions will be
> generally useful with other project backends (I can reuse them on Java
> projects, etc).

+1.

You're unlikely to be able to rewrite *all* of ada-mode's 
project-related functions like that, but being able to do that for some 
of them will be a win already.

Re: project.el semantics

[Dmitry Gutov]

On 11/22/2015 11:27 PM, John Wiegley wrote:

>    - All files containing source code (i.e., no generated content)

And no txt/markdown/jpeg/object/etc files, right?

>    - All files under version control

That's unrelated to the project's configuration. And anyway, I don't 
think you can implement this in any way except by taking all files, and 
calling vc-responsible-backend on each of them, one by one.

>    - All generated files and directories
>    - The arguments one would provide to GNU find to produce a similar list of
>      files to any of the above
>    - The directories and files relating to the project, with some indication of
>      whether those directories should be recursed (and how)

Makes sense.

Also, the lists of files should really match the corresponding lists of 
directories in some way, wouldn't you say? Recursive traversal or not, 
as an API consumer I'd be surprised if "list of source code directories" 
tells me one thing, and "list of source code files" tells me something 
different (files from directories not mentioned in the former list, or 
missing files from the said directories, for non-obvious reasons).

So ideally, the way a project backend defines the information should be 
as concise and unambiguous as possible.

> To achieve this, we take the "project set" to be intensionally defined, and
> provide an API for querying along the multiple views. The data returned from
> these views should be generative (using stream.el), so that element-by-element
> actions do not require excess allocation up front.

It would help to understand the use cases where stream.el would help, first.

We should also evaluate the overhead that stream.el (or generator.el, 
which would be my preferred approach to this) will add to listing all 
files in a big project. No excess allocation up front is nice, but the 
operations we provide must be fast.

> For convenience, the DEFINITION can often be determined from context, such as
> by querying Git or VC.

We'll be reading project files for that. Or some third-party project 
backends will, at least.

> For convenience, the QUERY can be reduced to a set of possibilities, like
> "source code", or "ignored files", or "library files", etc.

A set of combined conditions, I'd say. "source code inside the project 
roots", "ignored files in directory D", "library files anywhere". 
(What's library files? We'll probably avoid having this category).

> While the surface API might encode these conveniences as short-hands for what
> are likely to be common uses, the underlying API should allow the general form
> of a rich query. The project is, in a sense, a database of entities, and it's
> hard to pre-determine all possible useful queries of such data.

We should define a set of basic, orthogonal axes of information a 
project can tell about it, and then add to it later. Preferably in areas 
orthogonal to the already defined ones.

> How these conveniences are implemented is completely open; `cl-defgeneric'
> methods that encode the QUERY&SELECTION against an auto-determined DEFINITION
> is just fine, and is how project.el is written today.

If project.el satisfies our basic QUERY&SELECTION needs, we could as 
well document it already, because the "more flexible ways" would just 
build on top of it, as a set of utility functions (right?).

On the other hand, I'm not sure we've reached that consensus.

Re: project.el semantics

[Richard Stallman]

[[[ To any NSA and FBI agents reading my email: please consider    ]]]
[[[ whether defending the US Constitution against all enemies,     ]]]
[[[ foreign or domestic, requires you to follow Snowden's example. ]]]

  > I think Dmitry and I talked about a more general way of yielding metadata
  > associated with project-related "entities" (mainly directories and files in
  > this version), and that this metadata could indicate recursion, or even how to
  > perform the recursion. For example, it could be recursion with a file name, or
  > i-node testing, predicate.

There is something to watch out for, here: extra conceptual levels can
make the system and/or its manual harder to understand.  This can be a
net loss, if the extra levels don't do a lot of good and if you can't
make them invisible to beginning users.

I don't know the details, so I won't venture to say that that WILL happen
in this case, but it is something to think about.

-- 
Dr Richard Stallman
President, Free Software Foundation (gnu.org, fsf.org)
Internet Hall-of-Famer (internethalloffame.org)
Skype: No way! See stallman.org/skype.html.

Re: project.el semantics

[Stephen Leake]

John Wiegley <jwiegley@gmail.com> writes:

> For convenience, the DEFINITION can often be determined from context, such as
> by querying Git or VC.
>
> For convenience, the SELECTION can often be reduced to small set of options,
> such as "all file paths" or "files and directory roots".
>
> For convenience, the QUERY can be reduced to a set of possibilities, like
> "source code", or "ignored files", or "library files", etc.
>
> While the surface API might encode these conveniences as short-hands for what
> are likely to be common uses, the underlying API should allow the general form
> of a rich query. The project is, in a sense, a database of entities, and it's
> hard to pre-determine all possible useful queries of such data.
>
> How these conveniences are implemented is completely open; `cl-defgeneric'
> methods that encode the QUERY&SELECTION against an auto-determined DEFINITION
> is just fine, and is how project.el is written today.
>
> What is presently missing is the lower-level query API, and the freedom to
> specify the DEFINITION, QUERY and SELECTION in more flexible ways.
>
> *That said*, I'd be willing to postone such an API until a future version,
> since, from the user's perpective, it could be seen as an implementation
> detail behind the convenience APIs.
>
> So maybe this has come full circle, but if so, I hope it adds some clarity.

Yes, I like this description.

We are left with what set of convenience functions to implement in 
Emacs 25 project.el.

We don't have a good process for determining that.

I think it would be helpful to go back to one of your original
suggestions, and collect a list of requirements for what we want
project.el to be able to do.

To that end, here is what the project features of ada-mode provide; they
are all candidates for what project.el could provide:

- Set `compilation-search-path' so `next-error' works correctly with
  compilation results. This is a non-recursive list of project
  directories, since the compiler default settings do not include the
  directory in the error messages.

  Some language compilers include the full directory in the error
  messages, so `compilation-search-path' is not needed at all; some
  include a relative path, so only the roots are needed.

- Set `ff-search-directories' and other `ff-' variables so
  `ff-find-the-other-file' works correctly, both for body/header files
  and for "with" (similar to C "include").

  For Ada, C, C++, `ff-search-directories' is a non-recursive list of
  project directories. For Java, it may be only the project roots (the
  core Emacs java-mode does not make this work for Java 'import').
  
- Set the xref backend so M-. finds multiple language definitions, and
  M-? finds multiple language references. (ada-mode does not yet use the
  actual xref.el, but it does the equivalent; the cross reference
  tool supports Ada, C, C++).

- Implement find-file-in-project to find any file included by the build
  tool project definition files, with completion on the file name (not
  the directories).

  The implementation uses `locate-file' with a non-recursive list of
  project directories, as returned by the build tool.

  For other project backends, this could be replaced by a call to an
  external 'find' with directory roots and ignores, or to 'global',
  which has it's own search path.

- Provide a menu of known projects, from which the user selects the
  currently active one.

- Invoke the build tool to build the main executable in the project.

  This assumes there is only one main. I normally don't use this
  feature; I use Makefiles directly since they are much more powerful.
  But others, particularly newbies, find this feature useful.


-- 
-- Stephe

Re: project.el semantics

[Dmitry Gutov]

On 11/23/2015 09:43 AM, Stephen Leake wrote:

> - Set the xref backend so M-. finds multiple language definitions, and
>    M-? finds multiple language references. (ada-mode does not yet use the
>    actual xref.el, but it does the equivalent; the cross reference
>    tool supports Ada, C, C++).

Does its API have any differences worth mentioning, that we might want 
to incorporate?

Re: project.el semantics

[Steinar Bang]

>>>>> Dmitry Gutov <dgutov@yandex.ru>:

> On 11/22/2015 11:27 PM, John Wiegley wrote:
>> - All files containing source code (i.e., no generated content)

> And no txt/markdown/jpeg/object/etc files, right?

(I'm not sure markdown and other document formats that generate code,
like e.g. DocBook XML or info, should be excluded (I'm not sure they
should be included either... but I'm not sure they should be
excluded...))

Re: project.el semantics

[Dmitry Gutov]

On 11/24/2015 12:04 AM, Steinar Bang wrote:

>> And no txt/markdown/jpeg/object/etc files, right?
>
> (I'm not sure markdown and other document formats that generate code,
> like e.g. DocBook XML or info, should be excluded (I'm not sure they
> should be included either... but I'm not sure they should be
> excluded...))

Yeah, we're conflating two notions here: programming languages sources 
files (which might be useful to run etags over, for instance), and 
generated (or non-generated) files.

files-you-generate-other-files-from might also be useful, but I can't 
think of a good application right now.

And generated files are usually included in the ignores list, so we 
won't special handling for them, most of the time.

Re: project.el semantics

[Dmitry Gutov]

On 11/22/2015 11:27 PM, John Wiegley wrote:

> So maybe this has come full circle, but if so, I hope it adds some clarity.

To get back to this question, not really, as far as I'm concerned. The 
concept sounds nice, but the proposal is low on specifics, and the 
changes that I try to vaguely imagine complicate the API considerably.

It would be nice to discuss particular example of commands that don't 
work well with the current API (nor with trivial extensions of it), but 
will work better with some specific changes.

Or, like, if we'll have a family of commands that's similar along a 
certain axis, that the upgraded API will simplify reuse.

  * * *

I've tried out the "categories" change that we've discussed, along with 
a couple of variations, but to be honest, I'm less convinced now than 
before. Using effectively an alist where we can use separate generic 
functions seems inconsistent, and treating all directories the same way 
might imply some guarantees that we won't want to give.

See the branches scratch/project-directories*.

scratch/project-directories-with-shallow illustrates what we'll have to 
do to have some directories of explicit "nonrecursive" kind. The result 
is less scarier than I expected (when calling find, just need to pass 
'-maxdepth 1' to it), but even so, the "shallow" argument threads 
through every related function and complicates both 
project-subtract-directories and project-combine-directories. Even 
though it should be 100% equivalent to just adding "./*/" to the ignores 
list for each shallow directory. So unless someone demonstrates that 
using 'find -maxdepth 1' instead of ignores makes a real difference in 
practice, I remain in opposition.

scratch/project-directories-filtering-in-backend seems to be a natural 
extension of scratch/project-directories, but really it seems that this 
way we'll be forcing the backends to implement a rather involved piece 
of logic, in unstructured fashion. And even so, the proposed signature 
only covers the intersections of categories; what if we want to get 
directories "in A, but not in B"? Better to return the raw data to the 
user, and let them use the combinator functions.

scratch/project-directories itself basically seems wasteful: we're 
forcing the backend to produce directories of all kinds at the same 
time, always. A step forward would be to change the arguments list from 
"&rest categories" to one required argument "category".

But then, we'll have to document some allowed categories anyway, and we 
can just as well define the corresponding generic functions instead, 
like project-source-directories and project-test-directories.

To sum up, barring new, advanced proposals from someone else (or more 
detailed proposal from John), I'd rather keep the current structure, 
with possibly following changes:

- Rename project-library-roots to project-external-roots, as a more 
neutral name. Document it as a list of directories related to the 
current project but lying outside of it.

- Like in scratch/project-directories, rename and move the 
project-library-roots-function variable to the project-vc- namespace, 
and for now document that it'll only affect the project-vc backend.

Re: project.el semantics

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> To sum up, barring new, advanced proposals from someone else (or more
> detailed proposal from John), I'd rather keep the current structure, with
> possibly following changes:

Hi Dimtry,

Having had more time to think about this, I may have been getting ahead of
myself. Let's start with the API you've already proposed, and allow
experimentation in the field to determine what users actually need. So, please
continue with what you think is best for the current version. I also like the
external-roots change.

-- 
John Wiegley                  GPG fingerprint = 4710 CF98 AF9B 327B B80F
http://newartisans.com                          60E1 46C4 BD1A 7AC1 4BA2

Re: project.el semantics

[Dmitry Gutov]

Hi John,

On 12/16/2015 08:52 AM, John Wiegley wrote:
> Let's start with the API you've already proposed, and allow
> experimentation in the field to determine what users actually need. So, please
> continue with what you think is best for the current version. I also like the
> external-roots change.

Changed. Renaming things yet-again is pretty embarrassing, but hopefully 
the updated and expanded documentation (docstrings and commentary) makes 
up for that.

Re: find-file-project

[Dmitry Gutov]

[-- Attachment #1: Type: text/plain, Size: 715 bytes --]

Hi Stefan,

On 09/18/2015 08:07 PM, Stefan Monnier wrote:
> Of course, project-find-file could wrap the backend's completion table
> so as to add its own `category' info to the meatadata, but
> it's cumbersome.

Getting back to the question of the new category, do you think 
`project-file' is a good enough name?

Could we want to reuse it for non-project, or non-file completions, in 
the future?

Here's the basic patch that I want to install, which should be enough to 
introduce the feature in 25.1 (to make project.el reasonably interesting 
at least for some fraction of users).

Using partial-completion here requires starting input with a slash, but 
that's still better than no partial completion at all.

[-- Attachment #2: project-find-file.diff --]
[-- Type: text/x-patch, Size: 5507 bytes --]

diff --git a/lisp/minibuffer.el b/lisp/minibuffer.el
index 1251a4e..f6bcbd9 100644
--- a/lisp/minibuffer.el
+++ b/lisp/minibuffer.el
@@ -831,7 +831,8 @@ completion-styles
 
 (defvar completion-category-defaults
   '((buffer (styles . (basic substring)))
-    (unicode-name (styles . (basic substring))))
+    (unicode-name (styles . (basic substring)))
+    (project-file (styles . (basic substring partial-completion))))
   "Default settings for specific completion categories.
 Each entry has the shape (CATEGORY . ALIST) where ALIST is
 an association list that can specify properties such as:
diff --git a/lisp/progmodes/project.el b/lisp/progmodes/project.el
index d771587..2e3222f 100644
--- a/lisp/progmodes/project.el
+++ b/lisp/progmodes/project.el
@@ -45,10 +45,12 @@
 
 ;;; TODO:
 
-;; * Commands `project-find-file' and `project-or-external-find-file'.
-;;   Currently blocked on adding a new completion style that would let
-;;   the user enter just the base file name (or a part of it), and get
-;;   it expanded to the absolute file name.
+;; * Reliably cache the list of files in the project, probably using
+;;   filenotify.el (if supported) to invalidate.  And avoiding caching
+;;   if it's not available (manual cache invalidation is not nice).
+;;
+;; * Allow the backend to override the file-listing logic?  Maybe also
+;;   to delegate file name completion to an external tool.
 ;;
 ;; * Build tool related functionality.  Start with a `project-build'
 ;;   command, which should provide completions on tasks to run, and
@@ -258,6 +260,7 @@ project--value-in-dir
 (declare-function xref-collect-matches "xref")
 (declare-function xref--show-xrefs "xref")
 (declare-function xref-backend-identifier-at-point "xref")
+(declare-function xref--find-ignores-arguments "xref")
 
 ;;;###autoload
 (defun project-find-regexp (regexp)
@@ -302,5 +305,42 @@ project--find-regexp-in
       (user-error "No matches for: %s" regexp))
     (xref--show-xrefs xrefs nil)))
 
+(defun project-find-file ()
+  (interactive)
+  (let* ((pr (project-current t))
+         (dirs (project-roots pr)))
+    (project--find-file-in dirs pr)))
+
+(defun project-or-external-find-file ()
+  (interactive)
+  (let* ((pr (project-current t))
+         (dirs (append
+                (project-roots pr)
+                (project-external-roots pr))))
+    (project--find-file-in dirs pr)))
+
+;; FIXME: Uniquely abbreviate the roots?
+(defun project--find-file-in (dirs project)
+  (let* ((all-files
+          (cl-mapcan
+           (lambda (dir)
+             (let ((command
+                    (format "%s %s %s -type f -print0"
+                            find-program
+                            dir
+                            (xref--find-ignores-arguments
+                             (project-ignores project dir)
+                             (expand-file-name dir)))))
+               (split-string (shell-command-to-string command) "\0" t)))
+           dirs))
+         (table (lambda (string pred action)
+                  (cond
+                   ((eq action 'metadata)
+                    '(metadata . ((category . project-file))))
+                   (t
+                    (complete-with-action action all-files string pred))))))
+    (find-file
+     (completing-read "Find file: " table nil t))))
+
 (provide 'project)
 ;;; project.el ends here
diff --git a/lisp/progmodes/xref.el b/lisp/progmodes/xref.el
index ae5ec61..5970011 100644
--- a/lisp/progmodes/xref.el
+++ b/lisp/progmodes/xref.el
@@ -876,7 +876,9 @@ xref-collect-matches
                                                        grep-find-template t t))
          (grep-highlight-matches nil)
          (command (xref--rgrep-command (xref--regexp-to-extended regexp)
-                                       files dir ignores))
+                                       files
+                                       (expand-file-name dir)
+                                       ignores))
          (orig-buffers (buffer-list))
          (buf (get-buffer-create " *xref-grep*"))
          (grep-re (caar grep-regexp-alist))
@@ -912,23 +914,28 @@ xref--rgrep-command
            " "
            (shell-quote-argument ")"))
    dir
-   (concat
-    (shell-quote-argument "(")
-    " -path "
-    (mapconcat
-     (lambda (ignore)
-       (when (string-match-p "/\\'" ignore)
-         (setq ignore (concat ignore "*")))
-       (if (string-match "\\`\\./" ignore)
-           (setq ignore (replace-match dir t t ignore))
-         (unless (string-prefix-p "*" ignore)
-           (setq ignore (concat "*/" ignore))))
-       (shell-quote-argument ignore))
-     ignores
-     " -o -path ")
-    " "
-    (shell-quote-argument ")")
-    " -prune -o ")))
+   (xref--find-ignores-arguments ignores dir)))
+
+(defun xref--find-ignores-arguments (ignores dir)
+  ;; `shell-quote-argument' quotes the tilde as well.
+  (cl-assert (not (string-match-p "\\`~" dir)))
+  (concat
+   (shell-quote-argument "(")
+   " -path "
+   (mapconcat
+    (lambda (ignore)
+      (when (string-match-p "/\\'" ignore)
+        (setq ignore (concat ignore "*")))
+      (if (string-match "\\`\\./" ignore)
+          (setq ignore (replace-match dir t t ignore))
+        (unless (string-prefix-p "*" ignore)
+          (setq ignore (concat "*/" ignore))))
+      (shell-quote-argument ignore))
+    ignores
+    " -o -path ")
+   " "
+   (shell-quote-argument ")")
+   " -prune -o "))
 
 (defun xref--regexp-to-extended (str)
   (replace-regexp-in-string

Re: find-file-project

[Stefan Monnier]

> Could we want to reuse it for non-project, or non-file completions, in
> the future?

I'm not sure I understand the question.  This category is only there so
people can tweak the completion style for this particular use of
completion, so I think `project-file' is OK.

Maybe what you're getting it at, is that maybe we should introduce
a notion of hierarchy within "categories".  I think we can do that later
if/when the need arise.

> +    (unicode-name (styles . (basic substring)))
> +    (project-file (styles . (basic substring partial-completion))))
                                                ^^^^^^^^^^^^^^^^^^
This should not be needed since it's already in completion-styles.


        Stefan

Re: find-file-project

[Stephen Leake]

[-- Attachment #1: Type: text/plain, Size: 1228 bytes --]

Dmitry Gutov <dgutov@yandex.ru> writes:

> Hi Stefan,
>
> On 09/18/2015 08:07 PM, Stefan Monnier wrote:
>> Of course, project-find-file could wrap the backend's completion table
>> so as to add its own `category' info to the meatadata, but
>> it's cumbersome.
>
> Getting back to the question of the new category, do you think
> `project-file' is a good enough name?
>
> Could we want to reuse it for non-project, or non-file completions, in
> the future?
>
> Here's the basic patch that I want to install, which should be enough
> to introduce the feature in 25.1 (to make project.el reasonably
> interesting at least for some fraction of users).

This does not address the issue of duplicate file names in the path,
which I think is important.

For example:

$ stephe@takver2$ stephe@takver2$ find /Projects/emacs/master/lisp/ -name "locate.el"
/Projects/emacs/master/lisp/cedet/ede/locate.el
/Projects/emacs/master/lisp/locate.el

Attached is my implementation of locate-uniquified-file, which addresses
this. That can be used to implement project-find-file.

It uses a "path iterator" object, to avoid the issue of recursive
path vs non-recursive path.

I have tests for these as well, if you are interested.

-- 
-- Stephe

[-- Attachment #2: uniquify-files.el --]
[-- Type: application/emacs-lisp, Size: 26469 bytes --]

[-- Attachment #3: path-iterator.el --]
[-- Type: application/emacs-lisp, Size: 7222 bytes --]

Re: find-file-project

[Dmitry Gutov]

On 01/07/2016 07:52 AM, Stefan Monnier wrote:

> I'm not sure I understand the question.  This category is only there so
> people can tweak the completion style for this particular use of
> completion, so I think `project-file' is OK.

I was asking if the category should be more coarse than this. But if you 
think this one's okay, great!

> Maybe what you're getting it at, is that maybe we should introduce
> a notion of hierarchy within "categories".  I think we can do that later
> if/when the need arise.

Yes, shouldn't be hard.

>> +    (unicode-name (styles . (basic substring)))
>> +    (project-file (styles . (basic substring partial-completion))))
>                                                  ^^^^^^^^^^^^^^^^^^
> This should not be needed since it's already in completion-styles.

Oh, that's a bit surprising: going by the docstrings (the 
`completion-styles' docstring, at least), the values in 
completion-category-defaults are supposed to "override" 
completion-styles (which I take to mean "replace"), not just get prepended.

In this case, either way's fine, but there's a discrepancy with the 
manual as well: the bottom of (info "(emacs) Completion Styles") says:

    You can use different completion styles in different situations, by
setting the variable ‘completion-category-overrides’.  For example, the
default setting says to use only ‘basic’ and ‘substring’ completion for
buffer names.

Note the word "only".

Re: find-file-project

[Dmitry Gutov]

Hi Stephen,

On 01/07/2016 10:12 AM, Stephen Leake wrote:

> This does not address the issue of duplicate file names in the path,
> which I think is important.
>
> For example:
>
> $ stephe@takver2$ stephe@takver2$ find /Projects/emacs/master/lisp/ -name "locate.el"
> /Projects/emacs/master/lisp/cedet/ede/locate.el
> /Projects/emacs/master/lisp/locate.el

It kind of does:

M-x project-find-file, type `ede/locate.el', press RET.

or, to open the other one, type `lisp/locate.el', RET.

Admittedly, if you've already typed `locate.el', and are trying to 
resolve the ambiguity, choosing the second option takes more keypresses 
than one might expect:

C-a lisp.el C-e RET

I'd call it a bug in completion-at-point, but it might be expected 
behavior, caused by the interaction between the completion styles.

Re: find-file-project

[Dmitry Gutov]

On 01/07/2016 10:12 AM, Stephen Leake wrote:

> Attached is my implementation of locate-uniquified-file, which addresses
> this. That can be used to implement project-find-file.
>
> It uses a "path iterator" object, to avoid the issue of recursive
> path vs non-recursive path.

Thanks, I'll have to play with it.

Does it handle ignores? How's the performance in the Emacs repository?

project-find-file currently takes 0.5s, with warm cache on my SSD, to 
generate the files list.

And apparently, a lot of it is due to the ignores: if I clear 
.gitignore, it shows more files, but the time is down to 0.2s.

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 01/07/2016 10:12 AM, Stephen Leake wrote:
>
>> Attached is my implementation of locate-uniquified-file, which addresses
>> this. That can be used to implement project-find-file.
>>
>> It uses a "path iterator" object, to avoid the issue of recursive
>> path vs non-recursive path.
>
> Thanks, I'll have to play with it.
>
> Does it handle ignores? 

Yes, in the path iterator object.

> How's the performance in the Emacs repository?

There is a noticeable delay, mostly due to computing the partial
completion (before I added that, there was no noticeable delay).

This is a pure elisp implementation. However, the path iterator object
could provide the directory list to the subprocess find function as
well.

-- 
-- Stephe

Re: find-file-project

[Dmitry Gutov]

On 01/07/2016 10:58 PM, Stephen Leake wrote:

>> Thanks, I'll have to play with it.
>>
>> Does it handle ignores?
>
> Yes, in the path iterator object.

Tried it, and it did what advertised after I added (find-file ...) to 
the locate-uniquified-file's definition. Has some issues (type `<emacs', 
press TAB, get "Internal error: xxx doesn't match yyy"), but it's not 
too important at this stage.

As written, though, it only lists files at the top-level of load-path, 
without recursing. So it's hard to compare it with project-find-file 
that uses the find program.

Note that it doesn't list lisp/cedet/ede/locate.el that you brought up 
as an example earlier (maybe because I don't have EDE loaded).

FWIW, I'm not partial to the suffix-style of uniquification.

>> How's the performance in the Emacs repository?
>
> There is a noticeable delay, mostly due to computing the partial
> completion (before I added that, there was no noticeable delay).

Did you try that using an iterator that visits the whole of Emacs repo, 
not just load-path? E.g., I can't visit xdisp.c this way.

> This is a pure elisp implementation. However, the path iterator object
> could provide the directory list to the subprocess find function as
> well.

Will Emacs then call `find' for each of the "shallow" directories in the 
list? That seems inefficient. Feel free to prove me wrong here, of course.

I've been thinking about how we could deletage file-listing (as well as 
completion, even) to an external tool, so that Emacs can manage to 
navigate huge projects.

It could be a generic method which returns a value similar to EDE's 
locate-obj (to support a set of operations we'd dispatch: list files, 
list directories (both with completion), search through files...).

But as a first draft, let's imagine having it simply return a list of 
files in a directory. Certain backends could implement that to traverse 
some of their "roots" non-recursively. But I think this will only work 
okay if the list of roots is not too long (that is, converting recursive 
dirs into non-recursive, anywhere, doesn't fit this approach).

Re: find-file-project

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> Could we want to reuse it for non-project, or non-file completions, in the
> future?

What sort of things were you thinking of, Dmitry? We do already have a few
completion frameworks in core.

-- 
John Wiegley                  GPG fingerprint = 4710 CF98 AF9B 327B B80F
http://newartisans.com                          60E1 46C4 BD1A 7AC1 4BA2

Re: find-file-project

[Dmitry Gutov]

On 01/08/2016 04:26 AM, John Wiegley wrote:

> What sort of things were you thinking of, Dmitry?

Wasn't thinking of anything in particular. Some other completion for a 
file name from a flat list, that doesn't go through read-file-name?

> We do already have a few
> completion frameworks in core.

Frameworks? completion-styles "only" affects completion-all-completions 
and friends.

Re: find-file-project

[Richard Copley]

project.el looks great, it will be very useful. Thanks to all the
contributors.

It looks as though project-find-file depends on "xref.el" but doesn't
`require' it. From emacs -Q:

M-x load-lib RET project RET
M-x project-find-file RET
=> Symbol’s function definition is void: xref--find-ignores-arguments

Testing in emacs -Q in a project directory with unsafe directory locals,
I also noticed that I get re-prompted to accept the dir-locals when I
execute project-find-file. Like this:

emacs -Q /my/project/dir
=> *Local Variables* buffer and y-or-n-p prompt
n
M-x project-find-file RET
=> *Local Variables* buffer and y-or-n-p prompt

I'll file a bug report (or two) if you want.

Re: find-file-project

[Stefan Monnier]

> Wasn't thinking of anything in particular. Some other completion for a file
> name from a flat list, that doesn't go through read-file-name?

I'm not sure if you'd count filecache.el as "goes through
read-file-name", but it seems relevant.

As for "completion in outside tools", one "easyish" way would be to hook
right into completion--nth-completion.  IOW instead of providing
ad-hoc implementations for try-completion, all-completions,
completion-boundaries, etc... you'd provide ad-hoc implementations of
minibuffer-try-completion and minibuffer-all-completions, which have the
advantage of allowing more flexibility in its output (e.g. the input
doesn't need to be a prefix of the output).


        Stefan

Re: find-file-project

[Dmitry Gutov]

On 01/08/2016 05:19 AM, Richard Copley wrote:
> project.el looks great, it will be very useful. Thanks to all the
> contributors.

My pleasure.

> It looks as though project-find-file depends on "xref.el" but doesn't
> `require' it. From emacs -Q:
>...
> Testing in emacs -Q in a project directory with unsafe directory locals,
> I also noticed that I get re-prompted to accept the dir-locals when I
> execute project-find-file.

Thanks for the report, both should be fixed now.

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 01/07/2016 10:58 PM, Stephen Leake wrote:
>
>>> Thanks, I'll have to play with it.
>>>
>>> Does it handle ignores?
>>
>> Yes, in the path iterator object.
>
> Tried it, and it did what advertised after I added (find-file ...) to
> the locate-uniquified-file's definition. 

Hmm? the doc string of `locate-uniquified-file' says "return an
absolute-filename", not "visit a file".

Here's how I implement find-file-in-project:

(cl-defgeneric project-find-file (prj filename)
  "Find FILENAME with completion in project PRJ."
  (let* ((iter (make-path-iterator
		:user-path-non-recursive nil
		:user-path-recursive (project-all-roots prj)
		:ignore-function (lambda (dir) (project-ignore-dir prj dir))))
	 (pred (lambda (abs-file-name)
		 (not (string-match
		       ;; FIXME: this assumes `project-ignores' can accept dir that is not root
		       (project-ignores-regexp prj (file-name-directory abs-file-name))
		       abs-file-name))))
	 (absfilename (locate-uniquified-file-iter iter pred filename)))

    (if (file-readable-p absfilename)
	(find-file absfilename)
      ;; FIXME: need human-readable name for project
      (error "'%s' not found/readable in project." filename)))
  )

(defun find-file-in-project (filename)
  "Find FILENAME (default prompt) with completion in current project.
With prefix arg, FILENAME defaults to filename at point."
  (interactive (list (when current-prefix-arg (thing-at-point 'filename))))
  (project-find-file (project-current) filename))



> As written, though, it only lists files at the top-level of load-path,
> without recursing. 

Yes, that's consistent with `locate-file', and all other uses of
`load-path'.

To add all directories that contain elisp files, you need to build the
path iterator directly:

(let ((emacs-root
       (file-name-directory
	(directory-file-name
	 (file-name-directory (locate-library "man"))))))
  (make-path-iterator
   :user-path-non-recursive
   (append
    load-path
    (list
     (concat emacs-root "test/automated")
     (concat emacs-root "lisp/term")))
   :user-path-recursive
   (list (concat emacs-root "lisp/cedet")))
  )

> Note that it doesn't list lisp/cedet/ede/locate.el that you brought up
> as an example earlier (maybe because I don't have EDE loaded).

Because you did not add the "cedet/ede" directory to the path iterator.

As we have discussed before, the user must tell the code what
directories in `load-path' should be treated recursively.

> FWIW, I'm not partial to the suffix-style of uniquification.

Right; one of these days I'll see if I can convert that to prefix. But
the completion code makes the assumption that the prefix of the strings
match, so it would be a big change.

But once you get used to it, you'll find that you simply don't care
about the directory; you really only need it to disambiguate colliding
names. So the suffix style makes sense.

>>> How's the performance in the Emacs repository?
>>
>> There is a noticeable delay, mostly due to computing the partial
>> completion (before I added that, there was no noticeable delay).
>
> Did you try that using an iterator that visits the whole of Emacs
> repo, not just load-path? E.g., I can't visit xdisp.c this way.

Yes; I use a variant of the above path-iterator constructor that adds
the C directories. But that also requires the project code to know about
multiple languages, so it's part of a multi-language project package.

>> This is a pure elisp implementation. However, the path iterator object
>> could provide the directory list to the subprocess find function as
>> well.
>
> Will Emacs then call `find' for each of the "shallow" directories in
> the list? 

Only if you write the code that way. The path-iterator constructor takes
both recursive and non-recursive arguments.

'find' requires a single recursive root plus ignores, so to be really
useful with 'find', the path-iterator object would have to be enhanced
to return those values.

-- 
-- Stephe

Re: find-file-project

[Dmitry Gutov]

On 01/08/2016 10:11 PM, Stephen Leake wrote:

> Hmm? the doc string of `locate-uniquified-file' says "return an
> absolute-filename", not "visit a file".

Sorry, I got confused. I looked for an interactive function to showcase 
the logic, and there were only two of them. And only the first one 
actually worked interactively.

>> As written, though, it only lists files at the top-level of load-path,
>> without recursing.
>
> Yes, that's consistent with `locate-file', and all other uses of
> `load-path'.

I suppose so.

It's not how I'd imagine a `project-find-file' command to work.

> To add all directories that contain elisp files, you need to build the
> path iterator directly:

I would hesitate to ask that of every user.

> But once you get used to it, you'll find that you simply don't care
> about the directory; you really only need it to disambiguate colliding
> names. So the suffix style makes sense.

One drawback of your method is that I don't know what to type, in 
advance. The necessary entry might have duplicates (and thus, a 
disambiguating prefix/suffix), or it might not. So it's hard to navigate 
to a file based on muscle memory alone.

The current project-find-file completion is not ideal, but it already 
makes that less of an issue.

>>>> How's the performance in the Emacs repository?
>>>
>>> There is a noticeable delay, mostly due to computing the partial
>>> completion (before I added that, there was no noticeable delay).
>>
>> Did you try that using an iterator that visits the whole of Emacs
>> repo, not just load-path? E.g., I can't visit xdisp.c this way.
>
> Yes; I use a variant of the above path-iterator constructor that adds
> the C directories. But that also requires the project code to know about
> multiple languages, so it's part of a multi-language project package.

Do you have any performance numbers?

Re: find-file-project

[Stephen Leake]

Dmitry Gutov <dgutov@yandex.ru> writes:

> On 01/08/2016 10:11 PM, Stephen Leake wrote:
>
>> Hmm? the doc string of `locate-uniquified-file' says "return an
>> absolute-filename", not "visit a file".
>
> Sorry, I got confused. I looked for an interactive function to
> showcase the logic, and there were only two of them. And only the
> first one actually worked interactively.
>
>>> As written, though, it only lists files at the top-level of load-path,
>>> without recursing.
>>
>> Yes, that's consistent with `locate-file', and all other uses of
>> `load-path'.
>
> I suppose so.
>
> It's not how I'd imagine a `project-find-file' command to work.
>
>> To add all directories that contain elisp files, you need to build the
>> path iterator directly:
>
> I would hesitate to ask that of every user.

What is the alternative?

Only the user knows which directories in load-path should be treated
recursively.

I do have a special function for elisp:

(defun elisp-find-file (filename)
  ;; see also sal-project-find-file below
  "Prompt for an elisp file name, find it on load-path exteded with cedet/**."
  (interactive (list (when current-prefix-arg (thing-at-point 'filename))))
  (let* ((cedet (file-name-directory (locate-library "cedet")))
	 (iter (make-path-iterator
		:user-path-non-recursive load-path ;; contains cedet, but not cedet/*
		:user-path-recursive (list cedet))))
    (find-file (locate-uniquified-file-iter iter nil filename "elisp file: "))
    ))

That assumes only CEDET is handled recursively; if I add another project
that has recursive dirs, I'll have to add that to this function.

>> But once you get used to it, you'll find that you simply don't care
>> about the directory; you really only need it to disambiguate colliding
>> names. So the suffix style makes sense.
>
> One drawback of your method is that I don't know what to type, in
> advance. The necessary entry might have duplicates (and thus, a
> disambiguating prefix/suffix), or it might not. So it's hard to
> navigate to a file based on muscle memory alone.

I don't follow.

If "locate.el" has duplicates, but I don't know that, then I start by
typing "loc", and the completion shows both completions, with
disambiguating suffix.

If the first one is the one I want, I hit <ret>. otherwise I hit <right>
<ret>.

That's the same pattern needed to distinguish "filter.el" from
"filters.el", except the disambiguation in the first case happens to
include directories.

If we used prefix directories, there might be a problem; depends on the
details.

On the other hand, with your implementation, I'm required to type
"ede/locate.el" vs "locate.el" to find these two files. So I have a
problem; I don't want to have to memorize all the files in ede, srecode,
and semantic!

-- 
-- Stephe

Re: find-file-project

[Dmitry Gutov]

On 01/09/2016 03:18 PM, Stephen Leake wrote:

> What is the alternative?
>
> Only the user knows which directories in load-path should be treated
> recursively.

I don't see the harm in treating all of them recursively.

> That assumes only CEDET is handled recursively; if I add another project
> that has recursive dirs, I'll have to add that to this function.

This is unnecessarily complex, I think.

> If "locate.el" has duplicates, but I don't know that, then I start by
> typing "loc", and the completion shows both completions, with
> disambiguating suffix.

And here's the problem: the user is obliged to refer to your list of 
disambiguated entries, to pick the right one. It's harder to guess, in 
advance, what I can type. Which makes the process slower than it has to be.

> If the first one is the one I want, I hit <ret>. otherwise I hit <right>
> <ret>.

<right>? Are you using ido-mode? I was discussing how it looks with the 
default completion mechanism.

If you're using icomplete-mode, or ivy-mode, you can also M-x 
project-find-file now, type locate.el, and pick either of the two 
matches quickly, using C-./C-, or C-n/C-p (respectively) followed by RET.

> That's the same pattern needed to distinguish "filter.el" from
> "filters.el", except the disambiguation in the first case happens to
> include directories.

Right, and in any case, I can admit that for files with unique base 
names your approach yields fewer false positives than fuzzy matching 
across full relative names.

On the other hand, I find being able to see the relative names rather 
useful. E.g. I may remember the actual base file name incorrectly; 
looking at the directory the file with the given name is in, can help me 
avoid the wrong choice and recall the right name.

> If we used prefix directories, there might be a problem; depends on the
> details.

IME, this approach works fairly well.

> On the other hand, with your implementation, I'm required to type
> "ede/locate.el" vs "locate.el" to find these two files. So I have a
> problem; I don't want to have to memorize all the files in ede, srecode,
> and semantic!

One ends up memorizing directory names anyway, in the course of working 
on a project.

But no, you don't have to memorize them: you type 'locate.el', press TAB 
(or RET), and see the directory names in the completions list.

Re: find-file-project

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> Wasn't thinking of anything in particular. Some other completion for a file
> name from a flat list, that doesn't go through read-file-name?

I seem to be missing something. Can you describe exactly what it is you need,
so I can compare it with existing capabilities? pcomplete completes against
file lists, for example.

-- 
John Wiegley                  GPG fingerprint = 4710 CF98 AF9B 327B B80F
http://newartisans.com                          60E1 46C4 BD1A 7AC1 4BA2

Re: find-file-project

[Dmitry Gutov]

On 01/18/2016 12:23 AM, John Wiegley wrote:

> I seem to be missing something. Can you describe exactly what it is you need,
> so I can compare it with existing capabilities? pcomplete completes against
> file lists, for example.

I'm not sure what's left to explain.

See the variable completion-category-overrides.

Re: find-file-project

[John Wiegley]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> I'm not sure what's left to explain.
> See the variable completion-category-overrides.

I'm sorry, that communicated nothing to me. As I'm a bit too buried to go
figuring this on my own, can you please summarize?

-- 
John Wiegley                  GPG fingerprint = 4710 CF98 AF9B 327B B80F
http://newartisans.com                          60E1 46C4 BD1A 7AC1 4BA2

Re: find-file-project

[Dmitry Gutov]

On 01/18/2016 02:07 AM, John Wiegley wrote:

> I'm sorry, that communicated nothing to me. As I'm a bit too buried to go
> figuring this on my own, can you please summarize?

Maybe if you ask a specific question?

I don't really need anything anymore: Stefan responded that it's okay 
for a completion-category-overrides value to be per-command, so 
project-file is fine as one of the values.

Re: find-file-project

[Dmitry Gutov]

On 01/08/2016 10:39 AM, Stefan Monnier wrote:

> I'm not sure if you'd count filecache.el as "goes through
> read-file-name", but it seems relevant.

Indeed, it does seem relevant. But it doesn't define a completion table 
in the standard way, so the question is moot.

> As for "completion in outside tools", one "easyish" way would be to hook
> right into completion--nth-completion.

I mostly meant the possibility in the Project API. But yes, to make it 
really useful, completion code needs to interface with external tools 
better as well. It's also one of the blockers for deprecating 
company-mode's own backends.

> IOW instead of providing
> ad-hoc implementations for try-completion, all-completions,
> completion-boundaries, etc... you'd provide ad-hoc implementations of
> minibuffer-try-completion and minibuffer-all-completions, which have the
> advantage of allowing more flexibility in its output (e.g. the input
> doesn't need to be a prefix of the output).

I think you mean completion-try-completion and 
completion-all-completions. How will this happen? Will these functions 
become generics? Do we expect the external tools to honor 
completion-styles values anyway somehow?

It would be great if you could file a bug and describe your current 
views on this subject there, so it's not buried in this discussion.

Re: find-file-project

[Stefan Monnier]

>> I'm not sure if you'd count filecache.el as "goes through
>> read-file-name", but it seems relevant.
> Indeed, it does seem relevant.  But it doesn't define a completion table in
> the standard way, so the question is moot.

Ah, indeed, I have a local hack which turns it into a "standard
completion table" but I never pushed it (and I'm not sure it should be
either, because it's not very pretty: the behavior doesn't fit very well).

> It's also one of the blockers for deprecating company-mode's
> own backends.

[ Side note: I recently changed nxml-mode to provide a proper capf, so
  it now "works" with company-mode, but company mode usually kicks in
  too late in my opinion.  If you have time to spare, I'd appreciate if
  you could try to fine-tune it a bit.  ]

>> IOW instead of providing ad-hoc implementations for try-completion,
>> all-completions, completion-boundaries, etc... you'd provide ad-hoc
>> implementations of minibuffer-try-completion and
>> minibuffer-all-completions, which have the advantage of allowing more
>> flexibility in its output (e.g. the input doesn't need to be a prefix
>> of the output).
> I think you mean completion-try-completion and
> completion-all-completions.

Right.

> How will this happen? Will these functions become generics?

I was thinking of adding a new method to completion tables (beside the
existing try, all, test, boundaries, and metadata), but maybe making
them generics would work as well.

Actually, all the completion-table methods could be turned into
generics, but that requires figuring out on what to dispatch since
currently all completion tables are either functions, lists, arrays or
hash-tables and the "new table types" are sub-cases of function.

> It would be great if you could file a bug and describe your current views on
> this subject there, so it's not buried in this discussion.

Done,


        Stefan

Re: find-file-project

[Dmitry Gutov]

On 01/19/2016 05:20 PM, Stefan Monnier wrote:

> Ah, indeed, I have a local hack which turns it into a "standard
> completion table" but I never pushed it (and I'm not sure it should be
> either, because it's not very pretty: the behavior doesn't fit very well).

Standard completion doesn't fit? And simply iterating between the 
suggestions fits it better?

What's the main difference between the list provided by filecache.el and 
the list of files used by project-find-file?

>> It's also one of the blockers for deprecating company-mode's
>> own backends.
>
> [ Side note: I recently changed nxml-mode to provide a proper capf, so
>    it now "works" with company-mode, but company mode usually kicks in
>    too late in my opinion.  If you have time to spare, I'd appreciate if
>    you could try to fine-tune it a bit.  ]

Too late how?

- If you have to wait too long after typing a few chars, you can lower 
company-idle-delay.
- If you have to type too many chars for completion to kick in, lower 
company-minimum-prefix-length? Alternatively, a "native" company backend 
can return a cons value with a fake "length" to compare against the 
latter variable's value (or t, to mean that the check will succeed). 
I've recently accepted a patch adding a similar capability to 
company-capf, on github: 
https://github.com/company-mode/company-mode/pull/450.

If nxml-mode completion always uses a limited set of 
tags/attributes/values, maybe it could always return t as the "prefix 
length". (I don't really know if that's true or not.)

> I was thinking of adding a new method to completion tables (beside the
> existing try, all, test, boundaries, and metadata), but maybe making
> them generics would work as well.

It seems to me the functional completion tables could be complex enough 
already.

> Actually, all the completion-table methods could be turned into
> generics, but that requires figuring out on what to dispatch since
> currently all completion tables are either functions, lists, arrays or
> hash-tables and the "new table types" are sub-cases of function.

lists/arrays/hash-tables could be handled by the default methods. Maybe 
functions, too (or use a different method for them, with a specializer 
using functionp?). "New" completion tables could be conses tag+function, 
or cl-structs.

But that's a bit of a separate concern: since completion-try-completion 
and completion-all-completions are on a higher level, I think *they* 
could be generics, whereas the all-completions/etc could stay as they are.

>> It would be great if you could file a bug and describe your current views on
>> this subject there, so it's not buried in this discussion.
>
> Done,

Thank you. Although you've omitted the part about generics, so I'm 
unsure where to continue that part of the discussion.

Re: find-file-project

[Stefan Monnier]

>> Ah, indeed, I have a local hack which turns it into a "standard
>> completion table" but I never pushed it (and I'm not sure it should be
>> either, because it's not very pretty: the behavior doesn't fit very well).
> Standard completion doesn't fit?

Hmm... can't remember exactly what were the problems.  I think it had to
do with my desire to reproduce the exact same behavior as the old
filecache.el.

FWIW, see the current diff I have.

> What's the main difference between the list provided by filecache.el and the
> list of files used by project-find-file?

Probably nothing very significant.  But filecache.el uses a particular
completion scheme that I wanted to try and preserve.

> Too late how?

In that by the time (not sure if measured in seconds or in chars)
company kicks in, there's only a single completion left, and it's just
a couple extra chars.

> - If you have to type too many chars for completion to kick in, lower
> company-minimum-prefix-length?

Yes, I think that's what's going on.  E.g. after typing "<", I think we
can pretty much immediately pop up the completion menu.  Or at most
after one more char.

> Alternatively, a "native" company backend can
> return a cons value with a fake "length" to compare against the latter
> variable's value (or t, to mean that the check will succeed). I've recently
> accepted a patch adding a similar capability to company-capf, on github:
> https://github.com/company-mode/company-mode/pull/450.

Good, so we should be able to do it within nxml's capf.

> If nxml-mode completion always uses a limited set of tags/attributes/values,
> maybe it could always return t as the "prefix length". (I don't really know
> if that's true or not.)

Not sure in general (e.g. for attributes), but for tag names at least,
I think that's pretty much the case.

> It seems to me the functional completion tables could be complex
> enough already.

Not sure what you mean by "complex enough": do you mean "flexible enough
that we don't need a new mechanism", or "so complex that adding yet
another method would make them really unbearable"?

> But that's a bit of a separate concern: since completion-try-completion and
> completion-all-completions are on a higher level, I think *they* could be
> generics, whereas the all-completions/etc could stay as they are.

But the only argument they receive is the completion-table, so we need
them to be "dispatchable".
[ Side note: I've been toying with the idea of "callable objects", by
which I mean thingies which have slots and dispatchable types (like
cl-structs or eieio objects) but which can also be passed to funcall.
We could use them for the advice objects of nadvice.el, for the stream
objects of stream.el, and potentially here as well.  ]


        Stefan

Re: find-file-project

[Dmitry Gutov]

On 01/20/2016 04:32 AM, Stefan Monnier wrote:

> Hmm... can't remember exactly what were the problems.  I think it had to
> do with my desire to reproduce the exact same behavior as the old
> filecache.el.
>
> FWIW, see the current diff I have.

Were you going to attach it?

>> Alternatively, a "native" company backend can
>> return a cons value with a fake "length" to compare against the latter
>> variable's value (or t, to mean that the check will succeed). I've recently
>> accepted a patch adding a similar capability to company-capf, on github:
>> https://github.com/company-mode/company-mode/pull/450.
>
> Good, so we should be able to do it within nxml's capf.

Yup.

> Not sure in general (e.g. for attributes), but for tag names at least,
> I think that's pretty much the case.

Attribute values could be a problem, but why not in attribute names? Do 
we expect to work with freakish schemas, with thousands of possible 
attributes?

> Not sure what you mean by "complex enough": do you mean "flexible enough
> that we don't need a new mechanism", or "so complex that adding yet
> another method would make them really unbearable"?

The latter. =/

>> But that's a bit of a separate concern: since completion-try-completion and
>> completion-all-completions are on a higher level, I think *they* could be
>> generics, whereas the all-completions/etc could stay as they are.
>
> But the only argument they receive is the completion-table, so we need
> them to be "dispatchable".

They who? completion-try-completion and the other? The default method 
will handle lists/alists/hash-tables and functions. The specialized 
methods will handle "dispatchable" types.

> [ Side note: I've been toying with the idea of "callable objects", by
> which I mean thingies which have slots and dispatchable types (like
> cl-structs or eieio objects) but which can also be passed to funcall.
> We could use them for the advice objects of nadvice.el, for the stream
> objects of stream.el, and potentially here as well.  ]

Like a closure, but with named fields as its environment? I can see how 
it could be handy for debugging, but not how it would help with the 
issue at hand.

Re: find-file-project

[Stefan Monnier]

> Were you going to attach it?

I was, yes.

>> Not sure in general (e.g. for attributes), but for tag names at least,
>> I think that's pretty much the case.
> Attribute values could be a problem,

Haven't thought much about them, but I don't think so: they tend to
either have few variants or offer no completion at all (allow
pretty much anything).

> but why not in attribute names?

Yes, for attribute names that's pretty much the case as well I think.

> Do we expect to work with freakish schemas, with thousands of
> possible attributes?

Sounds unlikely.

>>> But that's a bit of a separate concern: since completion-try-completion and
>>> completion-all-completions are on a higher level, I think *they* could be
>>> generics, whereas the all-completions/etc could stay as they are.
>> But the only argument they receive is the completion-table, so we need
>> them to be "dispatchable".
> They who? completion-try-completion and the other?

Yes.

> The default method will handle lists/alists/hash-tables and
> functions.  The specialized methods will handle "dispatchable" types.

Right, but that still requires the a new "dispatchable" kind of
completion-table.

>> [ Side note: I've been toying with the idea of "callable objects", by
>> which I mean thingies which have slots and dispatchable types (like
>> cl-structs or eieio objects) but which can also be passed to funcall.
>> We could use them for the advice objects of nadvice.el, for the stream
>> objects of stream.el, and potentially here as well.  ]
> Like a closure, but with named fields as its environment? I can see how it
> could be handy for debugging, but not how it would help with the issue
> at hand.

That would allow us to keep using functions (rather than add a new kind
of completion-table), and simply give them a dispatchable type when we
need it.


        Stefan


diff --git a/lisp/filecache.el b/lisp/filecache.el
index e754190..56b7f43 100644
--- a/lisp/filecache.el
+++ b/lisp/filecache.el
@@ -1,4 +1,4 @@
-;;; filecache.el --- find files using a pre-loaded cache
+;;; filecache.el --- Find files using a pre-loaded cache  -*- lexical-binding: t -*-
 
 ;; Copyright (C) 1996, 2000-2016 Free Software Foundation, Inc.
 
@@ -499,7 +499,7 @@ If called interactively, read the directory names one by one."
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Returns the name of a directory for a file in the cache
-(defun file-cache-directory-name  (file)
+(defun file-cache-directory-name (file)
   (let* ((directory-list (cdr (assoc-string
 			       file file-cache-alist
 			       file-cache-ignore-case)))
@@ -517,8 +517,11 @@ If called interactively, read the directory names one by one."
       (error "Filecache: no directory found for key %s" file))
      ;; Multiple elements
      (t
+      ;; FIXME: the use of minibuffer-contents here means that
+      ;; filecache can only be used in the minibuffer :-(
       (let* ((minibuffer-dir (file-name-directory (minibuffer-contents)))
-	     (dir-list       (member minibuffer-dir directory-list)))
+	     (dir-list       (member (expand-file-name minibuffer-dir)
+                                     directory-list)))
 	(setq directory
 	      ;; If the directory is in the list, return the next element
 	      ;; Otherwise, return the first element
@@ -533,9 +536,9 @@ If called interactively, read the directory names one by one."
     directory))
 
 ;; Returns the name of a file in the cache
-(defun file-cache-file-name  (file)
+(defun file-cache-file-name (file)
   (let ((directory (file-cache-directory-name file)))
-    (concat directory file)))
+    (abbreviate-file-name (concat directory file))))
 
 ;; Return a canonical directory for comparison purposes.
 ;; Such a directory ends with a forward slash.
@@ -557,78 +560,151 @@ If called interactively, read the directory names one by one."
 ;;
 ;; The default is to do the former; a prefix arg forces the latter.
 
+(defun file-cache-minibuffer-message (msg)
+  ;; Can't output a minibuffer-message naively from the
+  ;; completion-table because the completion hasn't been performed
+  ;; yet, so the sit-for would do the wrong thing.
+  ;; (minibuffer-message file-cache-multiple-directory-message)
+  (let ((buf (current-buffer))
+        (ol (if (minibufferp (current-buffer))
+                (make-overlay (point-max) (point-max)
+                              nil t t)))
+        (timer ())
+        (fun ()))
+    (if (null ol)
+        (message msg)
+      (unless (zerop (length msg))
+        ;; The current C cursor code doesn't know to use the overlay's
+        ;; marker's stickiness to figure out whether to place the cursor
+        ;; before or after the string, so let's spoon-feed it the pos.
+        (setq msg (copy-sequence msg))
+        (put-text-property 0 1 'cursor t msg))
+      (overlay-put ol 'after-string msg))
+    (setq fun (lambda ()
+                (with-current-buffer buf
+                  (if (overlay-buffer ol)
+                      (delete-overlay ol)
+                    (message nil))
+                  (when timer (cancel-timer timer) (setq timer nil))
+                  (remove-hook 'pre-command-hook fun 'local))))
+    (add-hook 'pre-command-hook fun nil 'local)
+    (when minibuffer-message-timeout
+      (setq timer (run-with-timer minibuffer-message-timeout nil fun)))))
+
+(defun file-cache-completion-table (minibuffer-contents pred action)
+  (let* ((completion-ignore-case file-cache-completion-ignore-case)
+         (case-fold-search       file-cache-case-fold-search)
+         (string                 (file-name-nondirectory minibuffer-contents))
+         ;; Ignore completion-regexp-list since it applies to the complete
+         ;; filenames, where here we're mostly just handling the
+         ;; nondirectory parts.
+         (completion-regexp-list nil)
+         ;; First look at the nondirectory part.
+         (completion-string (try-completion string file-cache-alist))
+         (dirs (assoc-string (if (stringp completion-string)
+                                 completion-string string)
+                             file-cache-alist file-cache-ignore-case)))
+    (cond
+     ;; If it's an exact match, complete on the directories by cycling.
+     ((or current-prefix-arg (eq completion-string t)
+          (and (equal string completion-string) dirs
+               ;; FIXME: This use of this/last-command to decide
+               ;; whether to start cycling or not is an ugly
+               ;; hack.  Previous code used a global
+               ;; `file-cache-last-completion' var, but that
+               ;; doesn't work now that we're in a completion
+               ;; table that can be called several times
+               ;; for a single completion command.
+               (setq this-command 'file-cache-complete-but-no-unique)
+               (eq last-command this-command))
+          ;; Also start cycling right away if there's only one
+          ;; completion for the filename part.
+
+          ;; FIXME: this has one bug, which was already present in the
+          ;; old code, in that if the current file is already in the
+          ;; first dir, we skip straight to the second.
+          ;; Then again, maybe this is a feature, tho, since the user
+          ;; could have used normal completion if he wanted the file
+          ;; in the current dir.
+          (and completion-string
+               (eq t (try-completion completion-string file-cache-alist))))
+      (if (eq completion-string t) (setq completion-string string))
+      (let ((file-cache-string (file-cache-file-name completion-string)))
+        (cond
+         ;; FIXME: to cycle, we have to behave in a non-standard way,
+         ;; e.g. the list of completions returned for all-completions
+         ;; will mostly not match the given "prefix".
+         ;; Instead, we should have a way for the completion table to
+         ;; say "use cycling now" or "this completion table is not
+         ;; prefix-based".  This will imply things like "don't use
+         ;; partial matching".
+         ;; Return the next directory.
+         ((eq action nil)
+          (cond
+           ((string= file-cache-string minibuffer-contents) t)
+           (current-prefix-arg
+            ;; By returning the same string, we hopefully cause
+            ;; minibuffer-complete to call minibuffer-completion-help.
+            ;; But subsequent completions will then try to scroll that
+            ;; window unless we change this-command.
+            (setq this-command 'file-cache-completion-help)
+            ;; To make sure we show completion-help even if
+            ;; completion-auto-help is `lazy', we also set
+            ;; last-command.
+            (setq last-command 'file-cache-completion-help)
+            minibuffer-contents)
+           (t
+            (when file-cache-multiple-directory-message
+              (file-cache-minibuffer-message
+               file-cache-multiple-directory-message))
+            file-cache-string)))
+         (t
+          ;; FIXME: if action is t (i.e. all-completions), we
+          ;; return a list of completions which don't match the
+          ;; prefix.  This is necessary for the completion-help to display
+          ;; the actual list of possible directories, but it also has
+          ;; some undesirable side-effects.  E.g. completion-help will
+          ;; tend to assume that the returned completions match the
+          ;; prefix and will blindly highlight the "following" char.
+          (complete-with-action
+           action
+           (mapcar (lambda (d) (abbreviate-file-name
+                           (concat d completion-string)))
+                   (cdr dirs))
+           (if (or (not (memq action '(t)))
+                   (string= file-cache-string minibuffer-contents))
+               minibuffer-contents "")
+           pred)))))
+
+     ;; We don't want to cycle, instead do normal completion on the
+     ;; filename part.  Here partial-completion and friends should
+     ;; work just fine.  We could even make `initials' completion
+     ;; working there.
+     (t
+      (completion-table-with-context
+       (or (file-name-directory minibuffer-contents) "")
+       ;; Ignore the predicate here since this is only an intermediate
+       ;; state where we complete file names that will usually not be yet
+       ;; in the right directory.
+       file-cache-alist string nil action)))))
+
 ;;;###autoload
-(defun file-cache-minibuffer-complete (arg)
+(defun file-cache-minibuffer-complete (_arg)
   "Complete a filename in the minibuffer using a preloaded cache.
 Filecache does two kinds of substitution: it completes on names in
 the cache, and, once it has found a unique name, it cycles through
-the directories that the name is available in.  With a prefix argument,
-the name is considered already unique; only the second substitution
-\(directories) is done."
+the directories that the name is available in."
   (interactive "P")
-  (let*
-      (
-       (completion-ignore-case file-cache-completion-ignore-case)
-       (case-fold-search       file-cache-case-fold-search)
-       (string                 (file-name-nondirectory (minibuffer-contents)))
-       (completion-string      (try-completion string file-cache-alist))
-       (completion-list)
-       (len)
-       (file-cache-string))
-    (cond
-     ;; If it's the only match, replace the original contents
-     ((or arg (eq completion-string t))
-      (setq file-cache-string (file-cache-file-name string))
-      (if (string= file-cache-string (minibuffer-contents))
-	  (minibuffer-message file-cache-sole-match-message)
-	(delete-minibuffer-contents)
-	(insert file-cache-string)
-	(if file-cache-multiple-directory-message
-	    (minibuffer-message file-cache-multiple-directory-message))))
-
-     ;; If it's the longest match, insert it
-     ((stringp completion-string)
-      ;; If we've already inserted a unique string, see if the user
-      ;; wants to use that one
-      (if (and (string= string completion-string)
-	       (assoc-string string file-cache-alist
-			     file-cache-ignore-case))
-	  (if (and (eq last-command this-command)
-		   (string= file-cache-last-completion completion-string))
-	      (progn
-		(delete-minibuffer-contents)
-		(insert (file-cache-file-name completion-string))
-		(setq file-cache-last-completion nil))
-	    (minibuffer-message file-cache-non-unique-message)
-	    (setq file-cache-last-completion string))
-	(setq file-cache-last-completion string)
-	(setq completion-list (all-completions string file-cache-alist)
-	      len             (length completion-list))
-	(if (> len 1)
-	    (progn
-	      (goto-char (point-max))
-	      (insert
-	       (substring completion-string (length string)))
-	      ;; Add our own setup function to the Completions Buffer
-	      (let ((completion-setup-hook
-                     (append completion-setup-hook
-                             (list 'file-cache-completion-setup-function))))
-		(with-output-to-temp-buffer file-cache-completions-buffer
-		  (display-completion-list
-                   (completion-hilit-commonality completion-list
-                                                 (length string))))))
-	  (setq file-cache-string (file-cache-file-name completion-string))
-	  (if (string= file-cache-string (minibuffer-contents))
-	      (minibuffer-message file-cache-sole-match-message)
-	    (delete-minibuffer-contents)
-	    (insert file-cache-string)
-	    (if file-cache-multiple-directory-message
-		(minibuffer-message file-cache-multiple-directory-message)))
-	  )))
-
-     ;; No match
-     ((eq completion-string nil)
-      (minibuffer-message file-cache-no-match-message)))))
+  (let ((minibuffer-completion-table 'file-cache-completion-table)
+        ;; When cycling, partial completion doesn't work at all.
+        (completion-styles (if (eq 'partial-completion (car completion-styles))
+                               (cons 'basic completion-styles)
+                             completion-styles))
+        (completion-setup-hook
+         (append completion-setup-hook
+                 (list 'file-cache-completion-setup-function))))
+    ;; FIXME: Use completion-in-region?
+    (minibuffer-complete)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Completion functions
@@ -636,7 +712,14 @@ the name is considered already unique; only the second substitution
 
 (defun file-cache-completion-setup-function  ()
   (with-current-buffer standard-output ;; i.e. file-cache-completions-buffer
-    (use-local-map file-cache-completions-keymap)))
+    (if (save-excursion
+          (goto-char (point-min))
+          (next-completion 1)
+          (file-name-absolute-p
+           (buffer-substring (point) (line-end-position))))
+        ;; FIXME: we could strip the bogus highlighting here, actually.
+        nil
+      (use-local-map file-cache-completions-keymap))))
 
 (defun file-cache-choose-completion (&optional event)
   "Choose a completion in the `*Completions*' buffer."

Re: find-file-project

[Dmitry Gutov]

On 01/20/2016 05:25 AM, Stefan Monnier wrote:
>> Were you going to attach it?
>
> I was, yes.

Thank you.

>> The default method will handle lists/alists/hash-tables and
>> functions.  The specialized methods will handle "dispatchable" types.
>
> Right, but that still requires the a new "dispatchable" kind of
> completion-table.

But why do we care about that? Each backend author could invent their 
own "dispatchable" completion table (with cl-defstruct), or just use

(cons
  'my-special-table
  (lambda (string pred action)
    ...))

> That would allow us to keep using functions (rather than add a new kind
> of completion-table), and simply give them a dispatchable type when we
> need it.

Ah, so they'll be kind of named closure subtypes.

I'm still not sure how I feel about dispatching on such type. E.g. with 
cl-defmethod it would be like adding a new "method" to [the new special 
kind of] function.

Do we really need for the new kind of table to be callable? Because if 
we just use the (cons ...) value above for this purpose, we can be sure 
that nobody will call all-completions on it by mistake (or they'll get a 
very weird result, at least).

Re: find-file-project

[Stefan Monnier]

> But why do we care about that? Each backend author could invent their own
> "dispatchable" completion table (with cl-defstruct),

That might break existing code which looks at minibuffer-completion-table.

> or just use
> (cons
>  'my-special-table
>  (lambda (string pred action)
>    ...))

(my-special-table toto) is a valid completion table.

> Ah, so they'll be kind of named closure subtypes.

Yes.  They'd be sub-types of `function'.

I kind of like the idea of adding such a functionality because it would
come in handy in stream.el and nadvice.el (could improve performance of
stream.el, and would clean up the hackish part of nadvice).

This use for completion-tables is not a good justification to introduce
the feature, OTOH.  We could use this feature there if it's available, tho.


        Stefan

Re: find-file-project

[Dmitry Gutov]

On 01/21/2016 12:58 AM, Stefan Monnier wrote:
>> But why do we care about that? Each backend author could invent their own
>> "dispatchable" completion table (with cl-defstruct),
>
> That might break existing code which looks at minibuffer-completion-table.

Wouldn't that breakage be justifiable? It's a new kind of table, and we 
wouldn't want it to be used in a way it's not designed to.

We can just say, no, all-completions doesn't apply to it (it may only 
apply to the value returned by completion-all-completions).

> (my-special-table toto) is a valid completion table.

Ok. Still, the output of operations on it will be obviously wrong, which 
would force the consumers to update their code.

Failing earlier would, of course, be preferable.

RE: find-file-project

[Drew Adams]

> >> But why do we care about that? Each backend author could invent their own
> >> "dispatchable" completion table (with cl-defstruct),
> >
> > That might break existing code which looks at minibuffer-completion-table.
> 
> Wouldn't that breakage be justifiable? It's a new kind of table, and we
> wouldn't want it to be used in a way it's not designed to.

Please do not break `minibuffer-completion-table' or its use.
I use it, including modifying it on the fly, during completion.

Re: find-file-project

[Dmitry Gutov]

On 01/21/2016 01:17 AM, Drew Adams wrote:

> Please do not break `minibuffer-completion-table' or its use.
> I use it, including modifying it on the fly, during completion.

Would it be that hard for you to update your code? It's in wiki.

RE: find-file-project

[Drew Adams]

> > Please do not break `minibuffer-completion-table' or its use.
> > I use it, including modifying it on the fly, during completion.
> 
> Would it be that hard for you to update your code? It's in wiki.

Yes, no doubt.  And why should I need to?

`minibuffer-completion-table' is documented for users, in the Elisp
manual (node Completion Commands), along with the other important
minibuffer variables - with which I hope you will also not mess.
And it has been thus since Day One (e.g., Emacs 18).

Why do you think you need to screw with this longstanding feature?
Would it be that hard for you to update _your_ code - to fit Emacs?

Re: find-file-project

[Dmitry Gutov]

On 01/21/2016 01:48 AM, Drew Adams wrote:

> Why do you think you need to screw with this longstanding feature?
> Would it be that hard for you to update _your_ code - to fit Emacs?

Not possible. all-completions is, like you say, documented to behave in 
a certain way.

We want to introduce completion tables that perform a more lax kind of 
completion. It would make sense if all-completions didn't work on them.

RE: find-file-project

[Drew Adams]

> > Why do you think you need to screw with this longstanding feature?
> > Would it be that hard for you to update _your_ code - to fit Emacs?
> 
> Not possible. all-completions is, like you say, documented to behave in
> a certain way.
> 
> We want to introduce completion tables that perform a more lax kind of
> completion. It would make sense if all-completions didn't work on them.

It doesn't bother me (a priori) if you add something, without
breaking anything.  Subtracting functionality is a different story.
Including `all-completions', `minibuffer-completion-table', etc.

I don't understand why you can't use lax completion in connection
with the existing framework (including `all-completions',
`minibuffer-completion-table', etc.).

I (and others) have been doing that for years, including for many
different kinds of lax completion.  What is so special about what
you think you need?

You no doubt have your reasons for doing things your own way.
Why not do that in your own library (compatible or incompatible)?

Re: find-file-project

[Dmitry Gutov]

On 01/21/2016 02:09 AM, Drew Adams wrote:

> I don't understand why you can't use lax completion in connection
> with the existing framework (including `all-completions',
> `minibuffer-completion-table', etc.).

all-completions is documented to use the "prefix" matching, not any 
other kind. Doing it differently would break some other code that relies 
on that.

> I (and others) have been doing that for years, including for many
> different kinds of lax completion.  What is so special about what
> you think you need?

I don't know what you've been doing, and how.

> You no doubt have your reasons for doing things your own way.
> Why not do that in your own library (compatible or incompatible)?

Stefan wanted company-mode to use completion-at-point-functions, and the 
completion tables they return. They're not sufficiently powerful, yet.

RE: find-file-project

[Drew Adams]

> > I don't understand why you can't use lax completion in connection
> > with the existing framework (including `all-completions',
> > `minibuffer-completion-table', etc.).
> 
> all-completions is documented to use the "prefix" matching, not any
> other kind. Doing it differently would break some other code that relies
> on that.

I use `all-completions' and nevertheless am able to do regexp matching,
substring matching, and several different kinds of fuzzy matching.

Maybe you meant `try-completion'?  And even that uses `completion-styles',
which is pretty flexible (not in the way that I wanted, but at least it
allows for different kinds of completion).

> > I (and others) have been doing that for years, including for many
> > different kinds of lax completion.  What is so special about what
> > you think you need?
> 
> I don't know what you've been doing, and how.

I don't know what your problem is with using the existing framework
to be able to do "lax completion".

[I've written here many times about what I've been doing, and how.
And the code is freely available, for anyone who is interested.
But the point is not what I have been doing (unless you are really
interested).]

The point is that you shouldn't need to sacrifice `all-completions'
or `minibuffer-completion-table' just to be able to get "lax
completion".  At least not any lax completion I've ever seen.

> > You no doubt have your reasons for doing things your own way.
> > Why not do that in your own library (compatible or incompatible)?
> 
> Stefan wanted company-mode to use completion-at-point-functions, and the
> completion tables they return. They're not sufficiently powerful, yet.

So don't use `completion-at-point-functions', if they don't give you
what you want.  I don't use them.  That shouldn't mean that you need
to screw `all-completions' or `minibuffer-completion-table'.

`completion-at-point-functions', too limited or not, is fairly recent.
`all-completions' and `minibuffer-completion-table' are decades old.
They are solid, basic tools.

Re: find-file-project

[Dmitry Gutov]

On 01/21/2016 02:46 AM, Drew Adams wrote:

> Maybe you meant `try-completion'?  And even that uses `completion-styles',
> which is pretty flexible (not in the way that I wanted, but at least it
> allows for different kinds of completion).

completion-styles are too slow. We want to be able to delegate matching 
to an external program.

Maybe try reading the thread next time, before jumping in?

> So don't use `completion-at-point-functions', if they don't give you
> what you want.  I don't use them.  That shouldn't mean that you need
> to screw `all-completions' or `minibuffer-completion-table'.

completion-at-point-functions are fine. The completion tables, as 
currently defined, are insufficiently powerful.

We also want to use the new kind of completion tables for e.g. 
project-find-file, which does use the minibuffer.

> `completion-at-point-functions', too limited or not, is fairly recent.
> `all-completions' and `minibuffer-completion-table' are decades old.
> They are solid, basic tools.

Emacs should improve over time.

RE: find-file-project

[Drew Adams]

> > Maybe you meant `try-completion'?  And even that uses `completion-styles',
> > which is pretty flexible (not in the way that I wanted, but at least it
> > allows for different kinds of completion).
> 
> completion-styles are too slow. We want to be able to delegate matching
> to an external program.

The question is why you need to break stuff to be able to do that.

> Maybe try reading the thread next time, before jumping in?

Maybe try explaining why you need to break `all-completions' and
`minibuffer-completion-table'.  Don't suggest breaking it and I
probably won't jump in at all.

Why can't you build your shiny new feature outside the existing
completion framework, if it doesn't suit your purpose?  Why do
you (think you) need to break it?

> > So don't use `completion-at-point-functions', if they don't give you
> > what you want.  I don't use them.  That shouldn't mean that you need
> > to screw `all-completions' or `minibuffer-completion-table'.
> 
> completion-at-point-functions are fine. The completion tables, as
> currently defined, are insufficiently powerful.

So use whatever alternative you (think you) need, but outside the
current framework - instead of just breaking that framework to fit
what you want to do.

If `minibuffer-completion-table' is not your cup of tea, leave it
alone and go make your own, "sufficiently powerful".  But leave it
alone.  That's all I'm asking: add (if really necessary); don't break.

> We also want to use the new kind of completion tables for e.g.
> project-find-file, which does use the minibuffer.

So what?  There are lots of ways to use the minibuffer.  And you
can add to them if they aren't enough for you.  You should not
need to break existing ways of using it just to add your own way.

New this, new that, insufficiently powerful this, more powerful that...

My point is that you can just do what you (think you) need to do
without sacrificing what already exists.

> > `completion-at-point-functions', too limited or not, is fairly recent.
> > `all-completions' and `minibuffer-completion-table' are decades old.
> > They are solid, basic tools.
> 
> Emacs should improve over time.

You can improve Emacs (or even change it in other ways that you
might think are improvements) without breaking the longstanding
completion framework.  Just add another one, for your "sufficiently
powerful" enhancements, if the existing one doesn't suit your needs.

Re: find-file-project

[Dmitry Gutov]

On 01/21/2016 03:08 AM, Drew Adams wrote:

> Maybe try explaining why you need to break `all-completions' and
> `minibuffer-completion-table'.  Don't suggest breaking it and I
> probably won't jump in at all.

We've discussed it in this thread already. Do you always ask for 
special, personal explanations?

> Why can't you build your shiny new feature outside the existing
> completion framework, if it doesn't suit your purpose?  Why do
> you (think you) need to break it?

It's a set of interconnected new features. When you want to introduce a 
new feature, it doesn't make sense to always create an entirely new 
system. For one thing, many existing users (of the previous system) will 
be disappointed that they got left behind.

And the new system will be inferior in terms of the audience (users and 
backends that plug into it).

> So use whatever alternative you (think you) need, but outside the
> current framework - instead of just breaking that framework to fit
> what you want to do.

You're implying there's a good alternative.

If you dislike change so much, why don't you stop upgrading Emacs?

> So what?  There are lots of ways to use the minibuffer.  And you
> can add to them if they aren't enough for you.  You should not
> need to break existing ways of using it just to add your own way.

That wouldn't make it lose in capability. Some existing code would just 
need to be updated.

Re: find-file-project

[Dmitry Gutov]

On 01/20/2016 04:32 AM, Stefan Monnier wrote:

> FWIW, see the current diff I have.

IIUC, it creates a non-standard completion table, too. I.e. one that 
violates the abstraction boundaries (e.g. using 
file-cache-minibuffer-message).

Re: find-file-project

[Stefan Monnier]

>> FWIW, see the current diff I have.
> IIUC, it creates a non-standard completion table, too. I.e. one that
> violates the abstraction boundaries (e.g. using
> file-cache-minibuffer-message).

I can't remember the details, but I do remember that it needs either
more work or a rewrite to implement it differently (and in either case,
probably a change in the user-visible behavior), indeed.


        Stefan