Re: 23.0.92; Bug in Directory Variables

Re: 23.0.92; Bug in Directory Variables

[Chong Yidong]

The following patch changes the directory local variables code so that
the cache stores the file atime.  Before using each cache entry, we
check that against the file modtime.

This is not an ideal time to make changes of this sort, but I think this
problem (i.e., inability to detect changes in .dir-locals.el files) is
quite a serious one.  So please help me review this patch.

*** trunk/lisp/files.el.~1.1044.~	2009-04-08 10:42:40.000000000 -0400
--- trunk/lisp/files.el	2009-04-08 22:50:03.000000000 -0400
***************
*** 3187,3194 ****
  (defvar dir-locals-class-alist '()
    "Alist mapping class names (symbols) to variable lists.")
  
! (defvar dir-locals-directory-alist '()
!   "Alist mapping directory roots to variable classes.")
  
  (defsubst dir-locals-get-class-variables (class)
    "Return the variable list for CLASS."
--- 3187,3199 ----
  (defvar dir-locals-class-alist '()
    "Alist mapping class names (symbols) to variable lists.")
  
! (defvar dir-locals-directory-cache '()
!   "List of cached directory roots for directory-local variable files.
! Each element in this list has the form (DIR ATIME CLASS), where
! DIR is the name of the directory, ATIME is the time when this
! cache entry was created (as a floating point number, per
! `float-time'), and CLASS is the name of a project class (a
! symbol).")
  
  (defsubst dir-locals-get-class-variables (class)
    "Return the variable list for CLASS."
***************
*** 3241,3247 ****
    (setq directory (file-name-as-directory (expand-file-name directory)))
    (unless (assq class dir-locals-class-alist)
      (error "No such class `%s'" (symbol-name class)))
!   (push (cons directory class) dir-locals-directory-alist))
  
  (defun dir-locals-set-class-variables (class variables)
    "Map the type CLASS to a list of variable settings.
--- 3246,3252 ----
    (setq directory (file-name-as-directory (expand-file-name directory)))
    (unless (assq class dir-locals-class-alist)
      (error "No such class `%s'" (symbol-name class)))
!   (push (list directory (float-time) class) dir-locals-directory-cache))
  
  (defun dir-locals-set-class-variables (class variables)
    "Map the type CLASS to a list of variable settings.
***************
*** 3284,3295 ****
  across different environments and users.")
  
  (defun dir-locals-find-file (file)
!   "Find the directory-local variables FILE.
! This searches upward in the directory tree.
! If a local variables file is found, the file name is returned.
! If the file is already registered, a cons from
! `dir-locals-directory-alist' is returned.
! Otherwise this returns nil."
    (setq file (expand-file-name file))
    (let* ((dir-locals-file-name
  	  (if (eq system-type 'ms-dos)
--- 3289,3304 ----
  across different environments and users.")
  
  (defun dir-locals-find-file (file)
!   "Find the directory-local variables for FILE.
! This searches upward in the directory tree from FILE.
! 
! If the file is already registered and has not been modified since
!  we added it to `dir-locals-directory-cache', this returns a cons
!  cell of the form (DIRECTORY . CLASS), using the entry in
!  `dir-locals-directory-cache'.
! Otherwise, if a local variables file is found, the file name is
!  returned.
! Otherwise, this returns nil."
    (setq file (expand-file-name file))
    (let* ((dir-locals-file-name
  	  (if (eq system-type 'ms-dos)
***************
*** 3300,3318 ****
      ;; `locate-dominating-file' may have abbreviated the name.
      (when locals-file
        (setq locals-file (expand-file-name dir-locals-file-name locals-file)))
!     (dolist (elt dir-locals-directory-alist)
        (when (and (eq t (compare-strings file nil (length (car elt))
  					(car elt) nil nil
  					(memq system-type
  					      '(windows-nt cygwin ms-dos))))
  		 (> (length (car elt)) (length (car dir-elt))))
  	(setq dir-elt elt)))
!     (if (and locals-file dir-elt)
! 	(if (> (length (file-name-directory locals-file))
! 	       (length (car dir-elt)))
! 	    locals-file
! 	  dir-elt)
!       (or locals-file dir-elt))))
  
  (defun dir-locals-read-from-file (file)
    "Load a variables FILE and register a new class and instance.
--- 3309,3338 ----
      ;; `locate-dominating-file' may have abbreviated the name.
      (when locals-file
        (setq locals-file (expand-file-name dir-locals-file-name locals-file)))
!     ;; Find the best cached value in `dir-locals-directory-cache'.
!     (dolist (elt dir-locals-directory-cache)
        (when (and (eq t (compare-strings file nil (length (car elt))
  					(car elt) nil nil
  					(memq system-type
  					      '(windows-nt cygwin ms-dos))))
  		 (> (length (car elt)) (length (car dir-elt))))
  	(setq dir-elt elt)))
!     (let ((use-cache (and dir-elt
! 			  (or (null locals-file)
! 			      (<= (length (file-name-directory locals-file))
! 				  (length (car dir-elt)))))))
!       (if use-cache
! 	  ;; Check the validity of the cache.
! 	  (if (and (file-readable-p (car dir-elt))
! 		   (< (float-time (nth 5 (file-attributes (car dir-elt))))
! 		      (nth 1 dir-elt)))
! 	      ;; This cache entry is OK.
! 	      (cons (nth 0 dir-elt) (nth 2 dir-elt))
! 	    ;; This cache entry is invalid; clear it.
! 	    (setq dir-locals-directory-cache
! 		  (delq dir-elt dir-locals-directory-cache))
! 	    locals-file)
! 	locals-file))))
  
  (defun dir-locals-read-from-file (file)
    "Load a variables FILE and register a new class and instance.

bug#2833: 23.0.92; Bug in Directory Variables

[David De La Harpe Golden]

Chong Yidong wrote:
> The following patch changes the directory local variables code so that
> the cache stores the file atime.  Before using each cache entry, we
> check that against the file modtime.
> 
> This is not an ideal time to make changes of this sort, but I think this
> problem (i.e., inability to detect changes in .dir-locals.el files) is
> quite a serious one.  So please help me review this patch.
>

I suspect the systems where a problem with the patch will be most 
noticeable will be windoze, since there's an extra file attributes call? 
  I have a suspicion you almost might as well abandon the caching given 
the scary timings we've seen for that in the past.  I could be wrong though.

Re: 23.0.92; Bug in Directory Variables

[David De La Harpe Golden]

Chong Yidong wrote:
> The following patch changes the directory local variables code so that
> the cache stores the file atime.  Before using each cache entry, we
> check that against the file modtime.
> 
> This is not an ideal time to make changes of this sort, but I think this
> problem (i.e., inability to detect changes in .dir-locals.el files) is
> quite a serious one.  So please help me review this patch.
>

I suspect the systems where a problem with the patch will be most 
noticeable will be windoze, since there's an extra file attributes call? 
  I have a suspicion you almost might as well abandon the caching given 
the scary timings we've seen for that in the past.  I could be wrong though.

bug#2833: 23.0.92; Bug in Directory Variables

[Stefan Monnier]

> The following patch changes the directory local variables code so that
> the cache stores the file atime.  Before using each cache entry, we
> check that against the file modtime.

I'm not sure this is the right thing to do:
- We don't know that the (float-time) is in sync with the filesystem's
  time, so the check may not work right.  Better check the file's
  current mtime against the file's mtime when it was last read.
- the variable you changed could previously be setq in the .emacs,
  whereas you changed it into an internal var.

FWIW, here's the patch I was working on instead.


        Stefan


--- files.el.~1.1044.~	2009-04-08 20:09:55.000000000 -0400
+++ files.el	2009-04-09 10:24:53.000000000 -0400
@@ -3314,21 +3314,31 @@
 	  dir-elt)
       (or locals-file dir-elt))))
 
+(defvar dir-locals--file-cache nil
+  "Cache of dir-locals files's contents.")
+
 (defun dir-locals-read-from-file (file)
   "Load a variables FILE and register a new class and instance.
 FILE is the name of the file holding the variables to apply.
 The new class name is the same as the directory in which FILE
 is found.  Returns the new class name."
+  (let ((mtime (nth 5 (file-attributes file)))
+        (cache (assoc file dir-locals--file-cache)))
+    (unless (equal (nth 1 cache) mtime)
+      (setq dir-locals--file-cache (delq cache dir-locals--file-cache))
+      (setq cache nil))
+    (if cache
+        (nth 2 cache)
+      (let ((val
   (with-temp-buffer
-    ;; We should probably store the modtime of FILE and then
-    ;; reload it whenever it changes.
     (insert-file-contents file)
     (let* ((dir-name (file-name-directory file))
 	   (class-name (intern dir-name))
 	   (variables (read (current-buffer))))
       (dir-locals-set-class-variables class-name variables)
-      (dir-locals-set-directory-class dir-name class-name)
-      class-name)))
+                 class-name))))
+        (push (list file mtime val) dir-locals--file-cache)
+        val))))
 
 (declare-function c-postprocess-file-styles "cc-mode" ())
 

bug#2833: 23.0.92; Bug in Directory Variables

[Chong Yidong]

Stefan Monnier <monnier@iro.umontreal.ca> writes:

> - We don't know that the (float-time) is in sync with the filesystem's
>   time, so the check may not work right.  Better check the file's
>   current mtime against the file's mtime when it was last read.

Yes, good point.

> - the variable you changed could previously be setq in the .emacs,
>   whereas you changed it into an internal var.

It was always a defvar instead of a defcustom.  I don't think it makes
sense for it to be user-customizable.

> FWIW, here's the patch I was working on instead.

Is this patch complete?  By the time dir-locals-read-from-file is
called, hack-dir-local-variables has already decided that there is no
cached value.  Conversely, if there is a cached value, it will be used
without calling dir-locals-read-from-file.  So we need to change
dir-locals-find-file as well (which is what my patch did).