all messages for Guix-related lists mirrored at yhetil.org
 help / color / mirror / code / Atom feed
* [PATCH] workflow: Consider unspecified free inputs when checking cache.
@ 2019-06-13  3:48 Kyle Meyer
  2019-06-24 13:54 ` Ricardo Wurmus
  0 siblings, 1 reply; 7+ messages in thread
From: Kyle Meyer @ 2019-06-13  3:48 UTC (permalink / raw)
  To: gwl-devel; +Cc: Kyle Meyer

When deciding whether a process is cached, we take into account the
modification time and size of free inputs, but only those that are
explicitly specified on the command line.  As a result, the same
updated input file that would invalidate the cache when given
explicitly to --input does _not_ invalidate the cache when not
specified but instead picked from the current working directory.

Correct this discrepancy by including unspecified free inputs in the
cache prefix calculation.

* gwl/workflows.scm (workflow-run): Pass unspecified free inputs to
make-process->cache-prefix.
---
Note: This is mostly code movement, so something like

  git show --color-moved=zebra --color-moved-ws=allow-indentation-change

might be helpful for reviewing.

 gwl/workflows.scm | 87 +++++++++++++++++++++++++----------------------
 1 file changed, 47 insertions(+), 40 deletions(-)

diff --git a/gwl/workflows.scm b/gwl/workflows.scm
index 50dc2ee..efdf089 100644
--- a/gwl/workflows.scm
+++ b/gwl/workflows.scm
@@ -262,46 +262,51 @@ (define* (workflow-run workflow engine
                 inputs)
       (() '())
       (mapping mapping)))
+
+  (define-values (input-names input-files)
+    (match inputs-map
+      (() (values '() '()))
+      (_ (apply values
+                (apply zip inputs-map)))))
+
+  (define unspecified-inputs
+    (lset-difference equal?
+                     (workflow-free-inputs workflow)
+                     input-names))
+
   (define (inputs-valid?)
-    (let-values (((input-names input-files)
-                  (match inputs-map
-                    (() (values '() '()))
-                    (_ (apply values
-                              (apply zip inputs-map))))))
-      (match (lset-difference equal?
-                              (workflow-free-inputs workflow)
-                              input-names)
-        (()
-         ;; verify input files
-         (match (filter (negate file-exists?) input-files)
-           (()
-            ;; Link all mapped input files to their target locations
-            ;; TODO: ensure that target directories exist.
-            (unless (null? inputs-map)
-              (for-each (match-lambda
-                          ((target source)
-                           (unless (file-exists? target)
-                             (link source target))))
-                        inputs-map))
-            #t)
-           (missing
-            (format (current-error-port)
-                    "Missing files: ~{~%  * ~a~}.~%"
-                    missing)
-            #f)))
-        (missing
-         ;; Try to find the files in the environment.
-         ;; XXX Tell user that we pick the files from the current
-         ;; working directory.
-         ;; XXX These files would need to be mapped into the
-         ;; container.
-         (let* ((found (filter file-exists? missing))
-               (really-missing (lset-difference equal? missing found)))
-           (or (null? really-missing)
-               (begin (format (current-error-port)
-                              "Missing inputs: ~{~%  * ~a~}.~%Provide them with --input=NAME=FILE.~%"
-                              really-missing)
-                      #f)))))))
+    (match unspecified-inputs
+      (()
+       ;; verify input files
+       (match (filter (negate file-exists?) input-files)
+         (()
+          ;; Link all mapped input files to their target locations
+          ;; TODO: ensure that target directories exist.
+          (unless (null? inputs-map)
+            (for-each (match-lambda
+                        ((target source)
+                         (unless (file-exists? target)
+                           (link source target))))
+                      inputs-map))
+          #t)
+         (missing
+          (format (current-error-port)
+                  "Missing files: ~{~%  * ~a~}.~%"
+                  missing)
+          #f)))
+      (missing
+       ;; Try to find the files in the environment.
+       ;; XXX Tell user that we pick the files from the current
+       ;; working directory.
+       ;; XXX These files would need to be mapped into the
+       ;; container.
+       (let* ((found (filter file-exists? missing))
+              (really-missing (lset-difference equal? missing found)))
+         (or (null? really-missing)
+             (begin (format (current-error-port)
+                            "Missing inputs: ~{~%  * ~a~}.~%Provide them with --input=NAME=FILE.~%"
+                            really-missing)
+                    #f))))))
   (define ordered-processes
     (workflow-run-order workflow #:parallel? parallel?))
   (define (run)
@@ -309,7 +314,9 @@ (define* (workflow-run workflow engine
           (runner (process-engine-runner engine)))
       (define process->cache-prefix
         (make-process->cache-prefix workflow
-                                    inputs-map
+                                    (append inputs-map
+                                            (map (lambda (x) (list x x))
+                                                 unspecified-inputs))
                                     ordered-processes
                                     engine))
       (define cached?
-- 
2.22.0

^ permalink raw reply related	[flat|nested] 7+ messages in thread

end of thread, other threads:[~2019-06-26  7:05 UTC | newest]

Thread overview: 7+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-06-13  3:48 [PATCH] workflow: Consider unspecified free inputs when checking cache Kyle Meyer
2019-06-24 13:54 ` Ricardo Wurmus
2019-06-25  4:30   ` Kyle Meyer
2019-06-25 17:33     ` zimoun
2019-06-25 18:33       ` Ricardo Wurmus
2019-06-26  1:31       ` Kyle Meyer
2019-06-26  7:05         ` Ricardo Wurmus

Code repositories for project(s) associated with this external index

	https://git.savannah.gnu.org/cgit/guix.git

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.