From mboxrd@z Thu Jan  1 00:00:00 1970
From: Kyle Meyer <kyle@kyleam.com>
Subject: [PATCH] workflow: Consider unspecified free inputs when checking
 cache.
Date: Wed, 12 Jun 2019 23:48:21 -0400
Message-Id: <20190613034821.1705513-1-kyle@kyleam.com>
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable
To: gwl-devel@gnu.org
Cc: Kyle Meyer <kyle@kyleam.com>
List-ID: <gwl-devel.gnu.org>

When deciding whether a process is cached, we take into account the
modification time and size of free inputs, but only those that are
explicitly specified on the command line.  As a result, the same
updated input file that would invalidate the cache when given
explicitly to --input does _not_ invalidate the cache when not
specified but instead picked from the current working directory.

Correct this discrepancy by including unspecified free inputs in the
cache prefix calculation.

* gwl/workflows.scm (workflow-run): Pass unspecified free inputs to
make-process->cache-prefix.
---
Note: This is mostly code movement, so something like

  git show --color-moved=3Dzebra --color-moved-ws=3Dallow-indentation-cha=
nge

might be helpful for reviewing.

 gwl/workflows.scm | 87 +++++++++++++++++++++++++----------------------
 1 file changed, 47 insertions(+), 40 deletions(-)

diff --git a/gwl/workflows.scm b/gwl/workflows.scm
index 50dc2ee..efdf089 100644
--- a/gwl/workflows.scm
+++ b/gwl/workflows.scm
@@ -262,46 +262,51 @@ (define* (workflow-run workflow engine
                 inputs)
       (() '())
       (mapping mapping)))
+
+  (define-values (input-names input-files)
+    (match inputs-map
+      (() (values '() '()))
+      (_ (apply values
+                (apply zip inputs-map)))))
+
+  (define unspecified-inputs
+    (lset-difference equal?
+                     (workflow-free-inputs workflow)
+                     input-names))
+
   (define (inputs-valid?)
-    (let-values (((input-names input-files)
-                  (match inputs-map
-                    (() (values '() '()))
-                    (_ (apply values
-                              (apply zip inputs-map))))))
-      (match (lset-difference equal?
-                              (workflow-free-inputs workflow)
-                              input-names)
-        (()
-         ;; verify input files
-         (match (filter (negate file-exists?) input-files)
-           (()
-            ;; Link all mapped input files to their target locations
-            ;; TODO: ensure that target directories exist.
-            (unless (null? inputs-map)
-              (for-each (match-lambda
-                          ((target source)
-                           (unless (file-exists? target)
-                             (link source target))))
-                        inputs-map))
-            #t)
-           (missing
-            (format (current-error-port)
-                    "Missing files: ~{~%  * ~a~}.~%"
-                    missing)
-            #f)))
-        (missing
-         ;; Try to find the files in the environment.
-         ;; XXX Tell user that we pick the files from the current
-         ;; working directory.
-         ;; XXX These files would need to be mapped into the
-         ;; container.
-         (let* ((found (filter file-exists? missing))
-               (really-missing (lset-difference equal? missing found)))
-           (or (null? really-missing)
-               (begin (format (current-error-port)
-                              "Missing inputs: ~{~%  * ~a~}.~%Provide th=
em with --input=3DNAME=3DFILE.~%"
-                              really-missing)
-                      #f)))))))
+    (match unspecified-inputs
+      (()
+       ;; verify input files
+       (match (filter (negate file-exists?) input-files)
+         (()
+          ;; Link all mapped input files to their target locations
+          ;; TODO: ensure that target directories exist.
+          (unless (null? inputs-map)
+            (for-each (match-lambda
+                        ((target source)
+                         (unless (file-exists? target)
+                           (link source target))))
+                      inputs-map))
+          #t)
+         (missing
+          (format (current-error-port)
+                  "Missing files: ~{~%  * ~a~}.~%"
+                  missing)
+          #f)))
+      (missing
+       ;; Try to find the files in the environment.
+       ;; XXX Tell user that we pick the files from the current
+       ;; working directory.
+       ;; XXX These files would need to be mapped into the
+       ;; container.
+       (let* ((found (filter file-exists? missing))
+              (really-missing (lset-difference equal? missing found)))
+         (or (null? really-missing)
+             (begin (format (current-error-port)
+                            "Missing inputs: ~{~%  * ~a~}.~%Provide them=
 with --input=3DNAME=3DFILE.~%"
+                            really-missing)
+                    #f))))))
   (define ordered-processes
     (workflow-run-order workflow #:parallel? parallel?))
   (define (run)
@@ -309,7 +314,9 @@ (define* (workflow-run workflow engine
           (runner (process-engine-runner engine)))
       (define process->cache-prefix
         (make-process->cache-prefix workflow
-                                    inputs-map
+                                    (append inputs-map
+                                            (map (lambda (x) (list x x))
+                                                 unspecified-inputs))
                                     ordered-processes
                                     engine))
       (define cached?
--=20
2.22.0