From: Kyle Meyer <kyle@kyleam.com>
To: gwl-devel@gnu.org
Cc: Kyle Meyer <kyle@kyleam.com>
Subject: [PATCH] workflow: Consider unspecified free inputs when checking cache.
Date: Wed, 12 Jun 2019 23:48:21 -0400 [thread overview]
Message-ID: <20190613034821.1705513-1-kyle@kyleam.com> (raw)
When deciding whether a process is cached, we take into account the
modification time and size of free inputs, but only those that are
explicitly specified on the command line. As a result, the same
updated input file that would invalidate the cache when given
explicitly to --input does _not_ invalidate the cache when not
specified but instead picked from the current working directory.
Correct this discrepancy by including unspecified free inputs in the
cache prefix calculation.
* gwl/workflows.scm (workflow-run): Pass unspecified free inputs to
make-process->cache-prefix.
---
Note: This is mostly code movement, so something like
git show --color-moved=zebra --color-moved-ws=allow-indentation-change
might be helpful for reviewing.
gwl/workflows.scm | 87 +++++++++++++++++++++++++----------------------
1 file changed, 47 insertions(+), 40 deletions(-)
diff --git a/gwl/workflows.scm b/gwl/workflows.scm
index 50dc2ee..efdf089 100644
--- a/gwl/workflows.scm
+++ b/gwl/workflows.scm
@@ -262,46 +262,51 @@ (define* (workflow-run workflow engine
inputs)
(() '())
(mapping mapping)))
+
+ (define-values (input-names input-files)
+ (match inputs-map
+ (() (values '() '()))
+ (_ (apply values
+ (apply zip inputs-map)))))
+
+ (define unspecified-inputs
+ (lset-difference equal?
+ (workflow-free-inputs workflow)
+ input-names))
+
(define (inputs-valid?)
- (let-values (((input-names input-files)
- (match inputs-map
- (() (values '() '()))
- (_ (apply values
- (apply zip inputs-map))))))
- (match (lset-difference equal?
- (workflow-free-inputs workflow)
- input-names)
- (()
- ;; verify input files
- (match (filter (negate file-exists?) input-files)
- (()
- ;; Link all mapped input files to their target locations
- ;; TODO: ensure that target directories exist.
- (unless (null? inputs-map)
- (for-each (match-lambda
- ((target source)
- (unless (file-exists? target)
- (link source target))))
- inputs-map))
- #t)
- (missing
- (format (current-error-port)
- "Missing files: ~{~% * ~a~}.~%"
- missing)
- #f)))
- (missing
- ;; Try to find the files in the environment.
- ;; XXX Tell user that we pick the files from the current
- ;; working directory.
- ;; XXX These files would need to be mapped into the
- ;; container.
- (let* ((found (filter file-exists? missing))
- (really-missing (lset-difference equal? missing found)))
- (or (null? really-missing)
- (begin (format (current-error-port)
- "Missing inputs: ~{~% * ~a~}.~%Provide them with --input=NAME=FILE.~%"
- really-missing)
- #f)))))))
+ (match unspecified-inputs
+ (()
+ ;; verify input files
+ (match (filter (negate file-exists?) input-files)
+ (()
+ ;; Link all mapped input files to their target locations
+ ;; TODO: ensure that target directories exist.
+ (unless (null? inputs-map)
+ (for-each (match-lambda
+ ((target source)
+ (unless (file-exists? target)
+ (link source target))))
+ inputs-map))
+ #t)
+ (missing
+ (format (current-error-port)
+ "Missing files: ~{~% * ~a~}.~%"
+ missing)
+ #f)))
+ (missing
+ ;; Try to find the files in the environment.
+ ;; XXX Tell user that we pick the files from the current
+ ;; working directory.
+ ;; XXX These files would need to be mapped into the
+ ;; container.
+ (let* ((found (filter file-exists? missing))
+ (really-missing (lset-difference equal? missing found)))
+ (or (null? really-missing)
+ (begin (format (current-error-port)
+ "Missing inputs: ~{~% * ~a~}.~%Provide them with --input=NAME=FILE.~%"
+ really-missing)
+ #f))))))
(define ordered-processes
(workflow-run-order workflow #:parallel? parallel?))
(define (run)
@@ -309,7 +314,9 @@ (define* (workflow-run workflow engine
(runner (process-engine-runner engine)))
(define process->cache-prefix
(make-process->cache-prefix workflow
- inputs-map
+ (append inputs-map
+ (map (lambda (x) (list x x))
+ unspecified-inputs))
ordered-processes
engine))
(define cached?
--
2.22.0
next reply other threads:[~2019-06-13 3:48 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-06-13 3:48 Kyle Meyer [this message]
2019-06-24 13:54 ` [PATCH] workflow: Consider unspecified free inputs when checking cache Ricardo Wurmus
2019-06-25 4:30 ` Kyle Meyer
2019-06-25 17:33 ` zimoun
2019-06-25 18:33 ` Ricardo Wurmus
2019-06-26 1:31 ` Kyle Meyer
2019-06-26 7:05 ` Ricardo Wurmus
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://www.guixwl.org/
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190613034821.1705513-1-kyle@kyleam.com \
--to=kyle@kyleam.com \
--cc=gwl-devel@gnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).