From mboxrd@z Thu Jan 1 00:00:00 1970 From: ludo@gnu.org (Ludovic =?UTF-8?Q?Court=C3=A8s?=) Subject: bug#22937: guix package fails when --substitute-urls specifies an HTTPS endpoint Date: Wed, 09 Mar 2016 23:51:57 +0100 Message-ID: <87pov38dte.fsf@gnu.org> References: <87k2ldjxz5.fsf@gmail.com> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" Return-path: Received: from eggs.gnu.org ([2001:4830:134:3::10]:57293) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1admyZ-0002CV-5i for bug-guix@gnu.org; Wed, 09 Mar 2016 17:53:08 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1admyU-0001bP-Do for bug-guix@gnu.org; Wed, 09 Mar 2016 17:53:07 -0500 Received: from debbugs.gnu.org ([208.118.235.43]:44821) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1admyU-0001bL-A1 for bug-guix@gnu.org; Wed, 09 Mar 2016 17:53:02 -0500 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84) (envelope-from ) id 1admyU-0005gf-5G for bug-guix@gnu.org; Wed, 09 Mar 2016 17:53:02 -0500 Sender: "Debbugs-submit" Resent-Message-ID: In-Reply-To: <87k2ldjxz5.fsf@gmail.com> (Chris Marusich's message of "Mon, 07 Mar 2016 22:16:30 -0800") List-Id: Bug reports for GNU Guix List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: bug-guix-bounces+gcggb-bug-guix=m.gmane.org@gnu.org Sender: bug-guix-bounces+gcggb-bug-guix=m.gmane.org@gnu.org To: Chris Marusich Cc: 22937@debbugs.gnu.org --=-=-= Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable First, the graceless error handling is fixed in 204d34ff961d6dabf18b255decc29712e03afef0. Second, here=E2=80=99s a preliminary patch that almost works with . The problem is that sometimes the server closes the connection unexpectedly, leading to an obscure backtrace like this: --8<---------------cut here---------------start------------->8--- substitute: 629: 6 [lookup-narinfos "https://hydra-mirror.marusich.info" #] substitute: 585: 5 [fetch-narinfos "https://hydra-mirror.marusich.info" #] substitute: 510: 4 [http-multiple-get # ...] substitute: In web/response.scm: substitute: 197: 3 [read-response #] substitute: In web/http.scm: substitute: 1157: 2 [read-response-line #] substitute: 151: 1 [read-header-line #] substitute: In unknown file: substitute: ?: 0 [%read-line #] substitute:=20 substitute: ERROR: In procedure %read-line: substitute: ERROR: Throw to key `gnutls-error' with args `(# fill_session_record_port_input)'. --8<---------------cut here---------------end--------------->8--- The =E2=80=9Cerror in the pull function=E2=80=9D is because =E2=80=98gnutls= _record_recv=E2=80=99 got ECONNRESET while reading. I wonder whether this could be due to the particular configuration of nginx at Cloudfront, so I=E2=80=99ll try with another server (I=E2=80=99ve = set up Let=E2=80=99s Encrypt on that server but it=E2=80=99s not accessible yet via port 443.) To be continued! Ludo=E2=80=99. --=-=-= Content-Type: text/x-patch; charset=utf-8 Content-Disposition: inline Content-Transfer-Encoding: quoted-printable diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index b82fc17..df95de0 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -32,6 +32,7 @@ #:use-module ((guix build utils) #:select (mkdir-p dump-port)) #:use-module ((guix build download) #:select (progress-proc uri-abbreviation + open-connection-for-uri store-path-abbreviation byte-count->string)) #:use-module (ice-9 rdelim) #:use-module (ice-9 regex) @@ -171,7 +172,7 @@ to the caller without emitting an error message." (let ((port (open-file (uri-path uri) (if buffered? "rb" "r0b")))) (values port (stat:size (stat port))))) - ((http) + ((http https) (guard (c ((http-get-error? c) (let ((code (http-get-error-code c))) (if (and (=3D code 404) quiet-404?) @@ -201,10 +202,10 @@ to the caller without emitting an error message." (close-port port)))) (begin (when (or (not port) (port-closed? port)) - (set! port (open-socket-for-uri uri)) + (set! port (open-connection-for-uri uri)) (unless buffered? (setvbuf port _IONBF))) - (http-fetch uri #:text? #f #:port port)))))))) + (http-fetch uri #:text? #f #:port port)))))) (else (leave (_ "unsupported substitute URI scheme: ~a~%") (uri->string uri))))) @@ -478,20 +479,26 @@ may be #f, in which case it indicates that PATH is un= available at CACHE-URL." ".narinfo"))) (build-request (string->uri url) #:method 'GET))) =20 -(define (http-multiple-get base-url proc seed requests) - "Send all of REQUESTS to the server at BASE-URL. Call PROC for each +(define (http-multiple-get base-uri proc seed requests) + "Send all of REQUESTS to the server at BASE-URI. Call PROC for each response, passing it the request object, the response, a port from which to read the response body, and the previous result, starting with SEED, =C3= =A0 la 'fold'. Return the final result." (let connect ((requests requests) (result seed)) - ;; (format (current-error-port) "connecting (~a requests left)..." - ;; (length requests)) - (let ((p (open-socket-for-uri base-url))) + (format (current-error-port) "connecting (~a requests left)..." + (length requests)) + (let ((p (open-connection-for-uri base-uri))) + ;; For HTTPS, P is not a file port and does not support 'setvbuf'. + (when (file-port? p) + (setvbuf p _IOFBF (expt 2 16))) + ;; Send all of REQUESTS in a row. - (setvbuf p _IOFBF (expt 2 16)) - (for-each (cut write-request <> p) requests) - (force-output p) + ;; XXX: Do our own caching to work around . + (let-values (((buffer get) (open-bytevector-output-port))) + (for-each (cut write-request <> buffer) requests) + (put-bytevector p (get)) + (force-output p)) =20 ;; Now start processing responses. (let loop ((requests requests) @@ -501,6 +508,8 @@ read the response body, and the previous result, starti= ng with SEED, =C3=A0 la (reverse result)) ((head tail ...) (let* ((resp (read-response p)) + ;; (xxx (format (current-error-port) + ;; "http response: ~s~%" resp)) (body (response-body-port resp)) (result (proc head resp body result))) ;; The server can choose to stop responding at any time, in w= hich @@ -570,10 +579,10 @@ if file doesn't exist, and the narinfo otherwise." =20 (define (do-fetch uri) (case (and=3D> uri uri-scheme) - ((http) + ((http https) (let ((requests (map (cut narinfo-request url <>) paths))) (update-progress!) - (let ((result (http-multiple-get url + (let ((result (http-multiple-get uri handle-narinfo-response '() requests))) (newline (current-error-port)) --=-=-=--