bug#26302: Deploying the i18n’d web site

["pelzflorian (Florian Pelz)" <pelzflorian@pelzflorian.de>]

[-- Attachment #1: Type: text/plain, Size: 1855 bytes --]

On Thu, Apr 09, 2020 at 08:58:14PM +0200, Bengt Richter wrote:
> Wondering, could dnsmasq help?

I am already using dnsmasq to access a virtual machine of the machine
hosting the Guix website so <guix.gnu.org/.i18n/de/> can replace the
Guix website while old URLs keep working.

https://git.savannah.gnu.org/cgit/guix/maintenance.git/tree/hydra/berlin.scm.

I configured dnsmasq with NetworkManager.

(network-manager-service-type config =>
  (network-manager-configuration
    (inherit config)
    (dns "dnsmasq")
    (vpn-plugins (list network-manager-openconnect)))))))

Also I made the network manager configuration as described in the Guix
manual.

I made the attached modifications to berlin.scm so I can test without
the VM starting to build the world.

Then I changed my /etc/hosts

127.0.0.1 localhost florianmacbook
::1       localhost florianmacbook
172.28.112.244 guix.gnu.org guix.gnu.org

Then I made the changes from my previous patch, except those
nginx-server-configuration that I removed because I cannot test TLS.

I compile the VM using

~/git/maintenance$ GUILE_LOAD_PATH=$GUILE_LOAD_PATH:$(pwd)/modules guix system vm-image --image-size=5G --fallback /home/florian/git/maintenance/hydra/berlin.scm

(More than --image-size=5G to be safe.)  I copy the resulting image
and name it berlin1.img, make it writable and run it via

qemu-system-x86_64 -enable-kvm berlin1.img -m 2048 -nic tap,ifname=tap0,script=no,downscript=no

When running the virtual machine, I log in as root and do “herd stop
mcron” so the VM does not get overloaded building all kinds of things
I do not need.  I run the command from “herd schedule mcron” to build
the guix.gnu.org website manually once.

I am confused by all the ways to rewrite, redirect and try_files in
nginx.  I would be happy if others with more knowledge could help.

Regards,
Florian

[-- Attachment #2: 0001-various-changes-for-local-testing.patch --]
[-- Type: text/plain, Size: 18144 bytes --]

From b914a165a2e9b9b7bd7760cec09a531c70611141 Mon Sep 17 00:00:00 2001
From: Florian Pelz <pelzflorian@pelzflorian.de>
Date: Wed, 8 Apr 2020 00:44:18 +0200
Subject: [PATCH 1/2] [various changes for local testing]

---
 hydra/berlin.scm                    |  80 +--------
 hydra/modules/sysadmin/services.scm |  39 +----
 hydra/nginx/berlin.scm              | 241 +---------------------------
 3 files changed, 7 insertions(+), 353 deletions(-)

diff --git a/hydra/berlin.scm b/hydra/berlin.scm
index ee828d8..529f4f0 100644
--- a/hydra/berlin.scm
+++ b/hydra/berlin.scm
@@ -31,7 +31,7 @@
 		  (ssh-public-key (local-file "keys/ssh/nckx.pub")))))
 
 ;;; XXX: relative includes don't seem to work with Guile 3.0
-(include "/root/maintenance/hydra/nginx/berlin.scm")
+(include "/home/florian/git/maintenance/hydra/nginx/berlin.scm")
 
 \f
 ;;;
@@ -116,29 +116,9 @@ Happy hacking!\n"))
                  ;; The root file system resides on just a single
                  ;; disk, no RAID :-/
                  (file-system
-                   (device (file-system-label "my-root"))
+                   (device "/dev/sda")
                    (mount-point "/")
                    (type "ext4"))
-                 ;; This is a large external storage array
-                 ;; connected via 2 HBA cards.  We only mount it
-                 ;; through one of the HBA cards.  We would need
-                 ;; to use multipathd otherwise.
-                 (file-system
-                   (device (uuid "a6455b66-59d2-40bd-bddb-0c572bb62a2f"))
-                   (mount-point "/gnu")
-                   (type "ext4"))
-                 ;; Bind mount cache to large external storage.
-                 (file-system
-                   (device "/gnu/cache")
-                   (mount-point "/var/cache")
-                   (flags '(bind-mount))
-                   (type "none"))
-                 ;; Access root file system without bind mounts.
-                 (file-system
-                   (device "/")
-                   (mount-point "/mnt/root-fs")
-                   (flags '(bind-mount))
-                   (type "none"))
                  %base-file-systems))
 
   ;; Local admin account for MDC maintenance.
@@ -164,52 +144,12 @@ Happy hacking!\n"))
                              activation-service-type
                              %copy-kernel-and-initrd)
 
-             ;; Connection to the DMZ for public access
-             ;; This is a 10G port.
-             (static-networking-service "enp129s0f0"
-                                        "141.80.181.40"
-                                        #:netmask "255.255.255.0"
-                                        #:gateway "141.80.181.1")
-             ;; Connection to build nodes
-             (static-networking-service "enp129s0f1"
-                                        "141.80.167.131"
-                                        #:netmask "255.255.255.192")
-
+             (service dhcp-client-service-type)
              ;; Allow login over serial console.
              (agetty-service (agetty-configuration
                               (tty "ttyS0")
                               (baud-rate "115200")))
 
-             ;; DNS
-             (service knot-service-type
-                      (knot-configuration
-                        (zones (list (knot-zone-configuration
-                                       (domain "guix.gnu.org")
-                                       (master '("bayfront-master"))
-                                       (acl '("notify-allow")))))
-                        (acls (list (knot-acl-configuration
-                                      (id "notify-allow")
-                                      (address (list bayfront-ip4))
-                                      (action '(notify)))))
-                        (remotes (list (knot-remote-configuration
-                                         (id "bayfront-master")
-                                         (address (list bayfront-ip4)))))))
-
-             ;; Monitoring
-             (service zabbix-agent-service-type)
-             (service zabbix-server-service-type
-                      (zabbix-server-configuration
-                       (include-files '("/root/zabbix-pass"))
-                       (extra-options "AlertScriptsPath=/root/zabbix-alert-scripts\n")))
-             (service zabbix-front-end-service-type
-                      (zabbix-front-end-configuration
-                       (nginx (list %zabbix-nginx-server))
-                       (db-secret-file "/root/zabbix-front-end-secrets")))
-
-             ;; For the Zabbix database.  It was created by manually
-             ;; following the instructions here:
-             ;; https://www.zabbix.com/documentation/3.0/manual/appendix/install/db_scripts
-             (postgresql-service)
 
              (service ntp-service-type)
 
@@ -226,22 +166,10 @@ Happy hacking!\n"))
              (service static-web-site-service-type
                       (static-web-site-configuration
                        (git-url
-                        "https://git.savannah.gnu.org/git/guix/guix-artwork.git")
+                        "git://pelzflorian.de/guix-artwork.git")
                        (directory "/srv/guix.gnu.org")
                        (build-file "website/.guix.scm")))
 
-             ;; 'wip-i18n' branch of guix.gnu.org.
-             (service static-web-site-service-type
-                      (static-web-site-configuration
-                       (git-url
-                        "https://git.savannah.gnu.org/git/guix/guix-artwork.git")
-                       (git-ref '(branch . "wip-i18n"))
-                       (directory "/srv/guix.gnu.org-i18n")
-                       (build-file "website/.guix.scm")
-                       (cache-directory "guix.gnu.org-i18n") ;avoid collision
-                       (environment-variables
-                        '(("GUIX_WEB_SITE_ROOT_PATH" . "/.i18n")))))
-
              ;; Manual for the latest stable release.
              (service static-web-site-service-type
                       (static-web-site-configuration
diff --git a/hydra/modules/sysadmin/services.scm b/hydra/modules/sysadmin/services.scm
index 3323efa..22765cf 100644
--- a/hydra/modules/sysadmin/services.scm
+++ b/hydra/modules/sysadmin/services.scm
@@ -96,20 +96,6 @@
 
 (define* (guix-daemon-config #:key (max-jobs 5) (cores 4))
   (guix-configuration
-   ;; Disable substitutes altogether.
-   (use-substitutes? #f)
-   (substitute-urls '())
-   (authorized-keys '())
-
-   ;; We don't want to let builds get stuck for too long, but we still want
-   ;; to allow building, say, Guile 2.2 on armhf-linux, which takes < 3h on
-   ;; an OverDrive 1000.
-   (max-silent-time 3600)
-   (timeout (* 6 3600))
-
-   (log-compression 'gzip)               ;be friendly to 'guix publish' users
-
-   (build-accounts (* 4 max-jobs))
    (extra-options (list "--max-jobs" (number->string max-jobs)
                         "--cores" (number->string cores)
                         "--cache-failures"
@@ -277,30 +263,10 @@
   (cons* (service rottlog-service-type (rottlog-configuration))
          (service mcron-service-type
                   (mcron-configuration
-                   (jobs (cons %certbot-job %gc-jobs))))
-
-         firewall-service
-
-         ;; The Web service.
-         (service guix-publish-service-type
-                  (guix-publish-configuration
-                   (port 3000)
-                   (cache "/var/cache/guix/publish")
-                   (ttl nar-ttl)
-                   (compression '(("gzip" 9) ("lzip" 9)))
-                   (workers publish-workers)))
+                   (jobs (list))))
 
          %nginx-mime-types
          %nginx-cache-activation
-
-         (service cuirass-service-type
-                  (cuirass-configuration
-                   (cuirass cuirass-without-fiber-tests)
-                   (interval (* 5 60))
-                   (ttl (quotient nar-ttl 2))
-                   (specifications (cuirass-specs systems))))
-
-         (service openssh-service-type)
          (service sysadmin-service-type sysadmins)
 
          (append (if nginx-config-file
@@ -309,9 +275,6 @@
                                      (file nginx-config-file))))
                      '())
                  (modify-services %base-services
-                   (guix-service-type
-                    config => (guix-daemon-config #:max-jobs max-jobs
-                                                  #:cores cores))
                    (login-service-type
                     config => (login-configuration
                                (inherit config)
diff --git a/hydra/nginx/berlin.scm b/hydra/nginx/berlin.scm
index 303fd35..954b47f 100644
--- a/hydra/nginx/berlin.scm
+++ b/hydra/nginx/berlin.scm
@@ -157,27 +157,6 @@
 PUBLISH-URL."
   (append (publish-locations publish-url)
           (list
-           ;; Cuirass.
-           (nginx-location-configuration
-            (uri "/")
-            (body (list "proxy_pass http://localhost:8081;")))
-           (nginx-location-configuration
-            (uri "~ ^/admin")
-            (body
-             (list "if ($ssl_client_verify != SUCCESS) { return 403; } proxy_pass http://localhost:8081;")))
-
-           (nginx-location-configuration
-            (uri "/static")
-            (body
-             (list
-              "proxy_pass http://localhost:8081;"
-              ;; Let browsers cache this for a while.
-              "expires 10d;"
-              ;; Cache quite aggressively.
-              "proxy_cache static;"
-              "proxy_cache_valid 200 5d;"
-              "proxy_cache_valid any 10m;"
-              "proxy_ignore_client_abort on;")))
 
            (nginx-location-configuration          ;certbot
             (uri "/.well-known")
@@ -485,27 +464,6 @@ PUBLISH-URL."
 (define %berlin-servers
   (list
    ;; Plain HTTP
-   (nginx-server-configuration
-    (listen '("80"))
-    (server-name '("berlin.guixsd.org"
-                   "ci.guix.info"
-                   "ci.guix.gnu.org"))
-    (locations (berlin-locations %publish-url))
-    (raw-content
-     (list
-      "access_log  /var/log/nginx/http.access.log;"
-      "proxy_set_header X-Forwarded-Host $host;"
-      "proxy_set_header X-Forwarded-Port $server_port;"
-      "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;")))
-
-   (nginx-server-configuration
-    (listen '("80"))
-    (server-name '("bootstrappable.org"
-                   "www.bootstrappable.org"))
-    (root "/home/rekado/bootstrappable.org")
-    (raw-content
-     (list
-      "access_log /var/log/nginx/bootstrappable.access.log;")))
 
    (nginx-server-configuration
     (listen '("80"))
@@ -515,203 +473,7 @@ PUBLISH-URL."
     (raw-content
      (list
       "access_log /var/log/nginx/guix-info.access.log;")))
-
-   (nginx-server-configuration
-    (listen '("80"))
-    (server-name '("guix.info"
-                   "www.guix.info"))
-    (locations guix.info-locations)
-    (raw-content
-     (append
-      %tls-settings
-      (list
-       "access_log /var/log/nginx/guix-info.https.access.log;"))))
-
-   (nginx-server-configuration
-    (listen '("80"))
-    (server-name '("issues.guix.info"
-                   "issues.guix.gnu.org"))
-    (root "/home/rekado/mumi/")
-    (locations
-     (list (nginx-location-configuration ;certbot
-            (uri "/.well-known")
-            (body (list "root /var/www;")))
-           (nginx-location-configuration
-            (uri "/")
-            (body '("proxy_pass http://localhost:1234;")))))
-    (raw-content
-     (list
-      "access_log /var/log/nginx/issues-guix-info.access.log;")))
-
-   (nginx-server-configuration
-    (listen '("80"))
-    (server-name '("workflows.guix.info"
-                   "workflow.guix.info"
-                   "guixwl.org"
-                   "www.guixwl.org"))
-    (root "/home/rekado/gwl/")
-    (locations
-     (list (nginx-location-configuration ;certbot
-            (uri "/.well-known")
-            (body (list "root /var/www;")))
-
-           (nginx-location-configuration
-            (uri "/manual")
-            (body (list "alias /srv/gwl-manual;")))
-
-           ;; Pass requests to 'guix workflow --web-interface'.
-           (nginx-location-configuration
-            (uri "/")
-            (body '("proxy_pass http://localhost:5000;")))))
-    (raw-content
-     (list
-      "access_log /var/log/nginx/workflows-guix-info.access.log;")))
-        
-   ;; HTTPS servers
-   (nginx-server-configuration
-    (listen '("443 ssl"))
-    (server-name '("berlin.guixsd.org"
-                   "ci.guix.info"
-                   "ci.guix.gnu.org"))
-    (ssl-certificate (le "berlin.guixsd.org"))
-    (ssl-certificate-key (le "berlin.guixsd.org" 'key))
-    (locations (berlin-locations %publish-url))
-    (raw-content
-     (append
-      %tls-settings
-      (list
-       "access_log  /var/log/nginx/https.access.log;"
-       "proxy_set_header X-Forwarded-Host $host;"
-       "proxy_set_header X-Forwarded-Port $server_port;"
-       "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"
-       ;; For Cuirass admin interface authentication
-       "ssl_client_certificate /etc/ssl-ca/certs/ca.crt;"
-       "ssl_crl /etc/ssl-ca/private/ca.crl;"
-       "ssl_verify_client optional;"))))
-
-   (nginx-server-configuration
-    (listen '("443 ssl"))
-    (server-name '("qualif.ci.guix.gnu.org"))
-    (locations (berlin-locations "http://localhost:3003"))
-    (raw-content
-     (append %tls-settings
-             '("access_log  /var/log/nginx/qualif.access.log;"))))
-
-   (nginx-server-configuration
-    (listen '("443 ssl"))
-    (server-name '("bootstrappable.org"
-                   "www.bootstrappable.org"))
-    (ssl-certificate (le "bootstrappable.org"))
-    (ssl-certificate-key (le "bootstrappable.org" 'key))
-    (root "/home/rekado/bootstrappable.org")
-    (raw-content
-     (append
-      %tls-settings
-      (list
-       "access_log /var/log/nginx/bootstrappable.https.access.log;"))))
-
-   (nginx-server-configuration
-    (listen '("443 ssl"))
-    (server-name '("guix.info"
-                   "www.guix.info"))
-    (ssl-certificate (le "guix.info"))
-    (ssl-certificate-key (le "guix.info" 'key))
-    (locations guix.info-locations)
-    (raw-content
-     (append
-      %tls-settings
-      (list
-       "access_log /var/log/nginx/guix-info.https.access.log;"))))
-
-   (nginx-server-configuration
-    (listen '("443 ssl"))
-    (server-name '("guix.gnu.org"))
-    (ssl-certificate (le "guix.gnu.org"))
-    (ssl-certificate-key (le "guix.gnu.org" 'key))
-    (root "/srv/guix.gnu.org")
-    (locations guix.gnu.org-locations)
-    (raw-content
-     (append
-      %tls-settings
-      (list
-       "access_log /var/log/nginx/guix-gnu-org.https.access.log;"))))
-
-   (nginx-server-configuration
-    (listen '("443 ssl"))
-    (server-name '("issues.guix.info"))
-    (ssl-certificate (le "issues.guix.info"))
-    (ssl-certificate-key (le "issues.guix.info" 'key))
-    (root "/home/rekado/mumi/")
-    (locations
-     (list (nginx-location-configuration
-            (uri "/")
-            (body '("proxy_pass http://localhost:1234;")))))
-    (raw-content
-     (append
-      %tls-settings
-      (list
-       "proxy_set_header X-Forwarded-Host $host;"
-       "proxy_set_header X-Forwarded-Port $server_port;"
-       "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"
-       "proxy_connect_timeout       600;"
-       "proxy_send_timeout          600;"
-       "proxy_read_timeout          600;"
-       "send_timeout                600;"
-       "access_log /var/log/nginx/issues-guix-info.https.access.log;"))))
-
-   (nginx-server-configuration
-    (listen '("443 ssl"))
-    (server-name '("issues.guix.gnu.org"))
-    (ssl-certificate (le "issues.guix.gnu.org"))
-    (ssl-certificate-key (le "issues.guix.gnu.org" 'key))
-    (root "/home/rekado/mumi/")
-    (locations
-     (list (nginx-location-configuration
-            (uri "/")
-            (body '("proxy_pass http://localhost:1234;")))))
-    (raw-content
-     (append
-      %tls-settings
-      (list
-       "proxy_set_header X-Forwarded-Host $host;"
-       "proxy_set_header X-Forwarded-Port $server_port;"
-       "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"
-       "proxy_connect_timeout       600;"
-       "proxy_send_timeout          600;"
-       "proxy_read_timeout          600;"
-       "send_timeout                600;"
-       "access_log /var/log/nginx/issues-guix-gnu-org.https.access.log;"))))
-
-   (nginx-server-configuration
-    (listen '("443 ssl"))
-    (server-name '("workflows.guix.info"
-                   "workflow.guix.info"
-                   "guixwl.org"
-                   "www.guixwl.org"))
-    (ssl-certificate (le "www.guixwl.org"))
-    (ssl-certificate-key (le "www.guixwl.org" 'key))
-    (root "/home/rekado/gwl/")
-    (locations
-     (list
-      (nginx-location-configuration
-       (uri "/manual")
-       (body (list "alias /srv/gwl-manual;")))
-      (nginx-location-configuration
-       (uri "/")
-       (body '("proxy_pass http://localhost:5000;")))))
-    (raw-content
-     (append
-      %tls-settings
-      (list
-       "proxy_set_header X-Forwarded-Host $host;"
-       "proxy_set_header X-Forwarded-Port $server_port;"
-       "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"
-       "proxy_connect_timeout       600;"
-       "proxy_send_timeout          600;"
-       "proxy_read_timeout          600;"
-       "send_timeout                600;"
-       "access_log /var/log/nginx/workflows-guix-info.https.access.log;"))))))
-
+))
 (define %extra-content
   (list
    "default_type  application/octet-stream;"
@@ -774,6 +536,7 @@ PUBLISH-URL."
 
 (define %nginx-configuration
   (nginx-configuration
+   (server-names-hash-bucket-size 64)
    (server-blocks %berlin-servers)
    (global-directives
     ;; This is a 72-core machine, but let's not use all of them for nginx.
-- 
2.26.0


[-- Attachment #3: 0002-remove-rottlog.patch --]
[-- Type: text/plain, Size: 909 bytes --]

From 8b8f7a404b29ab3341b7dd19dcd1c5415723c1a0 Mon Sep 17 00:00:00 2001
From: Florian Pelz <pelzflorian@pelzflorian.de>
Date: Wed, 8 Apr 2020 21:08:57 +0200
Subject: [PATCH 2/2] remove rottlog

---
 hydra/modules/sysadmin/services.scm | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/hydra/modules/sysadmin/services.scm b/hydra/modules/sysadmin/services.scm
index 22765cf..fe023ad 100644
--- a/hydra/modules/sysadmin/services.scm
+++ b/hydra/modules/sysadmin/services.scm
@@ -260,8 +260,7 @@
                             (nar-ttl (* 90 24 3600))
                             (publish-workers 6))
   "Return the list of services for the build farm frontend."
-  (cons* (service rottlog-service-type (rottlog-configuration))
-         (service mcron-service-type
+  (cons* (service mcron-service-type
                   (mcron-configuration
                    (jobs (list))))
 
-- 
2.26.0