[PATCH v2] test: replace notmuch_passwd_sanitize() with _libconfig_sanitize()

[PATCH v2] test: replace notmuch_passwd_sanitize() with _libconfig_sanitize()

[Tomi Ollila]

notmuch_passwd_sanitize() in test-lib.sh is too generic, it cannot
work in many cases...

The more specific version _libconfig_sanitize() replaces it in
T590-libconfig.sh and the code that uses it is modified to output
the keys (ascending numbers printed in hex) so the sanitizer knows
what to sanitize in which lines...

In addition to ".(none)" now also ".localdomain" if filtered from
USERNAME@FQDN.
---

Rebase of id:20210502181535.31292-1-tomi.ollila@iki.fi

* 'fn () {' style change
* added @ to fqdn replace, as it could be subset of user

 test/T590-libconfig.sh | 97 +++++++++++++++++++++++++-----------------
 test/test-lib.sh       | 20 ---------
 2 files changed, 59 insertions(+), 58 deletions(-)

diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh
index 745e1bb4..8e70b7b2 100755
--- a/test/T590-libconfig.sh
+++ b/test/T590-libconfig.sh
@@ -5,6 +5,26 @@ test_description="library config API"
 
 add_email_corpus
 
+_libconfig_sanitize() {
+    ${NOTMUCH_PYTHON} -c '
+import os, sys, pwd, socket
+
+pw = pwd.getpwuid(os.getuid())
+user = pw.pw_name
+name = pw.pw_gecos.partition(",")[0]
+fqdn = socket.getaddrinfo(socket.gethostname(), 0, 0,
+                          socket.SOCK_STREAM, 0, socket.AI_CANONNAME)[0][3]
+for l in sys.stdin:
+    if l[:3] == "8: ":
+        # fqdn could be subset of user; therefore @
+        l = l.replace(user, "USERNAME").replace("@" + fqdn, "@FQDN")
+        l = l.replace(".(none)", "").replace(".localdomain", "")
+    elif l[:3] == "a: ":
+        l = l.replace(name, "USER_FULL_NAME")
+    sys.stdout.write(l)
+'
+}
+
 cat <<EOF > c_head
 #include <string.h>
 #include <stdlib.h>
@@ -380,26 +400,26 @@ cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} '' %NULL%
 	 key < NOTMUCH_CONFIG_LAST;
 	 key = (notmuch_config_key_t)(key + 1)) {
 	const char *val = notmuch_config_get (db, key);
-        printf("%s\n", val ? val : "NULL" );
+	printf("%x: '%s'\n", key, val ? val : "NULL" );
     }
 }
 EOF
 
-notmuch_passwd_sanitize < OUTPUT > OUTPUT.clean
+_libconfig_sanitize < OUTPUT > OUTPUT.clean
 
 cat <<'EOF' >EXPECTED
 == stdout ==
-MAIL_DIR
-MAIL_DIR
-MAIL_DIR/.notmuch/hooks
-MAIL_DIR/.notmuch/backups
-
-unread;inbox
-
-true
-USERNAME@FQDN
-NULL
-USER_FULL_NAME
+0: 'MAIL_DIR'
+1: 'MAIL_DIR'
+2: 'MAIL_DIR/.notmuch/hooks'
+3: 'MAIL_DIR/.notmuch/backups'
+4: ''
+5: 'unread;inbox'
+6: ''
+7: 'true'
+8: 'USERNAME@FQDN'
+9: 'NULL'
+a: 'USER_FULL_NAME'
 == stderr ==
 EOF
 unset MAILDIR
@@ -694,23 +714,23 @@ cat c_head2 - c_tail <<'EOF' | test_C ${MAIL_DIR} %NULL% %NULL%
 	 key < NOTMUCH_CONFIG_LAST;
 	 key = (notmuch_config_key_t)(key + 1)) {
 	const char *val = notmuch_config_get (db, key);
-        printf("%s\n", val ? val : "NULL" );
+	printf("%x: '%s'\n", key, val ? val : "NULL" );
     }
 }
 EOF
 cat <<'EOF' >EXPECTED
 == stdout ==
-MAIL_DIR
-MAIL_DIR
-MAIL_DIR/.notmuch/hooks
-MAIL_DIR/.notmuch/backups
-foo;bar;fub
-unread;inbox
-sekrit_junk
-true
-test_suite@notmuchmail.org
-test_suite_other@notmuchmail.org;test_suite@otherdomain.org
-Notmuch Test Suite
+0: 'MAIL_DIR'
+1: 'MAIL_DIR'
+2: 'MAIL_DIR/.notmuch/hooks'
+3: 'MAIL_DIR/.notmuch/backups'
+4: 'foo;bar;fub'
+5: 'unread;inbox'
+6: 'sekrit_junk'
+7: 'true'
+8: 'test_suite@notmuchmail.org'
+9: 'test_suite_other@notmuchmail.org;test_suite@otherdomain.org'
+a: 'Notmuch Test Suite'
 == stderr ==
 EOF
 test_expect_equal_file EXPECTED OUTPUT
@@ -723,25 +743,26 @@ cat c_head2 - c_tail <<'EOF' | test_C ${MAIL_DIR} /nonexistent %NULL%
 	 key < NOTMUCH_CONFIG_LAST;
 	 key = (notmuch_config_key_t)(key + 1)) {
 	const char *val = notmuch_config_get (db, key);
-	printf("%s\n", val ? val : "NULL" );
+	printf("%x: '%s'\n", key, val ? val : "NULL" );
     }
 }
 EOF
 
-notmuch_passwd_sanitize < OUTPUT > OUTPUT.clean
+_libconfig_sanitize < OUTPUT > OUTPUT.clean
+
 cat <<'EOF' >EXPECTED
 == stdout ==
-MAIL_DIR
-MAIL_DIR
-MAIL_DIR/.notmuch/hooks
-MAIL_DIR/.notmuch/backups
-
-unread;inbox
-
-true
-USERNAME@FQDN
-NULL
-USER_FULL_NAME
+0: 'MAIL_DIR'
+1: 'MAIL_DIR'
+2: 'MAIL_DIR/.notmuch/hooks'
+3: 'MAIL_DIR/.notmuch/backups'
+4: ''
+5: 'unread;inbox'
+6: ''
+7: 'true'
+8: 'USERNAME@FQDN'
+9: 'NULL'
+a: 'USER_FULL_NAME'
 == stderr ==
 EOF
 test_expect_equal_file EXPECTED OUTPUT.clean
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 89eb744d..c379d140 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -564,26 +564,6 @@ notmuch_built_with_sanitize () {
     sed 's/^built_with[.]\(.*\)=.*$/built_with.\1=something/'
 }
 
-notmuch_passwd_sanitize () {
-    ${NOTMUCH_PYTHON} -c'
-import os, sys, pwd, socket
-
-pw = pwd.getpwuid(os.getuid())
-user = pw.pw_name
-name = pw.pw_gecos.partition(",")[0]
-fqdn = socket.getaddrinfo(socket.gethostname(), 0, 0, socket.SOCK_STREAM, 0, socket.AI_CANONNAME)[0][3]
-
-for l in sys.stdin:
-    if user:
-        l = l.replace(user, "USERNAME")
-    if fqdn:
-        l = l.replace(fqdn, "FQDN").replace(".(none)","")
-    if name:
-        l = l.replace(name, "USER_FULL_NAME")
-    sys.stdout.write(l)
-'
-}
-
 notmuch_config_sanitize () {
     notmuch_dir_sanitize | notmuch_built_with_sanitize
 }
-- 
2.31.1

Re: [PATCH v2] test: replace notmuch_passwd_sanitize() with _libconfig_sanitize()

[Felipe Contreras]

On Mon, May 17, 2021 at 2:33 PM Tomi Ollila <tomi.ollila@iki.fi> wrote:
>
> notmuch_passwd_sanitize() in test-lib.sh is too generic, it cannot
> work in many cases...
>
> The more specific version _libconfig_sanitize() replaces it in
> T590-libconfig.sh and the code that uses it is modified to output
> the keys (ascending numbers printed in hex) so the sanitizer knows
> what to sanitize in which lines...
>
> In addition to ".(none)" now also ".localdomain" if filtered from
> USERNAME@FQDN.
> ---
>
> Rebase of id:20210502181535.31292-1-tomi.ollila@iki.fi
>
> * 'fn () {' style change
> * added @ to fqdn replace, as it could be subset of user
>
>  test/T590-libconfig.sh | 97 +++++++++++++++++++++++++-----------------
>  test/test-lib.sh       | 20 ---------
>  2 files changed, 59 insertions(+), 58 deletions(-)
>
> diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh
> index 745e1bb4..8e70b7b2 100755
> --- a/test/T590-libconfig.sh
> +++ b/test/T590-libconfig.sh
> @@ -5,6 +5,26 @@ test_description="library config API"
>
>  add_email_corpus
>
> +_libconfig_sanitize() {
> +    ${NOTMUCH_PYTHON} -c '
> +import os, sys, pwd, socket
> +
> +pw = pwd.getpwuid(os.getuid())
> +user = pw.pw_name
> +name = pw.pw_gecos.partition(",")[0]
> +fqdn = socket.getaddrinfo(socket.gethostname(), 0, 0,
> +                          socket.SOCK_STREAM, 0, socket.AI_CANONNAME)[0][3]
> +for l in sys.stdin:
> +    if l[:3] == "8: ":
> +        # fqdn could be subset of user; therefore @
> +        l = l.replace(user, "USERNAME").replace("@" + fqdn, "@FQDN")
> +        l = l.replace(".(none)", "").replace(".localdomain", "")
> +    elif l[:3] == "a: ":
> +        l = l.replace(name, "USER_FULL_NAME")

This generates garbage when name is null. See ff5f141b (test: fix
passwd_sanitize(), 2021-05-01).

At the very least we need to check for that, even better if we
generate the proper output:

        if name:
            l = l.replace(name, "USER_FULL_NAME")
        else:
            l = "a: \'USER_FULL_NAME\'\\n"

This makes the test pass with no geckos name, but we need to use bash
strings: $'string'. Otherwise we would need to do something much more
weird.

Cheers.

-- 
Felipe Contreras

Re: [PATCH v2] test: replace notmuch_passwd_sanitize() with _libconfig_sanitize()

[Tomi Ollila]

On Mon, May 17 2021, Felipe Contreras wrote:

> On Mon, May 17, 2021 at 2:33 PM Tomi Ollila <tomi.ollila@iki.fi> wrote:
>>
>> notmuch_passwd_sanitize() in test-lib.sh is too generic, it cannot
>> work in many cases...
>>
>> The more specific version _libconfig_sanitize() replaces it in
>> T590-libconfig.sh and the code that uses it is modified to output
>> the keys (ascending numbers printed in hex) so the sanitizer knows
>> what to sanitize in which lines...
>>
>> In addition to ".(none)" now also ".localdomain" if filtered from
>> USERNAME@FQDN.
>> ---
>>
>> Rebase of id:20210502181535.31292-1-tomi.ollila@iki.fi
>>
>> * 'fn () {' style change
>> * added @ to fqdn replace, as it could be subset of user
>>
>>  test/T590-libconfig.sh | 97 +++++++++++++++++++++++++-----------------
>>  test/test-lib.sh       | 20 ---------
>>  2 files changed, 59 insertions(+), 58 deletions(-)
>>
>> diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh
>> index 745e1bb4..8e70b7b2 100755
>> --- a/test/T590-libconfig.sh
>> +++ b/test/T590-libconfig.sh
>> @@ -5,6 +5,26 @@ test_description="library config API"
>>
>>  add_email_corpus
>>
>> +_libconfig_sanitize() {
>> +    ${NOTMUCH_PYTHON} -c '
>> +import os, sys, pwd, socket
>> +
>> +pw = pwd.getpwuid(os.getuid())
>> +user = pw.pw_name
>> +name = pw.pw_gecos.partition(",")[0]
>> +fqdn = socket.getaddrinfo(socket.gethostname(), 0, 0,
>> +                          socket.SOCK_STREAM, 0, socket.AI_CANONNAME)[0][3]
>> +for l in sys.stdin:
>> +    if l[:3] == "8: ":
>> +        # fqdn could be subset of user; therefore @
>> +        l = l.replace(user, "USERNAME").replace("@" + fqdn, "@FQDN")
>> +        l = l.replace(".(none)", "").replace(".localdomain", "")
>> +    elif l[:3] == "a: ":
>> +        l = l.replace(name, "USER_FULL_NAME")
>
> This generates garbage when name is null. See ff5f141b (test: fix
> passwd_sanitize(), 2021-05-01).

You're right -- that was the reason I added ''s around,
just forgot that part :/

Anyway, there are other problems, will fix that along...

Thanks,

Tomi

>
> At the very least we need to check for that, even better if we
> generate the proper output:
>
>         if name:
>             l = l.replace(name, "USER_FULL_NAME")
>         else:
>             l = "a: \'USER_FULL_NAME\'\\n"

>
> This makes the test pass with no geckos name, but we need to use bash
> strings: $'string'. Otherwise we would need to do something much more
> weird.
>
> Cheers.
>
> -- 
> Felipe Contreras