unofficial mirror of meta@public-inbox.org
 help / color / mirror / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 2/2] www: use ->ALL for per-inbox manifest.js.gz, too
Date: Sat, 11 Sep 2021 23:30:46 +0000	[thread overview]
Message-ID: <20210911233046.23862-3-e@80x24.org> (raw)
In-Reply-To: <20210911233046.23862-1-e@80x24.org>

With 11 epochs on LKML, it's the 11 current epoch time goes from
around 60ms to around 10ms, so it's a significant improvement.
And improve test coverage while we're at it.
---
 lib/PublicInbox/ManifestJsGz.pm | 14 +++++++-------
 t/extindex-psgi.t               | 20 ++++++++++++++++++--
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index cde60245..d5048a96 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -53,6 +53,7 @@ sub slow_manifest_add ($$) {
 			manifest_add($ctx, $ibx);
 		}
 	};
+	warn "E: $@" if $@;
 }
 
 sub eidx_manifest_add ($$$) {
@@ -65,6 +66,10 @@ sub eidx_manifest_add ($$$) {
 		}
 	} else {
 		warn "E: `${\$ibx->eidx_key}' not indexed by $ALL->{topdir}\n";
+		# do not use slow path for global manifest since
+		# it can become catastrophically slow.  per-inbox manifest
+		# is not too bad with dozens of epochs, so never fail that:
+		slow_manifest_add($ctx, $ibx) if $ibx == $ctx->{ibx};
 	}
 }
 
@@ -85,12 +90,8 @@ sub response {
 sub ibx_entry {
 	my ($ctx, $ibx) = @_;
 	my $ALL = $ctx->{www}->{pi_cfg}->ALL;
-	if ($ALL) { # FIXME: test this in t/
-		eidx_manifest_add($ctx, $ALL, $ibx);
-	} else {
+	$ALL ? eidx_manifest_add($ctx, $ALL, $ibx) :
 		slow_manifest_add($ctx, $ibx);
-		warn "E: $@" if $@;
-	}
 }
 
 sub hide_key { 'manifest' } # for WwwListing->list_match_i
@@ -112,8 +113,7 @@ sub psgi_triple {
 
 sub per_inbox {
 	my ($ctx) = @_;
-	# only one inbox, slow is probably OK
-	slow_manifest_add($ctx, $ctx->{ibx});
+	ibx_entry($ctx, $ctx->{ibx});
 	psgi_triple($ctx);
 }
 
diff --git a/t/extindex-psgi.t b/t/extindex-psgi.t
index 4e26962e..98dc2e48 100644
--- a/t/extindex-psgi.t
+++ b/t/extindex-psgi.t
@@ -11,6 +11,7 @@ require_git(2.6);
 require_mods(qw(json DBD::SQLite Search::Xapian
 		HTTP::Request::Common Plack::Test URI::Escape Plack::Builder));
 use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
+use IO::Uncompress::Gunzip qw(gunzip);
 require PublicInbox::WWW;
 my ($ro_home, $cfg_path) = setup_public_inboxes;
 my ($tmpdir, $for_destroy) = tmpdir;
@@ -18,11 +19,11 @@ my $home = "$tmpdir/home";
 mkdir $home or BAIL_OUT $!;
 mkdir "$home/.public-inbox" or BAIL_OUT $!;
 my $pi_config = "$home/.public-inbox/config";
-cp("$ro_home/.public-inbox/config", $pi_config) or BAIL_OUT;
+cp($cfg_path, $pi_config) or BAIL_OUT;
 my $env = { HOME => $home };
 run_script([qw(-extindex --all), "$tmpdir/eidx"], $env) or BAIL_OUT;
 {
-	open my $cfgfh, '>', $pi_config or BAIL_OUT;
+	open my $cfgfh, '>>', $pi_config or BAIL_OUT;
 	$cfgfh->autoflush(1);
 	print $cfgfh <<EOM or BAIL_OUT;
 [extindex "all"]
@@ -30,6 +31,7 @@ run_script([qw(-extindex --all), "$tmpdir/eidx"], $env) or BAIL_OUT;
 	url = http://bogus.example.com/all
 [publicinbox]
 	wwwlisting = all
+	grokManifest = all
 EOM
 }
 my $www = PublicInbox::WWW->new(PublicInbox::Config->new($pi_config));
@@ -67,6 +69,20 @@ my $client = sub {
 	is($res->code, 404, 'no inboxes matched');
 	unlike($res->content, qr!no inboxes, yet!,
 		'we have inboxes, just no matches');
+
+	my $m = {};
+	for my $pfx (qw(/t1 /t2), '') {
+		$res = $cb->(GET($pfx.'/manifest.js.gz'));
+		gunzip(\($res->content) => \(my $js));
+		$m->{$pfx} = json_utf8->decode($js);
+	}
+	is_deeply([sort keys %{$m->{''}}],
+		[ sort(keys %{$m->{'/t1'}}, keys %{$m->{'/t2'}}) ],
+		't1 + t2 = all');
+	is_deeply([ sort keys %{$m->{'/t2'}} ], [ '/t2/git/0.git' ],
+		't2 manifest');
+	is_deeply([ sort keys %{$m->{'/t1'}} ], [ '/t1' ],
+		't2 manifest');
 };
 test_psgi(sub { $www->call(@_) }, $client);
 %$env = (%$env, TMPDIR => $tmpdir, PI_CONFIG => $pi_config);

      parent reply	other threads:[~2021-09-11 23:30 UTC|newest]

Thread overview: 3+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-09-11 23:30 [PATCH 0/2] www: per-inbox manifest.js.gz improvements Eric Wong
2021-09-11 23:30 ` [PATCH 1/2] manifest.js.gz: avoid long-lived per-epoch cat-file processes Eric Wong
2021-09-11 23:30 ` Eric Wong [this message]

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210911233046.23862-3-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).