[PATCH 00/18] WWW: patch, tree, git glossary

[PATCH 00/18] WWW: patch, tree, git glossary

[Eric Wong]

Raw format-patch and tree HTML output now supported for git
output.  I suppose tags can be displayed, too, at some point...

One thing I'm not 100% sure about is adding a git-related
glossary for stuff like trees, commits, etc...  It seems
to bloat the page a bit, but it could be useful in slowly
teaching basic git data concepts to beginners.

I suspect folks who have trouble learning git too focused on the
commands rather than the data concepts.  (IMHO, the same goes
for learning projects based on studying code vs studying
(DB schemas || struct layouts)).

I snuck one speedup in there, hopefully more to come...

Eric Wong (18):
  solver: create tmpdir lazily
  viewvcs: share File::Temp::Dir with solver
  viewvcs: delay stringification of solver debug log
  www: allow html_oneshot to take an array arg
  viewvcs: use array for highlighted blob display
  viewvcs: add patch download link for single-parent commits
  viewvcs: author date links to contemporary messages
  view: speed up /$INBOX/ landing page by 0.5-1.0%
  treewide: ditch inbox->recent method
  view: /$INBOX/: show "messages from $old to $new"
  view: cleanups and reuse for {obuf} preparation
  www: atom: fix "changed" href to nowhere
  www: provide text/help/#search anchor
  solver: early make hints detection more robust
  viewvcs: add tree view
  viewvcs: reduce hash assignments for commit info
  viewvcs: add glossary for commit
  viewvcs: show "blob $OID" rather than "$OID blob"

 lib/PublicInbox/ExtSearch.pm      |   1 -
 lib/PublicInbox/Inbox.pm          |   5 -
 lib/PublicInbox/LeiSavedSearch.pm |   1 -
 lib/PublicInbox/LeiXSearch.pm     |   7 -
 lib/PublicInbox/SolverGit.pm      |  22 +-
 lib/PublicInbox/View.pm           | 101 ++++-----
 lib/PublicInbox/ViewDiff.pm       |  18 +-
 lib/PublicInbox/ViewVCS.pm        | 341 ++++++++++++++++++++----------
 lib/PublicInbox/WWW.pm            |   2 +-
 lib/PublicInbox/WwwAltId.pm       |   6 +-
 lib/PublicInbox/WwwAtomStream.pm  |   1 +
 lib/PublicInbox/WwwStream.pm      |   7 +-
 lib/PublicInbox/WwwText.pm        |   3 +-
 t/convert-compact.t               |   2 +-
 t/indexlevels-mirror.t            |  10 +-
 t/lei_xsearch.t                   |   2 +-
 t/plack.t                         |   2 +-
 t/replace.t                       |   4 +-
 t/solver_git.t                    |   3 +-
 t/v1-add-remove-add.t             |   2 +-
 t/v2-add-remove-add.t             |   2 +-
 21 files changed, 333 insertions(+), 209 deletions(-)

[PATCH 01/18] solver: create tmpdir lazily

[Eric Wong]

"lei blob" doesn't currently need it at all in some cases, and
the next commit will allow viewvcs to share tmpdirs to show
commits as HTML.
---
 lib/PublicInbox/SolverGit.pm | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index dbb9b739..52dfaf3d 100644
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -106,6 +106,11 @@ sub solve_existing ($$) {
 	scalar(@$try);
 }
 
+sub _tmp {
+	$_[0]->{tmp} //=
+		File::Temp->newdir("solver.$_[0]->{oid_want}-XXXX", TMPDIR => 1);
+}
+
 sub extract_diff ($$) {
 	my ($p, $arg) = @_;
 	my ($self, $want, $smsg) = @$arg;
@@ -193,8 +198,8 @@ sub extract_diff ($$) {
 
 	my $path = ++$self->{tot};
 	$di->{n} = $path;
-	open(my $tmp, '>:utf8', $self->{tmp}->dirname . "/$path") or
-		die "open(tmp): $!";
+	my $f = _tmp($self)->dirname."/$path";
+	open(my $tmp, '>:utf8', $f) or die "open($f): $!";
 	print $tmp $di->{hdr_lines}, $patch or die "print(tmp): $!";
 	close $tmp or die "close(tmp): $!";
 
@@ -284,8 +289,7 @@ sub prepare_index ($) {
 # pure Perl "git init"
 sub do_git_init ($) {
 	my ($self) = @_;
-	my $dir = $self->{tmp}->dirname;
-	my $git_dir = "$dir/git";
+	my $git_dir = _tmp($self)->dirname.'/git';
 
 	foreach ('', qw(objects refs objects/info refs/heads)) {
 		mkdir("$git_dir/$_") or die "mkdir $_: $!";
@@ -478,7 +482,6 @@ sub apply_result ($$) {
 
 sub do_git_apply ($) {
 	my ($self) = @_;
-	my $dn = $self->{tmp}->dirname;
 	my $patches = $self->{patches};
 
 	# we need --ignore-whitespace because some patches are CRLF
@@ -501,7 +504,7 @@ sub do_git_apply ($) {
 	} while (@$patches && $len < $ARG_SIZE_MAX &&
 		 !oids_same_ish($patches->[0]->{oid_b}, $prv_oid_b));
 
-	my $opt = { 2 => 1, -C => $dn, quiet => 1 };
+	my $opt = { 2 => 1, -C => _tmp($self)->dirname, quiet => 1 };
 	my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env}, $opt);
 	$self->{-cur_di} = $di;
 	$qsp->{qsp_err} = \($self->{-qsp_err} = '');
@@ -683,7 +686,6 @@ sub solve ($$$$$) {
 	$self->{todo} = [ { %$hints, oid_b => $oid_want } ];
 	$self->{patches} = []; # [ $di, $di, ... ]
 	$self->{found} = {}; # { abbr => [ ::Git, oid, type, size, $di ] }
-	$self->{tmp} = File::Temp->newdir("solver.$oid_want-XXXX", TMPDIR => 1);
 
 	dbg($self, "solving $oid_want ...");
 	if (my $async = $env->{'pi-httpd.async'}) {

[PATCH 02/18] viewvcs: share File::Temp::Dir with solver

[Eric Wong]

This allows reusing inodes for /$COMMIT_OID/s/ requests.
We'll also replace `log' with `lh' in the field name to
avoid confusion with the `log' perlop.
---
 lib/PublicInbox/ViewVCS.pm | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 9bf010c2..c5d16478 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -231,12 +231,10 @@ sub show_commit ($$$$) {
 		'git show --encoding=UTF-8 --pretty=format:%n -M'.
 		" --stat -p $oid >p && ".
 		"git patch-id --stable <p" ];
-	my $xenv = { GIT_DIR => $git->{git_dir} };
-	my $tmp = File::Temp->newdir("show-$oid-XXXX", TMPDIR => 1);
-	my $qsp = PublicInbox::Qspawn->new($cmd, $xenv, { -C => "$tmp" });
+	my $e = { GIT_DIR => $git->{git_dir} };
+	my $qsp = PublicInbox::Qspawn->new($cmd, $e, { -C => "$ctx->{-tmp}" });
 	$qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
 	$ctx->{-logref} = $logref;
-	$ctx->{-tmp} = $tmp;
 	$ctx->{env}->{'qspawn.wcb'} = delete $ctx->{-wcb};
 	$ctx->{git} = $git;
 	$qsp->psgi_qx($ctx->{env}, undef, \&show_commit_start, $ctx);
@@ -260,7 +258,7 @@ sub show_other ($$$$) {
 # user_cb for SolverGit, called as: user_cb->($result_or_error, $uarg)
 sub solve_result {
 	my ($res, $ctx) = @_;
-	my ($log, $hints, $fn) = delete @$ctx{qw(log hints fn)};
+	my ($log, $hints, $fn) = delete @$ctx{qw(lh hints fn)};
 
 	unless (seek($log, 0, 0)) {
 		warn "seek(log): $!";
@@ -345,15 +343,16 @@ sub show ($$;$) {
 		defined(my $v = $qp->{$from}) or next;
 		$hints->{$to} = $v if $v ne '';
 	}
-
-	$ctx->{'log'} = tmpfile("solve.$oid_b") // die "tmpfile: $!";
 	$ctx->{fn} = $fn;
+	$ctx->{-tmp} = File::Temp->newdir("solver.$oid_b-XXXX", TMPDIR => 1);
+	open $ctx->{lh}, '+>>', "$ctx->{-tmp}/solve.log" or die "open: $!";
 	my $solver = PublicInbox::SolverGit->new($ctx->{ibx},
 						\&solve_result, $ctx);
+	$solver->{tmp} = $ctx->{-tmp}; # share tmpdir
 	# PSGI server will call this immediately and give us a callback (-wcb)
 	sub {
 		$ctx->{-wcb} = $_[0]; # HTTP write callback
-		$solver->solve($ctx->{env}, $ctx->{log}, $oid_b, $hints);
+		$solver->solve($ctx->{env}, $ctx->{lh}, $oid_b, $hints);
 	};
 }
 

[PATCH 03/18] viewvcs: delay stringification of solver debug log

[Eric Wong]

This will make future changes easier to work on as we pass more
stuff through $ctx and reduce parameter passing on the Perl stack.
---
 lib/PublicInbox/ViewVCS.pm | 130 +++++++++++++++++--------------------
 1 file changed, 61 insertions(+), 69 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index c5d16478..b04a5672 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -37,20 +37,35 @@ my $SHOW_FMT = '--pretty=format:'.join('%n', '%P', '%p', '%H', '%T', '%s',
 	'%an <%ae>  %ai', '%cn <%ce>  %ci', '%b%x00');
 
 sub html_page ($$$) {
-	my ($ctx, $code, $strref) = @_;
+	my ($ctx, $code, $str) = @_;
 	my $wcb = delete $ctx->{-wcb};
 	$ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
-	my $res = html_oneshot($ctx, $code, $strref);
+	my $res = html_oneshot($ctx, $code, \$str);
 	$wcb ? $wcb->($res) : $res;
 }
 
+sub dbg_log ($) {
+	my ($ctx) = @_;
+	my $log = delete $ctx->{lh} // die 'BUG: already captured debug log';
+	if (!seek($log, 0, 0)) {
+		warn "seek(log): $!";
+		return '<pre>debug log seek error</pre>';
+	}
+	$log = do { local $/; <$log> } // do {
+		warn "readline(log): $!";
+		return '<pre>debug log read error</pre>';
+	};
+	$ctx->{-linkify} //= PublicInbox::Linkify->new;
+	'<pre>debug log:</pre><hr /><pre>'.
+		$ctx->{-linkify}->to_html($log).'</pre>';
+}
+
 sub stream_blob_parse_hdr { # {parse_hdr} for Qspawn
 	my ($r, $bref, $ctx) = @_;
-	my ($res, $logref) = delete @$ctx{qw(-res -logref)};
-	my ($git, $oid, $type, $size, $di) = @$res;
+	my ($git, $oid, $type, $size, $di) = @{$ctx->{-res}};
 	my @cl = ('Content-Length', $size);
-	if (!defined $r) { # error
-		html_page($ctx, 500, $logref);
+	if (!defined $r) { # sysread error
+		html_page($ctx, 500, dbg_log($ctx));
 	} elsif (index($$bref, "\0") >= 0) {
 		[200, [qw(Content-Type application/octet-stream), @cl] ];
 	} else {
@@ -60,17 +75,16 @@ sub stream_blob_parse_hdr { # {parse_hdr} for Qspawn
 				'text/plain; charset=UTF-8', @cl ] ];
 		}
 		if ($r == 0) {
-			warn "premature EOF on $oid $$logref";
-			return html_page($ctx, 500, $logref);
+			my $log = dbg_log($ctx);
+			warn "premature EOF on $oid $log";
+			return html_page($ctx, 500, $log);
 		}
-		@$ctx{qw(-res -logref)} = ($res, $logref);
 		undef; # bref keeps growing
 	}
 }
 
-sub stream_large_blob ($$$$) {
-	my ($ctx, $res, $logref, $fn) = @_;
-	$ctx->{-logref} = $logref;
+sub stream_large_blob ($$) {
+	my ($ctx, $res) = @_;
 	$ctx->{-res} = $res;
 	my ($git, $oid, $type, $size, $di) = @$res;
 	my $cmd = ['git', "--git-dir=$git->{git_dir}", 'cat-file', $type, $oid];
@@ -80,18 +94,16 @@ sub stream_large_blob ($$$$) {
 	$qsp->psgi_return($env, undef, \&stream_blob_parse_hdr, $ctx);
 }
 
-sub show_other_result ($$) {
+sub show_other_result ($$) { # tag, tree, ...
 	my ($bref, $ctx) = @_;
-	my ($qsp_err, $logref) = delete @$ctx{qw(-qsp_err -logref)};
-	if ($qsp_err) {
-		$$logref .= "git show error:$qsp_err";
-		return html_page($ctx, 500, $logref);
+	if (my $qsp_err = delete $ctx->{-qsp_err}) {
+		return html_page($ctx, 500, dbg_log($ctx) .
+				"git show error:$qsp_err");
 	}
 	my $l = PublicInbox::Linkify->new;
 	utf8::decode($$bref);
-	$$bref = '<pre>'. $l->to_html($$bref);
-	$$bref .= '</pre><hr>' . $$logref;
-	html_page($ctx, 200, $bref);
+	html_page($ctx, 200, '<pre>', $l->to_html($$bref), '</pre><hr>',
+		dbg_log($ctx));
 }
 
 sub cmt_title { # git->cat_async callback
@@ -104,10 +116,9 @@ sub cmt_title { # git->cat_async callback
 
 sub show_commit_start { # ->psgi_qx callback
 	my ($bref, $ctx) = @_;
-	my ($qsp_err, $logref) = delete @$ctx{qw(-qsp_err -logref)};
-	if ($qsp_err) {
-		$$logref .= "git show/patch-id error:$qsp_err";
-		return html_page($ctx, 500, $logref);
+	if (my $qsp_err = delete $ctx->{-qsp_err}) {
+		return html_page($ctx, 500, dbg_log($ctx) .
+				"git show/patch-id error:$qsp_err");
 	}
 	my $patchid = (split(/ /, $$bref))[0]; # ignore commit
 	$ctx->{-q_value_html} = "patchid:$patchid" if defined $patchid;
@@ -135,7 +146,7 @@ sub show_commit_start { # ->psgi_qx callback
 
 sub cmt_finalize {
 	my ($ctx) = @_;
-	$ctx->{-linkify} = PublicInbox::Linkify->new;
+	$ctx->{-linkify} //= PublicInbox::Linkify->new;
 	# try to keep author and committer dates lined up
 	my ($au, $co) = delete @$ctx{qw(cmt_au cmt_co)};
 	my $x = length($au) - length($co);
@@ -219,8 +230,8 @@ EOM
 	delete($ctx->{env}->{'qspawn.wcb'})->([200, $res_hdr, [$x]]);
 }
 
-sub show_commit ($$$$) {
-	my ($ctx, $res, $logref, $fn) = @_;
+sub show_commit ($$) {
+	my ($ctx, $res) = @_;
 	my ($git, $oid) = @$res;
 	# patch-id needs two passes, and we use the initial show to ensure
 	# a patch embedded inside the commit message body doesn't get fed
@@ -234,84 +245,67 @@ sub show_commit ($$$$) {
 	my $e = { GIT_DIR => $git->{git_dir} };
 	my $qsp = PublicInbox::Qspawn->new($cmd, $e, { -C => "$ctx->{-tmp}" });
 	$qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
-	$ctx->{-logref} = $logref;
 	$ctx->{env}->{'qspawn.wcb'} = delete $ctx->{-wcb};
 	$ctx->{git} = $git;
 	$qsp->psgi_qx($ctx->{env}, undef, \&show_commit_start, $ctx);
 }
 
-sub show_other ($$$$) {
-	my ($ctx, $res, $logref, $fn) = @_;
+sub show_other ($$) {
+	my ($ctx, $res) = @_;
 	my ($git, $oid, $type, $size) = @$res;
-	if ($size > $MAX_SIZE) {
-		$$logref = "$oid is too big to show\n" . $$logref;
-		return html_page($ctx, 200, $logref);
-	}
+	$size > $MAX_SIZE and return html_page($ctx, 200,
+				"$oid is too big to show\n". dbg_log($ctx));
 	my $cmd = ['git', "--git-dir=$git->{git_dir}",
 		qw(show --encoding=UTF-8 --no-color --no-abbrev), $oid ];
 	my $qsp = PublicInbox::Qspawn->new($cmd);
 	$qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
-	$ctx->{-logref} = $logref;
 	$qsp->psgi_qx($ctx->{env}, undef, \&show_other_result, $ctx);
 }
 
 # user_cb for SolverGit, called as: user_cb->($result_or_error, $uarg)
 sub solve_result {
 	my ($res, $ctx) = @_;
-	my ($log, $hints, $fn) = delete @$ctx{qw(lh hints fn)};
-
-	unless (seek($log, 0, 0)) {
-		warn "seek(log): $!";
-		return html_page($ctx, 500, \'seek error');
-	}
-	$log = do { local $/; <$log> };
-
-	my $l = PublicInbox::Linkify->new;
-	$log = '<pre>debug log:</pre><hr /><pre>' .
-		$l->to_html($log) . '</pre>';
-
-	$res or return html_page($ctx, 404, \$log);
-	ref($res) eq 'ARRAY' or return html_page($ctx, 500, \$log);
+	my $hints = delete $ctx->{hints};
+	$res or return html_page($ctx, 404, dbg_log($ctx));
+	ref($res) eq 'ARRAY' or return html_page($ctx, 500, dbg_log($ctx));
 
 	my ($git, $oid, $type, $size, $di) = @$res;
-	return show_commit($ctx, $res, \$log, $fn) if $type eq 'commit';
-	return show_other($ctx, $res, \$log, $fn) if $type ne 'blob';
+	return show_commit($ctx, $res) if $type eq 'commit';
+	return show_other($ctx, $res) if $type ne 'blob';
 	my $path = to_filename($di->{path_b} // $hints->{path_b} // 'blob');
 	my $raw_link = "(<a\nhref=$path>raw</a>)";
 	if ($size > $MAX_SIZE) {
-		return stream_large_blob($ctx, $res, \$log, $fn) if defined $fn;
-		$log = "<pre><b>Too big to show, download available</b>\n" .
-			"$oid $type $size bytes $raw_link</pre>" . $log;
-		return html_page($ctx, 200, \$log);
+		return stream_large_blob($ctx, $res) if defined $ctx->{fn};
+		return html_page($ctx, 200, <<EOM . dbg_log($ctx));
+<pre><b>Too big to show, download available</b>
+"$oid $type $size bytes $raw_link</pre>
+EOM
 	}
 
 	my $blob = $git->cat_file($oid);
 	if (!$blob) { # WTF?
 		my $e = "Failed to retrieve generated blob ($oid)";
 		warn "$e ($git->{git_dir})";
-		$log = "<pre><b>$e</b></pre>" . $log;
-		return html_page($ctx, 500, \$log);
+		return html_page($ctx, 500, "<pre><b>$e</b></pre>".dbg_log($ctx))
 	}
 
 	my $bin = index(substr($$blob, 0, $BIN_DETECT), "\0") >= 0;
-	if (defined $fn) {
+	if (defined $ctx->{fn}) {
 		my $h = [ 'Content-Length', $size, 'Content-Type' ];
 		push(@$h, ($bin ? 'application/octet-stream' : 'text/plain'));
 		return delete($ctx->{-wcb})->([200, $h, [ $$blob ]]);
 	}
 
-	if ($bin) {
-		$log = "<pre>$oid $type $size bytes (binary)" .
-			" $raw_link</pre>" . $log;
-		return html_page($ctx, 200, \$log);
-	}
+	$bin and return html_page($ctx, 200,
+				"<pre>$oid $type $size bytes (binary)" .
+				" $raw_link</pre>".dbg_log($ctx));
 
 	# TODO: detect + convert to ensure validity
 	utf8::decode($$blob);
 	my $nl = ($$blob =~ s/\r?\n/\n/sg);
 	my $pad = length($nl);
 
-	$l->linkify_1($$blob);
+	($ctx->{-linkify} //= PublicInbox::Linkify->new)->linkify_1($$blob);
 	my $ok = $hl->do_hl($blob, $path) if $hl;
 	if ($ok) {
 		$blob = $ok;
@@ -320,17 +314,15 @@ sub solve_result {
 	}
 
 	# using some of the same CSS class names and ids as cgit
-	$log = "<pre>$oid $type $size bytes $raw_link</pre>" .
+	html_page($ctx, 200, "<pre>$oid $type $size bytes $raw_link</pre>" .
 		"<hr /><table\nclass=blob>".
 		"<tr><td\nclass=linenumbers><pre>" . join('', map {
 			sprintf("<a id=n$_ href=#n$_>% ${pad}u</a>\n", $_)
 		} (1..$nl)) . '</pre></td>' .
 		'<td><pre> </pre></td>'. # pad for non-CSS users
 		"<td\nclass=lines><pre\nstyle='white-space:pre'><code>" .
-		$l->linkify_2($$blob) .
-		'</code></pre></td></tr></table>' . $log;
-
-	html_page($ctx, 200, \$log);
+		$ctx->{-linkify}->linkify_2($$blob) .
+		'</code></pre></td></tr></table>'.dbg_log($ctx));
 }
 
 # GET /$INBOX/$GIT_OBJECT_ID/s/

[PATCH 04/18] www: allow html_oneshot to take an array arg

[Eric Wong]

Another step towards making our internal APIs more writev-like
and reducing the copies needed for `join' or `.=' concatenation.
---
 lib/PublicInbox/View.pm      | 4 ++--
 lib/PublicInbox/ViewVCS.pm   | 6 +++---
 lib/PublicInbox/WWW.pm       | 2 +-
 lib/PublicInbox/WwwAltId.pm  | 6 +++---
 lib/PublicInbox/WwwStream.pm | 7 +++----
 lib/PublicInbox/WwwText.pm   | 2 +-
 6 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 158feb6a..5fbdd1fa 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -64,7 +64,7 @@ sub no_over_html ($) {
 	}
 	delete $ctx->{obuf};
 	eval { $$obuf .= html_footer($ctx, $eml) };
-	html_oneshot($ctx, 200, $obuf);
+	html_oneshot($ctx, 200, $$obuf);
 }
 
 # public functions: (unstable)
@@ -1222,7 +1222,7 @@ sub pagination_footer ($$) {
 		$next = $next ? "$next | " : '             | ';
 		$prev .= qq[ | <a\nhref="$latest">latest</a>];
 	}
-	($next || $prev) ? \"<hr><pre>page: $next$prev</pre>" : \'';
+	($next || $prev) ? "<hr><pre>page: $next$prev</pre>" : '';
 }
 
 sub paginate_recent ($$) {
diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index b04a5672..23524ac0 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -36,11 +36,11 @@ my $BIN_DETECT = 8000; # same as git
 my $SHOW_FMT = '--pretty=format:'.join('%n', '%P', '%p', '%H', '%T', '%s',
 	'%an <%ae>  %ai', '%cn <%ce>  %ci', '%b%x00');
 
-sub html_page ($$$) {
-	my ($ctx, $code, $str) = @_;
+sub html_page ($$;@) {
+	my ($ctx, $code) = @_[0, 1];
 	my $wcb = delete $ctx->{-wcb};
 	$ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
-	my $res = html_oneshot($ctx, $code, \$str);
+	my $res = html_oneshot($ctx, $code, @_[2..$#_]);
 	$wcb ? $wcb->($res) : $res;
 }
 
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 77f463d3..1df5572d 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -320,7 +320,7 @@ sub get_altid_dump {
 sub need {
 	my ($ctx, $extra) = @_;
 	require PublicInbox::WwwStream;
-	PublicInbox::WwwStream::html_oneshot($ctx, 501, \<<EOF);
+	PublicInbox::WwwStream::html_oneshot($ctx, 501, <<EOF);
 <pre>$extra is not available for this public-inbox
 <a\nhref="../">Return to index</a></pre>
 EOF
diff --git a/lib/PublicInbox/WwwAltId.pm b/lib/PublicInbox/WwwAltId.pm
index e107dfe0..47056160 100644
--- a/lib/PublicInbox/WwwAltId.pm
+++ b/lib/PublicInbox/WwwAltId.pm
@@ -33,14 +33,14 @@ sub sqldump ($$) {
 	my $altid_map = $ibx->altid_map;
 	my $fn = $altid_map->{$altid_pfx};
 	unless (defined $fn) {
-		return html_oneshot($ctx, 404, \<<EOF);
+		return html_oneshot($ctx, 404, <<EOF);
 <pre>`$altid_pfx' is not a valid altid for this inbox</pre>
 EOF
 	}
 
 	if ($env->{REQUEST_METHOD} ne 'POST') {
 		my $url = $ibx->base_url($ctx->{env}) . "$altid_pfx.sql.gz";
-		return html_oneshot($ctx, 405, \<<EOF);
+		return html_oneshot($ctx, 405, <<EOF);
 <pre>A POST request is required to retrieve $altid_pfx.sql.gz
 
 	curl -d '' -O $url
@@ -54,7 +54,7 @@ or
 EOF
 	}
 
-	$sqlite3 //= which('sqlite3') // return html_oneshot($ctx, 501, \<<EOF);
+	$sqlite3 //= which('sqlite3') // return html_oneshot($ctx, 501, <<EOF);
 <pre>sqlite3 not available
 
 The administrator needs to install the sqlite3(1) binary
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index ab006c40..f2777fdc 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -164,8 +164,8 @@ sub getline {
 	$ctx->zflush(_html_end($ctx));
 }
 
-sub html_oneshot ($$;$) {
-	my ($ctx, $code, $sref) = @_;
+sub html_oneshot ($$;@) {
+	my ($ctx, $code) = @_[0, 1];
 	my $res_hdr = [ 'Content-Type' => 'text/html; charset=UTF-8',
 		'Content-Length' => undef ];
 	bless $ctx, __PACKAGE__;
@@ -174,8 +174,7 @@ sub html_oneshot ($$;$) {
 		$ctx->zmore(html_top($ctx));
 		$ctx->{base_url} = base_url($ctx);
 	};
-	$ctx->zmore($$sref) if $sref;
-	my $bdy = $ctx->zflush(_html_end($ctx));
+	my $bdy = $ctx->zflush(@_[2..$#_], _html_end($ctx));
 	$res_hdr->[3] = length($bdy);
 	[ $code, $res_hdr, [ $bdy ] ]
 }
diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm
index c6957e81..3f840c44 100644
--- a/lib/PublicInbox/WwwText.pm
+++ b/lib/PublicInbox/WwwText.pm
@@ -70,7 +70,7 @@ sub get_text {
 	$txt =~ s!\bPOP3\b!<a\nid=pop3>POP3</a>!;
 	$txt =~ s!\b(Newsgroups?)\b!<a\nid=nntp>$1</a>!;
 	$txt =~ s!\bIMAP\b!<a\nid=imap>IMAP</a>!;
-	PublicInbox::WwwStream::html_oneshot($ctx, $code, \$txt);
+	PublicInbox::WwwStream::html_oneshot($ctx, $code, $txt);
 }
 
 sub _srch_prefix ($$) {

[PATCH 05/18] viewvcs: use array for highlighted blob display

[Eric Wong]

This can avoid at least one expensive copy for displaying
large blobs with syntax highlighting.

However, we cannot blindly change everything to arrays, either:
the cost of invoking Compress::Raw::Zlib->deflate must be taken
into account.  Joining short strings via `.=', `.', `join' or
interpolation is typically faster since it avoids ->deflate
method calls (and non-magic perlops are the fastest dispatches
in Perl).
---
 lib/PublicInbox/ViewVCS.pm | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 23524ac0..8fb0844d 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -313,15 +313,15 @@ EOM
 		$$blob = ascii_html($$blob);
 	}
 
-	# using some of the same CSS class names and ids as cgit
-	html_page($ctx, 200, "<pre>$oid $type $size bytes $raw_link</pre>" .
+	my $x = "<pre>$oid $type $size bytes $raw_link</pre>" .
 		"<hr /><table\nclass=blob>".
-		"<tr><td\nclass=linenumbers><pre>" . join('', map {
-			sprintf("<a id=n$_ href=#n$_>% ${pad}u</a>\n", $_)
-		} (1..$nl)) . '</pre></td>' .
-		'<td><pre> </pre></td>'. # pad for non-CSS users
-		"<td\nclass=lines><pre\nstyle='white-space:pre'><code>" .
-		$ctx->{-linkify}->linkify_2($$blob) .
+		"<tr><td\nclass=linenumbers><pre>";
+	$x .= sprintf("<a id=n$_ href=#n$_>% ${pad}u</a>\n", $_) for (1..$nl);
+	$x .= '</pre></td><td><pre> </pre></td>'. # pad for non-CSS users
+		"<td\nclass=lines><pre\nstyle='white-space:pre'><code>";
+
+	# using some of the same CSS class names and ids as cgit
+	html_page($ctx, 200, $x, $ctx->{-linkify}->linkify_2($$blob),
 		'</code></pre></td></tr></table>'.dbg_log($ctx));
 }
 

[PATCH 06/18] viewvcs: add patch download link for single-parent commits

[Eric Wong]

The email signature field will echo the format-patch arguments
used so it can be used as hint for users learning to more of
the git CLI.
---
 lib/PublicInbox/ViewVCS.pm | 43 ++++++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 8fb0844d..84358d0e 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -23,6 +23,7 @@ use PublicInbox::Linkify;
 use PublicInbox::Tmpfile;
 use PublicInbox::ViewDiff qw(flush_diff);
 use PublicInbox::View;
+use PublicInbox::Eml;
 use Text::Wrap qw(wrap);
 use PublicInbox::Hval qw(ascii_html to_filename);
 my $hl = eval {
@@ -33,7 +34,7 @@ my $hl = eval {
 my %QP_MAP = ( A => 'oid_a', a => 'path_a', b => 'path_b' );
 our $MAX_SIZE = 1024 * 1024; # TODO: configurable
 my $BIN_DETECT = 8000; # same as git
-my $SHOW_FMT = '--pretty=format:'.join('%n', '%P', '%p', '%H', '%T', '%s',
+my $SHOW_FMT = '--pretty=format:'.join('%n', '%P', '%p', '%H', '%T', '%s', '%f',
 	'%an <%ae>  %ai', '%cn <%ce>  %ci', '%b%x00');
 
 sub html_page ($$;@) {
@@ -127,8 +128,8 @@ sub show_commit_start { # ->psgi_qx callback
 	chop(my $buf = do { local $/ = "\0"; <$fh> });
 	chomp $buf;
 	my ($P, $p);
-	($P, $p, @$ctx{qw(cmt_H cmt_T cmt_s cmt_au cmt_co cmt_b)})
-		= split(/\n/, $buf, 8);
+	($P, $p, @$ctx{qw(cmt_H cmt_T cmt_s cmt_f cmt_au cmt_co cmt_b)})
+		= split(/\n/, $buf, 9);
 	return cmt_finalize($ctx) if !$P;
 	@{$ctx->{-cmt_P}} = split(/ /, $P);
 	@{$ctx->{-cmt_p}} = split(/ /, $p); # abbreviated
@@ -164,7 +165,8 @@ sub cmt_finalize {
 	my ($P, $p, $pt) = delete @$ctx{qw(-cmt_P -cmt_p -cmt_pt)};
 	$_ = qq(<a href="$upfx$_/s/">).shift(@$p).'</a> '.shift(@$pt) for @$P;
 	if (@$P == 1) {
-		$x = qq(\n   parent $P->[0]);
+		$x = qq{ (<a
+href="$ctx->{cmt_f}.patch">patch</a>)\n   parent $P->[0]};
 	} elsif (@$P > 1) {
 		$x = qq(\n  parents $P->[0]\n);
 		shift @$P;
@@ -230,8 +232,41 @@ EOM
 	delete($ctx->{env}->{'qspawn.wcb'})->([200, $res_hdr, [$x]]);
 }
 
+sub stream_patch_parse_hdr { # {parse_hdr} for Qspawn
+	my ($r, $bref, $ctx) = @_;
+	if (!defined $r) { # sysread error
+		html_page($ctx, 500, dbg_log($ctx));
+	} elsif (index($$bref, "\n\n") >= 0) {
+		my $eml = bless { hdr => $bref }, 'PublicInbox::Eml';
+		my $fn = to_filename($eml->header('Subject') // '');
+		$fn = substr($fn // 'PATCH-no-subject', 6); # drop "PATCH-"
+		return [ 200, [ 'Content-Type', 'text/plain; charset=UTF-8',
+				'Content-Disposition',
+				qq(inline; filename=$fn.patch) ] ];
+	} elsif ($r == 0) {
+		my $log = dbg_log($ctx);
+		warn "premature EOF on $ctx->{patch_oid} $log";
+		return html_page($ctx, 500, $log);
+	} else {
+		undef; # bref keeps growing until "\n\n"
+	}
+}
+
+sub show_patch ($$) {
+	my ($ctx, $res) = @_;
+	my ($git, $oid) = @$res;
+	my @cmd = ('git', "--git-dir=$git->{git_dir}",
+		qw(format-patch -1 --stdout -C),
+		"--signature=git format-patch -1 --stdout -C $oid", $oid);
+	my $qsp = PublicInbox::Qspawn->new(\@cmd);
+	$ctx->{env}->{'qspawn.wcb'} = delete $ctx->{-wcb};
+	$ctx->{patch_oid} = $oid;
+	$qsp->psgi_return($ctx->{env}, undef, \&stream_patch_parse_hdr, $ctx);
+}
+
 sub show_commit ($$) {
 	my ($ctx, $res) = @_;
+	return show_patch($ctx, $res) if ($ctx->{fn} // '') =~ /\.patch\z/;
 	my ($git, $oid) = @$res;
 	# patch-id needs two passes, and we use the initial show to ensure
 	# a patch embedded inside the commit message body doesn't get fed

[PATCH 07/18] viewvcs: author date links to contemporary messages

[Eric Wong]

Sometimes it can be useful to figure out what's going in on the
associated inbox around the time the patch was authored.
---
 lib/PublicInbox/ViewVCS.pm | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 84358d0e..fd95e24e 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -148,6 +148,7 @@ sub show_commit_start { # ->psgi_qx callback
 sub cmt_finalize {
 	my ($ctx) = @_;
 	$ctx->{-linkify} //= PublicInbox::Linkify->new;
+	my $upfx = $ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
 	# try to keep author and committer dates lined up
 	my ($au, $co) = delete @$ctx{qw(cmt_au cmt_co)};
 	my $x = length($au) - length($co);
@@ -159,9 +160,15 @@ sub cmt_finalize {
 		$au =~ s/>/>$x/;
 	}
 	$_ = ascii_html($_) for ($au, $co);
+	$au =~ s!(> +)([0-9]{4,}-\S+ \S+)!
+		my ($gt, $t) = ($1, $2);
+		$t =~ tr/ :-//d;
+		qq($gt<a
+href="$upfx?t=$t"
+title="list contemporary emails">$2</a>)
+		!e;
 	my $s = $ctx->{-linkify}->to_html(delete $ctx->{cmt_s});
 	$ctx->{-title_html} = $s;
-	my $upfx = $ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
 	my ($P, $p, $pt) = delete @$ctx{qw(-cmt_P -cmt_p -cmt_pt)};
 	$_ = qq(<a href="$upfx$_/s/">).shift(@$p).'</a> '.shift(@$pt) for @$P;
 	if (@$P == 1) {

[PATCH 08/18] view: speed up /$INBOX/ landing page by 0.5-1.0%

[Eric Wong]

Array lookups and extra arithmetic in Perl is slower than
bumping the internal array offset inside the interpreter.
Fwiw, using: my ($level, $subj) = splice(@extra, 0, 2)
did not result in a performance improvement.
---
 lib/PublicInbox/View.pm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 5fbdd1fa..9846fa47 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -1188,9 +1188,9 @@ sub dump_topics {
 
 		my $s = "<a\nhref=\"$href/T/$anchor\">$top_subj</a>\n" .
 			" $ds UTC $n\n";
-		for (my $i = 0; $i < scalar(@extra); $i += 2) {
-			my $level = $extra[$i];
-			my $subj = $extra[$i + 1]; # already normalized
+		while (@extra) {
+			my $level = shift @extra;
+			my $subj = shift @extra; # already normalized
 			$mid = delete $seen->{$subj};
 			my @subj = split(/ /, $subj);
 			my @next_prev = @subj; # full copy

[PATCH 09/18] treewide: ditch inbox->recent method

[Eric Wong]

It's a needless wrapper, nowadays.  Originally, ->over was added
on experimental basis to optimize for /$INBOX/ where Xapian
->search is slower on gigantic (LKML-sized) inboxes.

Nowadays with extindex, ->over is here to stay given NNTP and
IMAP both benefit from it.  So reduce the interpreter stack
overhead and just access ->over directly.

lxs->recent was never used outside of tests, anyways.

And while we're in the area, avoid needlessly bumping the
refcount of $ctx->{ibx} in View::paginate_recent.
---
 lib/PublicInbox/ExtSearch.pm      |  1 -
 lib/PublicInbox/Inbox.pm          |  5 -----
 lib/PublicInbox/LeiSavedSearch.pm |  1 -
 lib/PublicInbox/LeiXSearch.pm     |  7 -------
 lib/PublicInbox/View.pm           |  5 ++---
 t/convert-compact.t               |  2 +-
 t/indexlevels-mirror.t            | 10 +++++-----
 t/lei_xsearch.t                   |  2 +-
 t/replace.t                       |  4 ++--
 t/v1-add-remove-add.t             |  2 +-
 t/v2-add-remove-add.t             |  2 +-
 11 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/lib/PublicInbox/ExtSearch.pm b/lib/PublicInbox/ExtSearch.pm
index 3eb864a2..a69c0e76 100644
--- a/lib/PublicInbox/ExtSearch.pm
+++ b/lib/PublicInbox/ExtSearch.pm
@@ -125,7 +125,6 @@ no warnings 'once';
 *smsg_by_mid = \&PublicInbox::Inbox::smsg_by_mid;
 *msg_by_mid = \&PublicInbox::Inbox::msg_by_mid;
 *modified = \&PublicInbox::Inbox::modified;
-*recent = \&PublicInbox::Inbox::recent;
 
 *max_git_epoch = *nntp_usable = *msg_by_path = \&mm; # undef
 *isrch = \&search;
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index acd7f338..8ac7eb30 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -351,11 +351,6 @@ sub msg_by_mid ($$) {
 	$smsg ? msg_by_smsg($self, $smsg) : msg_by_path($self, mid2path($mid));
 }
 
-sub recent {
-	my ($self, $opts, $after, $before) = @_;
-	$self->over->recent($opts, $after, $before);
-}
-
 sub modified {
 	my ($self) = @_;
 	if (my $over = $self->over) {
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 1d13aef6..ed92bfd1 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -299,7 +299,6 @@ no warnings 'once';
 *smsg_by_mid = \&PublicInbox::Inbox::smsg_by_mid;
 *msg_by_mid = \&PublicInbox::Inbox::msg_by_mid;
 *modified = \&PublicInbox::Inbox::modified;
-*recent = \&PublicInbox::Inbox::recent;
 *max_git_epoch = *nntp_usable = *msg_by_path = \&mm; # undef
 *isrch = *search = \&mm; # TODO
 *DESTROY = \&pause_dedupe;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 6f877019..90cb83b9 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -103,13 +103,6 @@ sub smsg_for {
 	$smsg;
 }
 
-sub recent {
-	my ($self, $qstr, $opt) = @_;
-	$opt //= {};
-	$opt->{relevance} //= -2;
-	$self->mset($qstr //= 'z:1..', $opt);
-}
-
 sub over {}
 
 sub _check_mset_limit ($$$) {
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 9846fa47..466ec6cf 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -1237,12 +1237,11 @@ sub paginate_recent ($$) {
 	$t =~ s/\A([0-9]{8,14})-// and $after = str2ts($1);
 	$t =~ /\A([0-9]{8,14})\z/ and $before = str2ts($1);
 
-	my $ibx = $ctx->{ibx};
-	my $msgs = $ibx->recent($opts, $after, $before);
+	my $msgs = $ctx->{ibx}->over->recent($opts, $after, $before);
 	my $nr = scalar @$msgs;
 	if ($nr < $lim && defined($after)) {
 		$after = $before = undef;
-		$msgs = $ibx->recent($opts);
+		$msgs = $ctx->{ibx}->over->recent($opts);
 		$nr = scalar @$msgs;
 	}
 	my $more = $nr == $lim;
diff --git a/t/convert-compact.t b/t/convert-compact.t
index 7270cab0..def09567 100644
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -101,7 +101,7 @@ foreach (@xdir) {
 	is($st[2] & 07777, -f _ ? 0444 : 0755,
 		'sharedRepository respected after v2 compact');
 }
-my $msgs = $ibx->recent({limit => 1000});
+my $msgs = $ibx->over->recent({limit => 1000});
 is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history');
 is(scalar @$msgs, 1, 'only one message in history');
 
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index ac85643d..463b35be 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -41,7 +41,7 @@ my $import_index_incremental = sub {
 		inboxdir => $ibx->{inboxdir},
 		indexlevel => $level
 	});
-	my $msgs = $ro_master->recent;
+	my $msgs = $ro_master->over->recent;
 	is(scalar(@$msgs), 1, 'only one message in master, so far');
 	is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');
 
@@ -71,7 +71,7 @@ my $import_index_incremental = sub {
 		inboxdir => $mirror,
 		indexlevel => $level,
 	});
-	$msgs = $ro_mirror->recent;
+	$msgs = $ro_mirror->over->recent;
 	is(scalar(@$msgs), 1, 'only one message, so far');
 	is($msgs->[0]->{mid}, 'm@1', 'read first message');
 
@@ -83,7 +83,7 @@ my $import_index_incremental = sub {
 	# mirror updates
 	is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
 	ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror again OK");
-	$msgs = $ro_mirror->recent;
+	$msgs = $ro_mirror->over->recent;
 	is(scalar(@$msgs), 2, '2nd message seen in mirror');
 	is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
 		['m@1','m@2'], 'got both messages in mirror');
@@ -91,7 +91,7 @@ my $import_index_incremental = sub {
 	# incremental index master (required for v1)
 	ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]),
 		'index master OK');
-	$msgs = $ro_master->recent;
+	$msgs = $ro_master->over->recent;
 	is(scalar(@$msgs), 2, '2nd message seen in master');
 	is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
 		['m@1','m@2'], 'got both messages in master');
@@ -120,7 +120,7 @@ my $import_index_incremental = sub {
 	# sync the mirror
 	is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
 	ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror again OK");
-	$msgs = $ro_mirror->recent;
+	$msgs = $ro_mirror->over->recent;
 	is(scalar(@$msgs), 1, '2nd message gone from mirror');
 	is_deeply([map { $_->{mid} } @$msgs], ['m@1'],
 		'message unavailable in mirror');
diff --git a/t/lei_xsearch.t b/t/lei_xsearch.t
index d9ddb297..fabceb41 100644
--- a/t/lei_xsearch.t
+++ b/t/lei_xsearch.t
@@ -61,7 +61,7 @@ for my $mi ($mset->items) {
 }
 is(scalar(@msgs), $nr, 'smsgs retrieved for all');
 
-$mset = $lxs->recent(undef, { limit => 1 });
+$mset = $lxs->mset('z:1..', { relevance => -2, limit => 1 });
 is($mset->size, 1, 'one result');
 
 my @ibxish = $lxs->locals;
diff --git a/t/replace.t b/t/replace.t
index 626cbe9b..0e121399 100644
--- a/t/replace.t
+++ b/t/replace.t
@@ -49,7 +49,7 @@ EOF
 	$im->done;
 	my $thread_a = $ibx->over->get_thread('replace@example.com');
 
-	my %before = map {; delete($_->{blob}) => $_ } @{$ibx->recent};
+	my %before = map {; delete($_->{blob}) => $_ } @{$ibx->over->recent};
 	my $reject = PublicInbox::Eml->new($orig->as_string);
 	foreach my $mid (['<replace@example.com>', '<extra@example.com>'],
 				[], ['<replaced@example.com>']) {
@@ -126,7 +126,7 @@ EOF
 	}
 
 	# check overview matches:
-	my %after = map {; delete($_->{blob}) => $_ } @{$ibx->recent};
+	my %after = map {; delete($_->{blob}) => $_ } @{$ibx->over->recent};
 	my @before_blobs = keys %before;
 	foreach my $blob (@before_blobs) {
 		delete $before{$blob} if delete $after{$blob};
diff --git a/t/v1-add-remove-add.t b/t/v1-add-remove-add.t
index a94bf7fd..ae045dfa 100644
--- a/t/v1-add-remove-add.t
+++ b/t/v1-add-remove-add.t
@@ -32,7 +32,7 @@ ok($im->add($mime), 'message added again');
 $im->done;
 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
 $rw->index_sync;
-my $msgs = $ibx->recent({limit => 10});
+my $msgs = $ibx->over->recent({limit => 10});
 is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history');
 is(scalar @$msgs, 1, 'only one message in history');
 is($ibx->mm->num_for('a-mid@b'), 2, 'exists with second article number');
diff --git a/t/v2-add-remove-add.t b/t/v2-add-remove-add.t
index 579cdcb6..6affc830 100644
--- a/t/v2-add-remove-add.t
+++ b/t/v2-add-remove-add.t
@@ -32,7 +32,7 @@ ok($im->add($mime), 'message added');
 ok($im->remove($mime), 'message removed');
 ok($im->add($mime), 'message added again');
 $im->done;
-my $msgs = $ibx->recent({limit => 1000});
+my $msgs = $ibx->over->recent({limit => 1000});
 is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history');
 is(scalar @$msgs, 1, 'only one message in history');
 

[PATCH 10/18] view: /$INBOX/: show "messages from $old to $new"

[Eric Wong]

With the ViewVCS commit view using /$INBOX/?t=YYYYMMDDhhmmss-
links, the use of `t=' may not be immediately obvious to a
reader and confuse them into thinking the inbox hasn't been
updated in a while.

So add a header to the top of the page whenever the `t=' query
parameter is used.

And kill a couple of redundant variable assignments while we're
at it.
---
 lib/PublicInbox/View.pm | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 466ec6cf..11ed2d76 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -20,6 +20,7 @@ use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::Reply;
 use PublicInbox::ViewDiff qw(flush_diff);
 use PublicInbox::Eml;
+use POSIX qw(strftime);
 use Time::Local qw(timegm);
 use PublicInbox::Smsg qw(subject_normalized);
 use PublicInbox::ContentHash qw(content_hash);
@@ -1161,9 +1162,10 @@ sub dump_topics {
 	}
 
 	my @out;
-	my $ibx = $ctx->{ibx};
-	my $obfs_ibx = $ibx->{obfuscate} ? $ibx : undef;
-
+	my $obfs_ibx = $ctx->{ibx}->{obfuscate} ? $ctx->{ibx} : undef;
+	if (my $note = delete $ctx->{t_note}) {
+		push @out, $note; # "messages from ... to ..."
+	}
 	# sort by recency, this allows new posts to "bump" old topics...
 	foreach my $topic (sort { $b->[0] <=> $a->[0] } @$order) {
 		my ($ds, $n, $seen, $top_subj, @extra) = @$topic;
@@ -1222,7 +1224,7 @@ sub pagination_footer ($$) {
 		$next = $next ? "$next | " : '             | ';
 		$prev .= qq[ | <a\nhref="$latest">latest</a>];
 	}
-	($next || $prev) ? "<hr><pre>page: $next$prev</pre>" : '';
+	($next || $prev) ? "<hr><pre id=nav>page: $next$prev</pre>" : '';
 }
 
 sub paginate_recent ($$) {
@@ -1238,21 +1240,29 @@ sub paginate_recent ($$) {
 	$t =~ /\A([0-9]{8,14})\z/ and $before = str2ts($1);
 
 	my $msgs = $ctx->{ibx}->over->recent($opts, $after, $before);
-	my $nr = scalar @$msgs;
-	if ($nr < $lim && defined($after)) {
+	if (defined($after) && scalar(@$msgs) < $lim) {
 		$after = $before = undef;
 		$msgs = $ctx->{ibx}->over->recent($opts);
-		$nr = scalar @$msgs;
 	}
-	my $more = $nr == $lim;
+	my $more = scalar(@$msgs) == $lim;
 	my ($newest, $oldest);
-	if ($nr) {
+	if (@$msgs) {
 		$newest = $msgs->[0]->{ts};
 		$oldest = $msgs->[-1]->{ts};
 		# if we only had $after, our SQL query in ->recent ordered
 		if ($newest < $oldest) {
 			($oldest, $newest) = ($newest, $oldest);
-			$more = 0 if defined($after) && $after < $oldest;
+			$more = undef if defined($after) && $after < $oldest;
+		}
+		if (defined($after // $before)) {
+			my $n = strftime('%Y-%m-%d %H:%M:%S', gmtime($newest));
+			my $o = strftime('%Y-%m-%d %H:%M:%S', gmtime($oldest));
+			$ctx->{t_note} = <<EOM;
+ messages from $o to $n UTC, [<a href="#nav">more...</a>]
+EOM
+			my $s = ts2str($newest);
+			$ctx->{prev_page} = qq[<a\nhref="?t=$s-"\nrel=prev>] .
+						'prev (newer)</a>';
 		}
 	}
 	if (defined($oldest) && $more) {
@@ -1260,11 +1270,6 @@ sub paginate_recent ($$) {
 		$ctx->{next_page} = qq[<a\nhref="?t=$s"\nrel=next>] .
 					'next (older)</a>';
 	}
-	if (defined($newest) && (defined($before) || defined($after))) {
-		my $s = ts2str($newest);
-		$ctx->{prev_page} = qq[<a\nhref="?t=$s-"\nrel=prev>] .
-					'prev (newer)</a>';
-	}
 	$msgs;
 }
 

[PATCH 11/18] view: cleanups and reuse for {obuf} preparation

[Eric Wong]

{obuf} will eventually go away and we'll write directly to
{zbuf}, but as an intermediate step we'll make some changes
to rely less on return values.

While we're in the area, reuse Linkify objects in more places
where possible to save some allocations.
---
 lib/PublicInbox/View.pm | 49 ++++++++++++++++++++---------------------
 t/plack.t               |  2 +-
 2 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 11ed2d76..354cdd93 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -38,7 +38,7 @@ sub msg_page_i {
 				: $ctx->gone('over');
 		$ctx->{mhref} = ($ctx->{nr} || $ctx->{smsg}) ?
 				"../${\mid_href($smsg->{mid})}/" : '';
-		my $obuf = $ctx->{obuf} = _msg_page_prepare_obuf($eml, $ctx);
+		my $obuf = _msg_page_prepare_obuf($eml, $ctx);
 		if (length($$obuf)) {
 			multipart_text_as_html($eml, $ctx);
 			$$obuf .= '</pre><hr>';
@@ -58,7 +58,7 @@ sub no_over_html ($) {
 	my $eml = PublicInbox::Eml->new($bref);
 	$ctx->{mhref} = '';
 	PublicInbox::WwwStream::init($ctx);
-	my $obuf = $ctx->{obuf} = _msg_page_prepare_obuf($eml, $ctx);
+	my $obuf = _msg_page_prepare_obuf($eml, $ctx);
 	if (length($$obuf)) {
 		multipart_text_as_html($eml, $ctx);
 		$$obuf .= '</pre><hr>';
@@ -661,9 +661,9 @@ sub add_text_body { # callback for each_part
 
 sub _msg_page_prepare_obuf {
 	my ($eml, $ctx) = @_;
-	my $over = $ctx->{ibx}->over;
+	my $have_over = !!$ctx->{ibx}->over;
 	my $obfs_ibx = $ctx->{-obfs_ibx};
-	my $rv = '';
+	$ctx->{obuf} = \(my $rv = '');
 	my $mids = mids_for_index($eml);
 	my $nr = $ctx->{nr}++;
 	if ($nr) { # unlikely
@@ -672,14 +672,13 @@ sub _msg_page_prepare_obuf {
 			return \$rv;
 		}
 		$rv .=
-"<pre>WARNING: multiple messages have this Message-ID\n</pre>";
-		$rv .= '<pre>';
+"<pre>WARNING: multiple messages have this Message-ID\n</pre><pre>";
 	} else {
 		$ctx->{first_hdr} = $eml->header_obj;
 		$ctx->{chash} = content_hash($eml) if $ctx->{smsg}; # reused MID
 		$rv .= "<pre\nid=b>"; # anchor for body start
 	}
-	$ctx->{-upfx} = '../' if $over;
+	$ctx->{-upfx} = '../' if $have_over;
 	my @title; # (Subject[0], From[0])
 	for my $v ($eml->header('From')) {
 		my @n = PublicInbox::Address::names($v);
@@ -704,7 +703,7 @@ sub _msg_page_prepare_obuf {
 		my $v = ascii_html(shift @subj);
 		obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
 		$rv .= 'Subject: ';
-		$rv .= $over ? qq(<a\nhref="#r"\nid=t>$v</a>\n) : "$v\n";
+		$rv .= $have_over ? qq(<a\nhref="#r"\nid=t>$v</a>\n) : "$v\n";
 		$title[0] = $v;
 		for $v (@subj) { # multi-Subject message :<
 			$v = ascii_html($v);
@@ -712,7 +711,7 @@ sub _msg_page_prepare_obuf {
 			$rv .= "Subject: $v\n";
 		}
 	} else { # dummy anchor for thread skeleton at bottom of page
-		$rv .= qq(<a\nhref="#r"\nid=t></a>) if $over;
+		$rv .= qq(<a\nhref="#r"\nid=t></a>) if $have_over;
 		$title[0] = '(no subject)';
 	}
 	for my $v ($eml->header('Date')) {
@@ -724,22 +723,22 @@ sub _msg_page_prepare_obuf {
 		$ctx->{-title_html} = join(' - ', @title);
 		$rv = $ctx->html_top . $rv;
 	}
+
+	$ctx->{-linkify} //= PublicInbox::Linkify->new;
 	if (scalar(@$mids) == 1) { # common case
 		my $mhtml = ascii_html($mids->[0]);
-		$rv .= "Message-ID: &lt;$mhtml&gt; ";
-		$rv .= "(<a\nhref=\"raw\">raw</a>)\n";
+		$rv .= qq[Message-ID: &lt;$mhtml&gt; (<a href="raw">raw</a>)\n];
 	} else {
 		# X-Alt-Message-ID can happen if a message is injected from
 		# public-inbox-nntpd because of multiple Message-ID headers.
-		my $lnk = PublicInbox::Linkify->new;
 		my $s = '';
 		for my $h (qw(Message-ID X-Alt-Message-ID)) {
 			$s .= "$h: $_\n" for ($eml->header_raw($h));
 		}
-		$lnk->linkify_mids('..', \$s, 1);
+		$ctx->{-linkify}->linkify_mids('..', \$s, 1);
 		$rv .= $s;
 	}
-	$rv .= _parent_headers($eml, $over);
+	_parent_headers($ctx, $eml);
 	$rv .= "\n";
 	\$rv;
 }
@@ -792,35 +791,35 @@ sub thread_skel ($$$) {
 }
 
 sub _parent_headers {
-	my ($hdr, $over) = @_;
-	my $rv = '';
+	my ($ctx, $hdr) = @_;
 	my @irt = $hdr->header_raw('In-Reply-To');
 	my $refs;
 	if (@irt) {
-		my $lnk = PublicInbox::Linkify->new;
-		$rv .= "In-Reply-To: $_\n" for @irt;
-		$lnk->linkify_mids('..', \$rv);
+		my $s = '';
+		$s .= "In-Reply-To: $_\n" for @irt;
+		$ctx->{-linkify}->linkify_mids('..', \$s);
+		${$ctx->{obuf}} .= $s;
 	} else {
 		$refs = references($hdr);
 		my $irt = pop @$refs;
 		if (defined $irt) {
 			my $html = ascii_html($irt);
 			my $href = mid_href($irt);
-			$rv .= "In-Reply-To: &lt;";
-			$rv .= "<a\nhref=\"../$href/\">$html</a>&gt;\n";
+			${$ctx->{obuf}} .= <<EOM;
+In-Reply-To: &lt;<a\nhref="../$href/">$html</a>&gt;
+EOM
 		}
 	}
 
 	# do not display References: if search is present,
 	# we show the thread skeleton at the bottom, instead.
-	return $rv if $over;
+	return if $ctx->{ibx}->over;
 
 	$refs //= references($hdr);
 	if (@$refs) {
-		@$refs = map { linkify_ref_no_over($_) } @$refs;
-		$rv .= 'References: '. join("\n\t", @$refs) . "\n";
+		$_ = linkify_ref_no_over($_) for @$refs;
+		${$ctx->{obuf}} .= 'References: '. join("\n\t", @$refs) . "\n";
 	}
-	$rv;
 }
 
 # returns a string buffer
diff --git a/t/plack.t b/t/plack.t
index 20f5d8d5..32209c7d 100644
--- a/t/plack.t
+++ b/t/plack.t
@@ -155,7 +155,7 @@ my $c1 = sub {
 	is(200, $res->code, "success for $path");
 	my $html = $res->content;
 	like($html, qr!<title>hihi - Me</title>!, 'HTML returned');
-	like($html, qr!<a\nhref="raw"!s, 'raw link present');
+	like($html, qr!<a\nhref=raw!s, 'raw link present');
 	like($html, qr!&gt; quoted text!s, 'quoted text inline');
 
 	$path .= 'f/';

[PATCH 12/18] www: atom: fix "changed" href to nowhere

[Eric Wong]

The HTML generated for the Atom feed doesn't have the footer
of /T/ and /t/ HTML-only views, so just make "changed" in
the diffstat go directly to the permalink #related anchor.

Fixes: 66512e177390 ("view: generate query in single-message and commit views")
---
 lib/PublicInbox/View.pm          | 2 +-
 lib/PublicInbox/ViewDiff.pm      | 4 ++--
 lib/PublicInbox/WwwAtomStream.pm | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 354cdd93..86f4a467 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -248,7 +248,7 @@ sub eml_entry {
 
 	# scan through all parts, looking for displayable text
 	$ctx->{mhref} = $mhref;
-	$ctx->{end_id} = "e$id";
+	$ctx->{changed_href} = "#e$id"; # for diffstat "files? changed,"
 	$ctx->{obuf} = \$rv;
 	$eml->each_part(\&add_text_body, $ctx, 1);
 	delete $ctx->{obuf};
diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index ee2d688c..8c1853e6 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -180,8 +180,8 @@ sub diff_before_or_after ($$) {
 			$$dst .= $linkify->to_html($l);
 		}
 		$$dst .= $x[2]; # $3 /^ \d+ files? /
-		my $end = $ctx->{end_id} // 'related';
-		$$dst .= "<a href=#$end>changed</a>,";
+		my $ch = $ctx->{changed_href} // '#related';
+		$$dst .= qq(<a href="$ch">changed</a>,);
 		$$dst .= ascii_html($x[1]); # $4: insertions/deletions
 		$$dst .= $linkify->to_html($x[0]); # notes, commit message, etc
 	} else {
diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm
index 7b7047ac..09b6facb 100644
--- a/lib/PublicInbox/WwwAtomStream.pm
+++ b/lib/PublicInbox/WwwAtomStream.pm
@@ -156,6 +156,7 @@ sub feed_entry {
 		qq(<pre\nstyle="white-space:pre-wrap">);
 	$ctx->{obuf} = \$s;
 	$ctx->{mhref} = $href;
+	$ctx->{changed_href} = "${href}#related";
 	PublicInbox::View::multipart_text_as_html($eml, $ctx);
 	delete $ctx->{obuf};
 	$s .= '</pre></div></content></entry>';

[PATCH 13/18] www: provide text/help/#search anchor

[Eric Wong]

This allows jumping to the appropriate section of the "help"
from under the dfblob textarea search.
---
 lib/PublicInbox/View.pm    | 2 +-
 lib/PublicInbox/WwwText.pm | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 86f4a467..f29fff50 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -849,7 +849,7 @@ action=$upfx
 href=#t>this message</a>:
 <textarea name=q cols=${\COLS} rows=$rows>$q</textarea>
 <input type=submit value=search
-/>\t(<a href=${upfx}_/text/help/>help</a>)</pre></form>
+/>\t(<a href=${upfx}_/text/help/#search>help</a>)</pre></form>
 EOM
 	}
 	if ($ctx->{ibx}->over) {
diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm
index 3f840c44..1a0bb07a 100644
--- a/lib/PublicInbox/WwwText.pm
+++ b/lib/PublicInbox/WwwText.pm
@@ -67,6 +67,7 @@ sub get_text {
 		$txt = ascii_html($txt);
 	}
 	$txt = '<pre>' . $l->linkify_2($txt) . '</pre>';
+	$txt =~ s!^search$!<a\nid=search>search</a>!sm;
 	$txt =~ s!\bPOP3\b!<a\nid=pop3>POP3</a>!;
 	$txt =~ s!\b(Newsgroups?)\b!<a\nid=nntp>$1</a>!;
 	$txt =~ s!\bIMAP\b!<a\nid=imap>IMAP</a>!;

[PATCH 14/18] solver: early make hints detection more robust

[Eric Wong]

Hints fields can change, so we'll use a simple boolean rather
than checking a static count.  We'll also short-circuit out
reliably regardless of hints when a full OID is given.
---
 lib/PublicInbox/SolverGit.pm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index 52dfaf3d..1e96d042 100644
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -81,9 +81,8 @@ sub solve_existing ($$) {
 	my $oid_b = $want->{oid_b};
 	my ($oid_full, $type, $size) = $git->check($oid_b);
 
-	# other than {oid_b, try_gits, try_ibxs}
-	my $have_hints = scalar keys %$want > 3;
-	if (defined($type) && (!$have_hints || $type eq 'blob')) {
+	if ($oid_b eq ($oid_full // '') || (defined($type) &&
+-				(!$self->{have_hints} || $type eq 'blob'))) {
 		delete $want->{try_gits};
 		return [ $git, $oid_full, $type, int($size) ]; # done, success
 	}
@@ -683,6 +682,7 @@ sub solve ($$$$$) {
 	$self->{seen_oid} = {};
 	$self->{tot} = 0;
 	$self->{psgi_env} = $env;
+	$self->{have_hints} = 1 if scalar keys %$hints;
 	$self->{todo} = [ { %$hints, oid_b => $oid_want } ];
 	$self->{patches} = []; # [ $di, $di, ... ]
 	$self->{found} = {}; # { abbr => [ ::Git, oid, type, size, $di ] }

[PATCH 15/18] viewvcs: add tree view

[Eric Wong]

This also includes some glossary definitions to help users
unfamiliar with git understand the relationship between
trees and blobs.
---
 lib/PublicInbox/ViewDiff.pm | 14 +++---
 lib/PublicInbox/ViewVCS.pm  | 89 +++++++++++++++++++++++++++++++++++--
 2 files changed, 94 insertions(+), 9 deletions(-)

diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index 8c1853e6..f16c7229 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -10,12 +10,11 @@ package PublicInbox::ViewDiff;
 use strict;
 use v5.10.1;
 use parent qw(Exporter);
-our @EXPORT_OK = qw(flush_diff);
+our @EXPORT_OK = qw(flush_diff uri_escape_path);
 use URI::Escape qw(uri_escape_utf8);
 use PublicInbox::Hval qw(ascii_html to_attr);
 use PublicInbox::Git qw(git_unquote);
 
-my $UNSAFE = "^A-Za-z0-9\-\._~/"; # '/' + $URI::Escape::Unsafe{RFC3986}
 my $OID_NULL = '0{7,}';
 my $OID_BLOB = '[a-f0-9]{7,}';
 my $LF = qr!\n!;
@@ -41,6 +40,11 @@ our $EXTRACT_DIFFS = qr/(
 		^\+{3}\x20($FN)$LF)/msx;
 our $IS_OID = qr/\A$OID_BLOB\z/s;
 
+sub uri_escape_path {
+	# '/' + $URI::Escape::Unsafe{RFC3986}
+	uri_escape_utf8($_[0], "^A-Za-z0-9\-\._~/");
+}
+
 # link to line numbers in blobs
 sub diff_hunk ($$$$) {
 	my ($dst, $dctx, $ca, $cb) = @_;
@@ -123,14 +127,14 @@ sub diff_header ($$$) {
 	$pa = (split(m'/', git_unquote($pa), 2))[1] if $pa ne '/dev/null';
 	$pb = (split(m'/', git_unquote($pb), 2))[1] if $pb ne '/dev/null';
 	if ($pa eq $pb && $pb ne '/dev/null') {
-		$dctx->{Q} = "?b=".uri_escape_utf8($pb, $UNSAFE);
+		$dctx->{Q} = '?b='.uri_escape_path($pb);
 	} else {
 		my @q;
 		if ($pb ne '/dev/null') {
-			push @q, 'b='.uri_escape_utf8($pb, $UNSAFE);
+			push @q, 'b='.uri_escape_path($pb);
 		}
 		if ($pa ne '/dev/null') {
-			push @q, 'a='.uri_escape_utf8($pa, $UNSAFE);
+			push @q, 'a='.uri_escape_path($pa);
 		}
 		$dctx->{Q} = '?'.join('&', @q);
 	}
diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index fd95e24e..a5545bcd 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -21,7 +21,7 @@ use PublicInbox::GitAsyncCat;
 use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::Linkify;
 use PublicInbox::Tmpfile;
-use PublicInbox::ViewDiff qw(flush_diff);
+use PublicInbox::ViewDiff qw(flush_diff uri_escape_path);
 use PublicInbox::View;
 use PublicInbox::Eml;
 use Text::Wrap qw(wrap);
@@ -37,6 +37,14 @@ my $BIN_DETECT = 8000; # same as git
 my $SHOW_FMT = '--pretty=format:'.join('%n', '%P', '%p', '%H', '%T', '%s', '%f',
 	'%an <%ae>  %ai', '%cn <%ce>  %ci', '%b%x00');
 
+my %GIT_MODE = (
+	'100644' => ' ', # blob
+	'100755' => 'x', # executable blob
+	'040000' => 'd', # tree
+	'120000' => 'l', # symlink
+	'160000' => 'g', # commit (gitlink)
+);
+
 sub html_page ($$;@) {
 	my ($ctx, $code) = @_[0, 1];
 	my $wcb = delete $ctx->{-wcb};
@@ -57,7 +65,7 @@ sub dbg_log ($) {
 		return '<pre>debug log read error</pre>';
 	};
 	$ctx->{-linkify} //= PublicInbox::Linkify->new;
-	'<pre>debug log:</pre><hr /><pre>'.
+	"<hr><pre>debug log:\n\n".
 		$ctx->{-linkify}->to_html($log).'</pre>';
 }
 
@@ -95,7 +103,7 @@ sub stream_large_blob ($$) {
 	$qsp->psgi_return($env, undef, \&stream_blob_parse_hdr, $ctx);
 }
 
-sub show_other_result ($$) { # tag, tree, ...
+sub show_other_result ($$) { # tag
 	my ($bref, $ctx) = @_;
 	if (my $qsp_err = delete $ctx->{-qsp_err}) {
 		return html_page($ctx, 500, dbg_log($ctx) .
@@ -296,7 +304,7 @@ sub show_other ($$) {
 	my ($ctx, $res) = @_;
 	my ($git, $oid, $type, $size) = @$res;
 	$size > $MAX_SIZE and return html_page($ctx, 200,
-				"$oid is too big to show\n". dbg_log($ctx));
+		ascii_html($type)." $oid is too big to show\n". dbg_log($ctx));
 	my $cmd = ['git', "--git-dir=$git->{git_dir}",
 		qw(show --encoding=UTF-8 --no-color --no-abbrev), $oid ];
 	my $qsp = PublicInbox::Qspawn->new($cmd);
@@ -304,6 +312,78 @@ sub show_other ($$) {
 	$qsp->psgi_qx($ctx->{env}, undef, \&show_other_result, $ctx);
 }
 
+sub show_tree_result ($$) {
+	my ($bref, $ctx) = @_;
+	if (my $qsp_err = delete $ctx->{-qsp_err}) {
+		return html_page($ctx, 500, dbg_log($ctx) .
+				"git ls-tree -z error:$qsp_err");
+	}
+	my @ent = split(/\0/, $$bref);
+	my $qp = delete $ctx->{qp};
+	my $l = $ctx->{-linkify} //= PublicInbox::Linkify->new;
+	my $pfx = $qp->{b};
+	$$bref = "<pre><a href=#tree>tree</a> $ctx->{tree_oid}";
+	if (defined $pfx) {
+		my $x = ascii_html($pfx);
+		$pfx .= '/';
+		$$bref .= qq(  <a href=#path>path</a>: $x</a>\n);
+	} else {
+		$pfx = '';
+		$$bref .= qq[  (<a href=#path>path</a> unknown)\n];
+	}
+	my ($x, $m, $t, $oid, $sz, $f, $n);
+	$$bref .= "\n	size	name";
+	for (@ent) {
+		($x, $f) = split(/\t/, $_, 2);
+		undef $_;
+		($m, $t, $oid, $sz) = split(/ +/, $x, 4);
+		$m = $GIT_MODE{$m} // '?';
+		utf8::decode($f);
+		$n = ascii_html($f);
+		if ($m eq 'g') { # gitlink submodule commit
+			$$bref .= "\ng\t\t$n @ <a\nhref=#g>commit</a>$oid";
+			next;
+		}
+		my $q = 'b='.ascii_html(uri_escape_path($pfx.$f));
+		if ($m eq 'd') { $n .= '/' }
+		elsif ($m eq 'x') { $n = "<b>$n</b>" }
+		elsif ($m eq 'l') { $n = "<i>$n</i>" }
+		$$bref .= qq(\n$m\t$sz\t<a\nhref="../../$oid/s/?$q">$n</a>);
+	}
+	$$bref .= dbg_log($ctx);
+	$$bref .= <<EOM;
+<pre>glossary
+--------
+<dfn
+id=tree>Tree</dfn> objects belong to commits or other tree objects.  Trees may
+reference blobs, sub-trees, or commits of submodules.
+
+<dfn
+id=path>Path</dfn> names are stored in tree objects, but trees do not know
+their own path name.  A tree's path name comes from their parent tree,
+or it is the root tree referenced by a commit object.  Thus, this web UI
+relies on the `b=' URI parameter as a hint to display the path name.
+
+<dfn title="submodule commit"
+id=g>Commit</dfn> objects may be stored in trees to reference submodules.</pre>
+EOM
+	chop $$bref;
+	html_page($ctx, 200, $$bref);
+}
+
+sub show_tree ($$) {
+	my ($ctx, $res) = @_;
+	my ($git, $oid, undef, $size) = @$res;
+	$size > $MAX_SIZE and return html_page($ctx, 200,
+			"tree $oid is too big to show\n". dbg_log($ctx));
+	my $cmd = [ 'git', "--git-dir=$git->{git_dir}",
+		qw(ls-tree -z -l --no-abbrev), $oid ];
+	my $qsp = PublicInbox::Qspawn->new($cmd);
+	$ctx->{tree_oid} = $oid;
+	$qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
+	$qsp->psgi_qx($ctx->{env}, undef, \&show_tree_result, $ctx);
+}
+
 # user_cb for SolverGit, called as: user_cb->($result_or_error, $uarg)
 sub solve_result {
 	my ($res, $ctx) = @_;
@@ -313,6 +393,7 @@ sub solve_result {
 
 	my ($git, $oid, $type, $size, $di) = @$res;
 	return show_commit($ctx, $res) if $type eq 'commit';
+	return show_tree($ctx, $res) if $type eq 'tree';
 	return show_other($ctx, $res) if $type ne 'blob';
 	my $path = to_filename($di->{path_b} // $hints->{path_b} // 'blob');
 	my $raw_link = "(<a\nhref=$path>raw</a>)";

[PATCH 16/18] viewvcs: reduce hash assignments for commit info

[Eric Wong]

This makes some of the associated code less verbose and
easier-to-read IMHO.
---
 lib/PublicInbox/ViewVCS.pm | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index a5545bcd..0a38f9de 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -136,8 +136,7 @@ sub show_commit_start { # ->psgi_qx callback
 	chop(my $buf = do { local $/ = "\0"; <$fh> });
 	chomp $buf;
 	my ($P, $p);
-	($P, $p, @$ctx{qw(cmt_H cmt_T cmt_s cmt_f cmt_au cmt_co cmt_b)})
-		= split(/\n/, $buf, 9);
+	($P, $p, @{$ctx->{cmt_info}}) = split(/\n/, $buf, 9);
 	return cmt_finalize($ctx) if !$P;
 	@{$ctx->{-cmt_P}} = split(/ /, $P);
 	@{$ctx->{-cmt_p}} = split(/ /, $p); # abbreviated
@@ -157,8 +156,8 @@ sub cmt_finalize {
 	my ($ctx) = @_;
 	$ctx->{-linkify} //= PublicInbox::Linkify->new;
 	my $upfx = $ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
+	my ($H, $T, $s, $f, $au, $co, $bdy) = @{delete $ctx->{cmt_info}};
 	# try to keep author and committer dates lined up
-	my ($au, $co) = delete @$ctx{qw(cmt_au cmt_co)};
 	my $x = length($au) - length($co);
 	if ($x > 0) {
 		$x = ' ' x $x;
@@ -175,13 +174,12 @@ sub cmt_finalize {
 href="$upfx?t=$t"
 title="list contemporary emails">$2</a>)
 		!e;
-	my $s = $ctx->{-linkify}->to_html(delete $ctx->{cmt_s});
-	$ctx->{-title_html} = $s;
+	$ctx->{-title_html} = $s = $ctx->{-linkify}->to_html($s);
 	my ($P, $p, $pt) = delete @$ctx{qw(-cmt_P -cmt_p -cmt_pt)};
 	$_ = qq(<a href="$upfx$_/s/">).shift(@$p).'</a> '.shift(@$pt) for @$P;
 	if (@$P == 1) {
 		$x = qq{ (<a
-href="$ctx->{cmt_f}.patch">patch</a>)\n   parent $P->[0]};
+href="$f.patch">patch</a>)\n   parent $P->[0]};
 	} elsif (@$P > 1) {
 		$x = qq(\n  parents $P->[0]\n);
 		shift @$P;
@@ -192,29 +190,28 @@ href="$ctx->{cmt_f}.patch">patch</a>)\n   parent $P->[0]};
 	}
 	PublicInbox::WwwStream::html_init($ctx);
 	$ctx->zmore(<<EOM);
-<pre>   commit $ctx->{cmt_H}$x
-     tree <a href="$upfx$ctx->{cmt_T}/s/">$ctx->{cmt_T}</a>
+<pre>   commit $H$x
+     tree <a href="$upfx$T/s/">$T</a>
    author $au
 committer $co
 
 <b>$s</b>
 EOM
-	$x = delete $ctx->{cmt_b};
-	$ctx->zmore("\n", $ctx->{-linkify}->to_html($x)) if length($x);
-	undef $x;
+	$ctx->zmore("\n", $ctx->{-linkify}->to_html($bdy)) if length($bdy);
+	$bdy = '';
 	open my $fh, '<:utf8', "$ctx->{-tmp}/p" or
 		die "open $ctx->{-tmp}/p: $!";
 	if (-s $fh > $MAX_SIZE) {
 		$ctx->zmore("---\n patch is too large to show\n");
 	} else { # prepare flush_diff:
-		$ctx->{obuf} = \$x;
+		read($fh, $x, -s _);
+		$ctx->{obuf} = \$bdy;
 		$ctx->{-apfx} = $ctx->{-spfx} = $upfx;
-		read($fh, my $bdy, -s _);
-		$bdy =~ s/\r?\n/\n/gs;
-		$ctx->{-anchors} = {} if $bdy =~ /^diff --git /sm;
-		flush_diff($ctx, \$bdy); # undefs $bdy
-		$ctx->zmore($x);
-		undef $x;
+		$x =~ s/\r?\n/\n/gs;
+		$ctx->{-anchors} = {} if $x =~ /^diff --git /sm;
+		flush_diff($ctx, \$x); # undefs $x
+		$ctx->zmore($bdy);
+		undef $bdy;
 		# TODO: should there be another textarea which attempts to
 		# search for the exact email which was applied to make this
 		# commit?

[PATCH 17/18] viewvcs: add glossary for commit

[Eric Wong]

IMHO, it's important to let users know on how commits tie trees
and parent commits together.
---
 lib/PublicInbox/ViewVCS.pm | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 0a38f9de..4c4cf3b3 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -179,19 +179,19 @@ title="list contemporary emails">$2</a>)
 	$_ = qq(<a href="$upfx$_/s/">).shift(@$p).'</a> '.shift(@$pt) for @$P;
 	if (@$P == 1) {
 		$x = qq{ (<a
-href="$f.patch">patch</a>)\n   parent $P->[0]};
+href="$f.patch">patch</a>)\n   <a href=#parent>parent</a> $P->[0]};
 	} elsif (@$P > 1) {
-		$x = qq(\n  parents $P->[0]\n);
+		$x = qq(\n  <a href=#parents>parents</a> $P->[0]\n);
 		shift @$P;
 		$x .= qq(          $_\n) for @$P;
 		chop $x;
 	} else {
-		$x = ' (root commit)';
+		$x = ' (<a href=#root_commit>root commit</a>)';
 	}
 	PublicInbox::WwwStream::html_init($ctx);
 	$ctx->zmore(<<EOM);
-<pre>   commit $H$x
-     tree <a href="$upfx$T/s/">$T</a>
+<pre>   <a href=#commit>commit</a> $H$x
+     <a href=#tree>tree</a> <a href="$upfx$T/s/">$T</a>
    author $au
 committer $co
 
@@ -238,7 +238,23 @@ id=related><pre>find related emails, including ancestors/descendants/conflicts
 EOM
 		}
 	}
-	$x = $ctx->zflush($ctx->_html_end);
+	chop($x = <<EOM);
+<pre>glossary
+--------
+<dfn
+id=commit>Commit</dfn> objects reference one tree, and zero or more parents.
+
+Single <dfn
+id=parent>parent</dfn> commits can typically generate a patch in
+unified diff format via `git format-patch'.
+
+Multiple <dfn id=parents>parents</dfn> means the commit is a merge.
+<dfn id=root_commit>Root commits</a> have no ancestor.  Note that it is
+possible to have multiple root commits when merging independent histories.
+
+Every commit references one top-level <dfn id=tree>tree</dfn> object.</pre>
+EOM
+	$x = $ctx->zflush($x, $ctx->_html_end);
 	my $res_hdr = delete $ctx->{-res_hdr};
 	push @$res_hdr, 'Content-Length', length($x);
 	delete($ctx->{env}->{'qspawn.wcb'})->([200, $res_hdr, [$x]]);

[19/18 PATCH] viewvcs: fixup commit glossary stuff

[Eric Wong]

Add <hr> before commit glossary since it's hard-to-tell where
the commit ends.  And put the "Root commit" definition into its
own line.

Ae also need to use </dfn> to close <dfn> tags :x

---
 lib/PublicInbox/ViewVCS.pm | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 8800afcb..3b4fa393 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -239,7 +239,7 @@ EOM
 		}
 	}
 	chop($x = <<EOM);
-<pre>glossary
+<hr><pre>glossary
 --------
 <dfn
 id=commit>Commit</dfn> objects reference one tree, and zero or more parents.
@@ -249,7 +249,8 @@ id=parent>parent</dfn> commits can typically generate a patch in
 unified diff format via `git format-patch'.
 
 Multiple <dfn id=parents>parents</dfn> means the commit is a merge.
-<dfn id=root_commit>Root commits</a> have no ancestor.  Note that it is
+
+<dfn id=root_commit>Root commits</dfn> have no ancestor.  Note that it is
 possible to have multiple root commits when merging independent histories.
 
 Every commit references one top-level <dfn id=tree>tree</dfn> object.</pre>

[PATCH 18/18] viewvcs: show "blob $OID" rather than "$OID blob"

[Eric Wong]

This is more consistent with the rest of the output where it's
"$TYPE $OID" rather than "$OID $TYPE".  The former also allows
easy copy+pasting into commands for both "git cat-file blob $OID"
and "lei blob $OID".
---
 lib/PublicInbox/ViewVCS.pm | 6 +++---
 t/solver_git.t             | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 4c4cf3b3..8800afcb 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -414,7 +414,7 @@ sub solve_result {
 		return stream_large_blob($ctx, $res) if defined $ctx->{fn};
 		return html_page($ctx, 200, <<EOM . dbg_log($ctx));
 <pre><b>Too big to show, download available</b>
-"$oid $type $size bytes $raw_link</pre>
+blob $oid $size bytes $raw_link</pre>
 EOM
 	}
 
@@ -433,7 +433,7 @@ EOM
 	}
 
 	$bin and return html_page($ctx, 200,
-				"<pre>$oid $type $size bytes (binary)" .
+				"<pre>blob $oid $size bytes (binary)" .
 				" $raw_link</pre>".dbg_log($ctx));
 
 	# TODO: detect + convert to ensure validity
@@ -449,7 +449,7 @@ EOM
 		$$blob = ascii_html($$blob);
 	}
 
-	my $x = "<pre>$oid $type $size bytes $raw_link</pre>" .
+	my $x = "<pre>blob $oid $size bytes $raw_link</pre>" .
 		"<hr /><table\nclass=blob>".
 		"<tr><td\nclass=linenumbers><pre>";
 	$x .= sprintf("<a id=n$_ href=#n$_>% ${pad}u</a>\n", $_) for (1..$nl);
diff --git a/t/solver_git.t b/t/solver_git.t
index 5c7bfa28..958af065 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -287,7 +287,8 @@ EOF
 		while (my ($label, $size) = each %bin) {
 			$res = $cb->(GET("/$name/$oid{$label}/s/"));
 			is($res->code, 200, "$label binary file");
-			ok(index($res->content, "blob $size bytes") >= 0,
+			ok(index($res->content,
+				"blob $oid{$label} $size bytes") >= 0,
 				"showed $label binary blob size");
 			$res = $cb->(GET("/$name/$oid{$label}/s/raw"));
 			is($res->code, 200, "$label raw binary download");