[PATCH 02/13] www: use WwwStream for dumping thread and search views

[Eric Wong <e@80x24.org>]

This allows us the HTTP server to react to backpressure
from slow clients when writing.  As a side effect, this
also makes it easier for us to maintain a consistent
header/footer across our HTML.
---
 lib/PublicInbox/Feed.pm       |   3 +-
 lib/PublicInbox/SearchView.pm | 106 +++++++++++++++++++++---------------------
 lib/PublicInbox/View.pm       | 103 +++++++++++++++-------------------------
 lib/PublicInbox/WWW.pm        |   9 ++--
 lib/PublicInbox/WwwStream.pm  |  14 +++++-
 5 files changed, 108 insertions(+), 127 deletions(-)

diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm
index 8e23306..36802fa 100644
--- a/lib/PublicInbox/Feed.pm
+++ b/lib/PublicInbox/Feed.pm
@@ -168,12 +168,13 @@ sub emit_html_index {
 sub emit_index_nosrch {
 	my ($ctx, $state) = @_;
 	my $ibx = $ctx->{-inbox};
+	my $fh = $state->{fh};
 	my (undef, $last) = each_recent_blob($ctx, sub {
 		my ($path, $commit, $ts, $u, $subj) = @_;
 		$state->{first} ||= $commit;
 
 		my $mime = do_cat_mail($ibx, $path) or return 0;
-		PublicInbox::View::index_entry($mime, 0, $state);
+		$fh->write(PublicInbox::View::index_entry($mime, $state));
 		1;
 	});
 	$last;
diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index fbef411..488822e 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -8,12 +8,14 @@ use warnings;
 use PublicInbox::SearchMsg;
 use PublicInbox::Hval qw/ascii_html/;
 use PublicInbox::View;
-use PublicInbox::MID qw(mid2path mid_mime);
+use PublicInbox::MID qw(mid2path mid_mime mid_clean);
 use Email::MIME;
 require PublicInbox::Git;
 require PublicInbox::Thread;
 our $LIM = 50;
 
+sub noop {}
+
 sub sres_top_html {
 	my ($ctx) = @_;
 	my $q = PublicInbox::SearchQuery->new($ctx->{qp});
@@ -27,44 +29,46 @@ sub sres_top_html {
 		relevance => $q->{r},
 	};
 	my ($mset, $total);
-
 	eval {
-		$mset = $ctx->{srch}->query($q->{q}, $opts);
+		$mset = $ctx->{srch}->query($q->{'q'}, $opts);
 		$total = $mset->get_matches_estimated;
 	};
 	my $err = $@;
-	my $res = html_start($q, $ctx) . '<pre>';
+	ctx_prepare($q, $ctx);
+	my $cb;
 	if ($err) {
 		$code = 400;
-		$res .= err_txt($ctx, $err) . "</pre><hr /><pre>" . foot($ctx);
+		$ctx->{-html_tip} = '<pre>'.err_txt($ctx, $err).'</pre><hr />';
+		$cb = *noop;
 	} elsif ($total == 0) {
 		$code = 404;
-		$res .= "\n\n[No results found]</pre><hr /><pre>".foot($ctx);
+		$ctx->{-html_tip} = "<pre>\n[No results found]</pre><hr />";
+		$cb = *noop;
 	} else {
 		my $x = $q->{x};
 		return sub { adump($_[0], $mset, $q, $ctx) } if ($x eq 'A');
 
-		$res .= search_nav_top($mset, $q) . "\n\n";
+		$ctx->{-html_tip} = search_nav_top($mset, $q) . "\n\n";
 		if ($x eq 't') {
-			return sub { tdump($_[0], $res, $mset, $q, $ctx) };
+			$cb = mset_thread($ctx, $mset, $q);
+		} else {
+			$cb = mset_summary($ctx, $mset, $q);
 		}
-		dump_mset(\$res, $mset);
-		$res .= '</pre>' . search_nav_bot($mset, $q) .
-			"\n\n" . foot($ctx);
 	}
 
-	$res .= "</pre></body></html>";
-	[$code, ['Content-Type'=>'text/html; charset=UTF-8'], [$res]];
+	[ $code, ['Content-Type', 'text/html; charset=UTF-8'],
+		PublicInbox::WwwStream->new($ctx, $cb) ];
 }
 
 # display non-threaded search results similar to what users expect from
 # regular WWW search engines:
-sub dump_mset {
-	my ($res, $mset) = @_;
+sub mset_summary {
+	my ($ctx, $mset, $q) = @_;
 
 	my $total = $mset->get_matches_estimated;
 	my $pad = length("$total");
 	my $pfx = ' ' x $pad;
+	my $res = \($ctx->{-html_tip});
 	foreach my $m ($mset->items) {
 		my $rank = sprintf("%${pad}d", $m->get_rank + 1);
 		my $pct = $m->get_percent;
@@ -77,6 +81,8 @@ sub dump_mset {
 			$s . "</a></b>\n";
 		$$res .= "$pfx  - by $f @ $ts UTC [$pct%]\n\n";
 	}
+	$$res .= search_nav_bot($mset, $q);
+	*noop;
 }
 
 sub err_txt {
@@ -85,14 +91,14 @@ sub err_txt {
 	$u = PublicInbox::Hval::prurl($ctx->{cgi}->{env}, $u);
 	$err =~ s/^\s*Exception:\s*//; # bad word to show users :P
 	$err = ascii_html($err);
-	"\n\nBad query: <b>$err</b>\n" .
+	"\nBad query: <b>$err</b>\n" .
 		qq{See <a\nhref="$u">$u</a> for Xapian query syntax};
 }
 
 sub search_nav_top {
 	my ($mset, $q) = @_;
 
-	my $rv = "Search results ordered by [";
+	my $rv = "<pre>Search results ordered by [";
 	if ($q->{r}) {
 		my $d = $q->qs_html(r => 0);
 		$rv .= qq{<a\nhref="?$d">date</a>|<b>relevance</b>};
@@ -122,7 +128,7 @@ sub search_nav_bot {
 	my $o = $q->{o};
 	my $end = $o + $nr;
 	my $beg = $o + 1;
-	my $rv = "<hr /><pre>Results $beg-$end of $total";
+	my $rv = "</pre><hr /><pre>Results $beg-$end of $total";
 	my $n = $o + $LIM;
 
 	if ($n < $total) {
@@ -135,13 +141,11 @@ sub search_nav_bot {
 		my $qs = $q->qs_html(o => ($p > 0 ? $p : 0));
 		$rv .= qq{<a\nhref="?$qs"\nrel=prev>prev</a>};
 	}
-	$rv;
+	$rv .= '</pre>';
 }
 
-sub tdump {
-	my ($cb, $res, $mset, $q, $ctx) = @_;
-	my $fh = $cb->([200, ['Content-Type'=>'text/html; charset=UTF-8']]);
-	$fh->write($res .= '</pre>');
+sub mset_thread {
+	my ($ctx, $mset, $q) = @_;
 	my %pct;
 	my @m = map {
 		my $i = $_;
@@ -163,14 +167,14 @@ sub tdump {
 	} else { # order by time (default for threaded view)
 		$th->order(*PublicInbox::View::sort_ts);
 	}
-	my $skel = '';
+	my $skel = search_nav_bot($mset, $q). "<pre>";
+	my $inbox = $ctx->{-inbox};
 	my $state = {
-		-inbox => $ctx->{-inbox},
+		-inbox => $inbox,
 		anchor_idx => 1,
 		ctx => $ctx,
 		cur_level => 0,
 		dst => \$skel,
-		fh => $fh,
 		mapping => {},
 		pct => \%pct,
 		prev_attr => '',
@@ -179,42 +183,40 @@ sub tdump {
 		srch => $ctx->{srch},
 		upfx => './',
 	};
-	$ctx->{searchview} = 1;
+
 	PublicInbox::View::walk_thread($th, $state,
 		*PublicInbox::View::pre_thread);
 
-	PublicInbox::View::thread_entry($state, $_, 0) for @m;
-
-	$fh->write(search_nav_bot($mset, $q). "\n\n" . $skel . "\n" .
-			foot($ctx). '</pre></body></html>');
-
-	$fh->close;
-}
-
-sub foot {
-	my ($ctx) = @_;
-	my $foot = $ctx->{footer} || '';
-	qq{Back to <a\nhref=".">index</a>.\n$foot};
+	my $msgs = \@m;
+	my $mime;
+	sub {
+		return unless $msgs;
+		while ($mime = shift @$msgs) {
+			my $mid = mid_clean(mid_mime($mime));
+			$mime = $inbox->msg_by_mid($mid) and last;
+		}
+		if ($mime) {
+			$mime = Email::MIME->new($mime);
+			return PublicInbox::View::index_entry($mime, $state);
+		}
+		$msgs = undef;
+		$skel .= "\n</pre>";
+	};
 }
 
-sub html_start {
+sub ctx_prepare {
 	my ($q, $ctx) = @_;
 	my $qh = ascii_html($q->{'q'});
-	my $A = $q->qs_html(x => 'A', r => undef);
-	my $res = '<html><head>' . PublicInbox::Hval::STYLE .
-		"<title>$qh - search results</title>" .
-		qq{<link\nrel=alternate\ntitle="Atom feed"\n} .
-		qq!href="?$A"\ntype="application/atom+xml"/></head>! .
-		qq{<body><form\naction="">} .
-		qq{<input\nname=q\nvalue="$qh"\ntype=text />};
-
-	$res .= qq{<input\ntype=hidden\nname=r />} if $q->{r};
+	$ctx->{-q_value_html} = $qh;
+	$ctx->{-atom} = '?'.$q->qs_html(x => 'A', r => undef);
+	$ctx->{-title_html} = "$qh - search results";
+	my $extra = '';
+	$extra .= qq{<input\ntype=hidden\nname=r />} if $q->{r};
 	if (my $x = $q->{x}) {
 		$x = ascii_html($x);
-		$res .= qq{<input\ntype=hidden\nname=x\nvalue="$x" />};
+		$extra .= qq{<input\ntype=hidden\nname=x\nvalue="$x" />};
 	}
-
-	$res .= qq{<input\ntype=submit\nvalue=search /></form>};
+	$ctx->{-extra_form_html} = $extra;
 }
 
 sub adump {
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 65788db..a774feb 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -97,8 +97,7 @@ sub nr_to_s ($$$) {
 
 # this is already inside a <pre>
 sub index_entry {
-	my ($mime, $level, $state) = @_;
-	$state->{anchor_idx}++;
+	my ($mime, $state) = @_;
 	my $ctx = $state->{ctx};
 	my $srch = $ctx->{srch};
 	my $hdr = $mime->header_obj;
@@ -118,7 +117,7 @@ sub index_entry {
 	$subj = "<u\nid=u>$subj</u>" if $root_anchor eq $id_m;
 
 	my $ts = _msg_date($hdr);
-	my $rv = "<pre><a\nhref=#e$id\nid=$id_m>#</a> ";
+	my $rv = "<a\nhref=#e$id\nid=$id_m>#</a> ";
 	$rv .= $subj;
 	my $mhref = $path.$href.'/';
 	my $from = _hdr_names($hdr, 'From');
@@ -154,13 +153,7 @@ sub index_entry {
 	if (my $pct = $state->{pct}) { # used by SearchView.pm
 		$rv .= " [relevance $pct->{$mid_raw}%]";
 	}
-	$state->{fh}->write($rv .= "\n</pre>"); # '\n' for lynx
-}
-
-sub thread_html {
-	my ($ctx, $foot, $srch) = @_;
-	# $_[0] in sub is the Plack callback
-	sub { emit_thread_html($_[0], $ctx, $foot, $srch) }
+	$rv .= "\n\n";
 }
 
 sub walk_thread {
@@ -186,25 +179,26 @@ sub pre_thread  {
 	skel_dump($state, $level, $node);
 }
 
-# only private functions below.
-
-sub emit_thread_html {
-	my ($res, $ctx, $foot, $srch) = @_;
+sub thread_html {
+	my ($ctx) = @_;
 	my $mid = $ctx->{mid};
-	my $sres = $srch->get_thread($mid, { asc => 1 });
+	my $sres = $ctx->{srch}->get_thread($mid, { asc => 1 });
 	my $msgs = load_results($sres);
 	my $nr = $sres->{total};
-	return missing_thread($res, $ctx) if $nr == 0;
-	my $skel = '';
+	return missing_thread($ctx) if $nr == 0;
+	my $skel = '</pre><hr /><pre>';
+	$skel .= $nr == 1 ? 'only message in thread' : 'end of thread';
+	$skel .= ", back to <a\nhref=\"../../\">index</a>";
+	$skel .= "\n<a\nid=t>$nr+ messages in thread:</a> (download: ";
+	$skel .= "<a\nhref=\"../t.mbox.gz\">mbox.gz</a>";
+	$skel .= " / follow: <a\nhref=\"../t.atom\">Atom feed</a>)\n";
 	my $state = {
-		anchor_idx => 0,
 		ctx => $ctx,
 		cur_level => 0,
 		dst => \$skel,
 		mapping => {}, # mid -> reply count
 		prev_attr => '',
 		prev_level => 0,
-		res => $res,
 		root_anchor => anchor_for($mid),
 		seen => {},
 		srch => $ctx->{srch},
@@ -213,21 +207,28 @@ sub emit_thread_html {
 
 	walk_thread(thread_results($msgs), $state, *pre_thread);
 
-	thread_entry($state, $_, 0) for @$msgs;
-
-	# there could be a race due to a message being deleted in git
-	# but still being in the Xapian index:
-	my $fh = delete $state->{fh} or return missing_thread($res, $ctx);
-
-	my $next = @$msgs == 1 ? 'only message in thread' : 'end of thread';
-	$next .= ", back to <a\nhref=\"../../\">index</a>";
-	$next .= "\n<a\nid=t>$nr+ messages in thread:</a> (download: ";
-	$next .= "<a\nhref=\"../t.mbox.gz\">mbox.gz</a>";
-	$next .= " / follow: <a\nhref=\"../t.atom\">Atom feed</a>)\n";
-	$next .= $skel;
-	$fh->write('<hr /><pre>' . $next . "\n\n".
-			$foot .  '</pre></body></html>');
-	$fh->close;
+	# lazy load the full message from mini_mime:
+	my $inbox = $ctx->{-inbox};
+	my $mime;
+	while ($mime = shift @$msgs) {
+		$mime = $inbox->msg_by_mid(mid_clean(mid_mime($mime))) and last;
+	}
+	$mime = Email::MIME->new($mime);
+	$ctx->{-upfx} = '../../';
+	$ctx->{-title_html} = ascii_html($mime->header('Subject'));
+	$ctx->{-html_tip} = '<pre>'.index_entry($mime, $state);
+	$mime = undef;
+	my $body = PublicInbox::WwwStream->new($ctx, sub {
+		return unless $msgs;
+		while ($mime = shift @$msgs) {
+			$mid = mid_clean(mid_mime($mime));
+			$mime = $inbox->msg_by_mid($mid) and last;
+		}
+		return index_entry(Email::MIME->new($mime), $state) if $mime;
+		$msgs = undef;
+		$skel .= "</pre>";
+	});
+	[ 200, ['Content-Type', 'text/html; charset=UTF-8'], $body ];
 }
 
 sub multipart_text_as_html {
@@ -539,20 +540,6 @@ sub anchor_for {
 	'm' . id_compress($msgid, 1);
 }
 
-sub thread_html_head {
-	my ($hdr, $state) = @_;
-	my $res = delete $state->{res} or die "BUG: no Plack callback in {res}";
-	my $fh = $res->([200, ['Content-Type'=> 'text/html; charset=UTF-8']]);
-	$state->{fh} = $fh;
-
-	my $s = ascii_html($hdr->header('Subject'));
-	$fh->write("<html><head><title>$s</title>".
-		qq{<link\nrel=alternate\ntitle="Atom feed"\n} .
-		qq!href="../t.atom"\ntype="application/atom+xml"/>! .
-		PublicInbox::Hval::STYLE .
-		"</head><body>");
-}
-
 sub ghost_parent {
 	my ($upfx, $mid) = @_;
 	# 'subject dummy' is used internally by Mail::Thread
@@ -564,21 +551,6 @@ sub ghost_parent {
 	qq{[parent not found: &lt;<a\nhref="$upfx$href/">$html</a>&gt;]};
 }
 
-sub thread_entry {
-	my ($state, $mime, $level) = @_;
-
-	# lazy load the full message from mini_mime:
-	$mime = eval {
-		my $mid = mid_clean(mid_mime($mime));
-		$state->{ctx}->{-inbox}->msg_by_mid($mid);
-	} or return;
-	$mime = Email::MIME->new($mime);
-
-	thread_html_head($mime, $state) if $state->{anchor_idx} == 0;
-	index_entry($mime, $level, $state);
-	1;
-}
-
 sub indent_for {
 	my ($level) = @_;
 	INDENT x ($level - 1);
@@ -606,10 +578,9 @@ sub thread_results {
 }
 
 sub missing_thread {
-	my ($res, $ctx) = @_;
+	my ($ctx) = @_;
 	require PublicInbox::ExtMsg;
-
-	$res->(PublicInbox::ExtMsg::ext_msg($ctx))
+	PublicInbox::ExtMsg::ext_msg($ctx);
 }
 
 sub _msg_date {
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 984268e..196486f 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -233,12 +233,10 @@ sub get_mid_html {
 
 # /$INBOX/$MESSAGE_ID/t/
 sub get_thread {
-	my ($ctx, $flat) = @_;
-	my $srch = searcher($ctx) or return need_search($ctx);
+	my ($ctx) = @_;
+	searcher($ctx) or return need_search($ctx);
 	require PublicInbox::View;
-	my $foot = footer($ctx);
-	$ctx->{flat} = $flat;
-	PublicInbox::View::thread_html($ctx, $foot, $srch);
+	PublicInbox::View::thread_html($ctx);
 }
 
 sub ctx_get {
@@ -414,7 +412,6 @@ sub msg_page {
 	't.atom' eq $e and return get_thread_atom($ctx);
 	't.mbox' eq $e and return get_thread_mbox($ctx);
 	't.mbox.gz' eq $e and return get_thread_mbox($ctx, '.gz');
-	'T/' eq $e and return get_thread($ctx, 1);
 	'raw' eq $e and return get_mid_txt($ctx);
 
 	# legacy, but no redirect for compatibility:
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index 34f32c0..d2bf318 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -22,10 +22,20 @@ sub _html_top ($) {
 	my $title = $ctx->{-title_html} || $desc;
 	my $upfx = $ctx->{-upfx} || '';
 	my $atom = $ctx->{-atom} || $upfx.'new.atom';
+	my $tip = $ctx->{-html_tip} || '';
 	my $top = "<b>$desc</b> (<a\nhref=\"$atom\">Atom feed</a>)";
 	if ($obj->search) {
+		my $q_val = $ctx->{-q_value_html};
+		if (defined $q_val && $q_val ne '') {
+			$q_val = qq(\nvalue="$q_val" );
+		} else {
+			$q_val = '';
+		}
+		# XXX gross, for SearchView.pm
+		my $extra = $ctx->{-extra_form_html} || '';
 		$top = qq{<form\naction="$upfx"><pre>$top} .
-			  qq{ <input\nname=q\ntype=text />} .
+			  qq{ <input\nname=q\ntype=text$q_val/>} .
+			  $extra .
 			  qq{<input\ntype=submit\nvalue=search />} .
 			  q{</pre></form>}
 	} else {
@@ -35,7 +45,7 @@ sub _html_top ($) {
 		"<link\nrel=alternate\ntitle=\"Atom feed\"\n".
 		"href=\"$atom\"\ntype=\"application/atom+xml\"/>" .
 		PublicInbox::Hval::STYLE .
-		"</head><body>$top";
+		"</head><body>". $top . $tip;
 }
 
 sub _html_end {
-- 
EW