[PATCH] mbox: stream entire thread, regardless of size

[PATCH] mbox: stream entire thread, regardless of size

[Eric Wong]

Since mbox is usually downloaded, support fetching infinitely large
responses via streaming.
---
 lib/PublicInbox/Mbox.pm   | 46 +++++++++++++++++++++++++++++-----------------
 lib/PublicInbox/Search.pm |  2 +-
 2 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 2ec5065..fc9df1a 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -9,27 +9,14 @@ use Fcntl qw(SEEK_SET);
 
 sub thread_mbox {
 	my ($ctx, $srch) = @_;
-	my $mid = mid_compressed($ctx->{mid});
-	my $res = $srch->get_thread($mid);
-	my $msgs = delete $res->{msgs};
-	require PublicInbox::GitCatFile;
-	require Email::Simple;
-	my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
-
 	sub {
-		my ($res) = @_; # Plack callback
-		my $w = $res->([200, [ 'Content-Type' => 'text/plain' ] ]);
-		while (defined(my $smsg = shift @$msgs)) {
-			my $msg = eval {
-				my $path = 'HEAD:' . mid2path($smsg->mid);
-				Email::Simple->new($git->cat_file($path));
-			};
-			emit($w, $msg) if $msg;
-		}
+		my ($response) = @_; # Plack callback
+		my $w = $response->([200, ['Content-Type' => 'text/plain']]);
+		emit_mbox($w, $ctx, $srch);
 	}
 }
 
-sub emit {
+sub emit_msg {
 	my ($fh, $simple) = @_; # Email::Simple object
 
 	# drop potentially confusing headers, ssoma already should've dropped
@@ -52,4 +39,29 @@ sub emit {
 	$fh->write($buf);
 }
 
+sub emit_mbox {
+	my ($fh, $ctx, $srch) = @_;
+
+	require PublicInbox::GitCatFile;
+	require Email::Simple;
+	my $mid = mid_compressed($ctx->{mid});
+	my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
+	my %opts = (offset => 0);
+	my $nr;
+	do {
+		my $res = $srch->get_thread($mid, \%opts);
+		my $msgs = $res->{msgs};
+		$nr = scalar @$msgs;
+		while (defined(my $smsg = shift @$msgs)) {
+			my $msg = eval {
+				my $p = 'HEAD:'.mid2path($smsg->mid);
+				Email::Simple->new($git->cat_file($p));
+			};
+			emit_msg($fh, $msg) if $msg;
+		}
+
+		$opts{offset} += $nr;
+	} while ($nr > 0);
+}
+
 1;
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index f004050..f1ff3a4 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -247,7 +247,7 @@ sub get_thread {
 	my $qtid = $qp->parse_query('thread:'.$smsg->thread_id);
 	my $qsub = $qp->parse_query('path:'.mid_compressed($smsg->path));
 	my $query = Search::Xapian::Query->new(OP_OR, $qtid, $qsub);
-	$self->do_enquire($query);
+	$self->do_enquire($query, $opts);
 }
 
 # private subs below
-- 
EW