From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 29FE01F9FE for ; Thu, 25 Feb 2021 10:11:07 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 1/4] lei convert: support IMAP output and "-F eml" inputs Date: Thu, 25 Feb 2021 10:11:03 +0000 Message-Id: <20210225101106.12505-2-e@80x24.org> In-Reply-To: <20210225101106.12505-1-e@80x24.org> References: <20210225101106.12505-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: eml ("message/rfc822" MIME type) is supported by "lei import", so it probably makes sense to support via convert, at least for tests. And IMAP support is supported in "lei q -o $MFOLDER", so this only required renaming {nrd} => {net} and initializing outputs before augment preparation (creating the IMAP folder) --- lib/PublicInbox/LeiConvert.pm | 47 +++++++++++++++++++++++------------ lib/PublicInbox/LeiImport.pm | 1 - lib/PublicInbox/NetWriter.pm | 3 ++- t/lei-convert.t | 15 +++++++++++ xt/net_writer-imap.t | 4 +++ 5 files changed, 52 insertions(+), 18 deletions(-) diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm index a7e47871..32aa2edb 100644 --- a/lib/PublicInbox/LeiConvert.pm +++ b/lib/PublicInbox/LeiConvert.pm @@ -28,25 +28,35 @@ sub mdir_cb { $self->{wcb}->(undef, { kw => $kw }, $eml); } +sub convert_fh ($$$$) { + my ($self, $ifmt, $fh, $name) = @_; + if ($ifmt eq 'eml') { + my $buf = do { local $/; <$fh> } // + return $self->{lei}->child_error(1 << 8, <<""); +error reading $name: $! + + my $eml = PublicInbox::Eml->new(\$buf); + $self->{wcb}->(undef, { kw => [] }, $eml); + } else { + PublicInbox::MboxReader->$ifmt($fh, \&mbox_cb, $self); + } +} + sub do_convert { # via wq_do my ($self) = @_; my $lei = $self->{lei}; my $in_fmt = $lei->{opt}->{'in-format'}; my $mics; - if (my $nrd = $lei->{nrd}) { # may prompt user once - $nrd->{mics_cached} = $nrd->imap_common_init($lei); - $nrd->{nn_cached} = $nrd->nntp_common_init($lei); - } if (my $stdin = delete $self->{0}) { - PublicInbox::MboxReader->$in_fmt($stdin, \&mbox_cb, $self); + convert_fh($self, $in_fmt, $stdin, ''); } for my $input (@{$self->{inputs}}) { my $ifmt = lc($in_fmt // ''); if ($input =~ m!\Aimaps?://!) { - $lei->{nrd}->imap_each($input, \&net_cb, $self); + $lei->{net}->imap_each($input, \&net_cb, $self); next; } elsif ($input =~ m!\A(?:nntps?|s?news)://!) { - $lei->{nrd}->nntp_each($input, \&net_cb, $self); + $lei->{net}->nntp_each($input, \&net_cb, $self); next; } elsif ($input =~ s!\A([a-z0-9]+):!!i) { $ifmt = lc $1; @@ -54,7 +64,7 @@ sub do_convert { # via wq_do if (-f $input) { open my $fh, '<', $input or return $lei->fail("open $input: $!"); - PublicInbox::MboxReader->$ifmt($fh, \&mbox_cb, $self); + convert_fh($self, $ifmt, $fh, $input); } elsif (-d _) { PublicInbox::MdirReader::maildir_each_eml($input, \&mdir_cb, $self); @@ -72,11 +82,12 @@ sub call { # the main "lei convert" method $opt->{kw} //= 1; my $self = $lei->{cnv} = bless {}, $cls; my $in_fmt = $opt->{'in-format'}; - my ($nrd, @f, @d); + my (@f, @d); $opt->{dedupe} //= 'none'; my $ovv = PublicInbox::LeiOverview->new($lei, 'out-format'); $lei->{l2m} or return $lei->fail("output not specified or is not a mail destination"); + my $net = $lei->{net}; # NetWriter may be created by l2m $opt->{augment} = 1 unless $ovv->{dst} eq '/dev/stdout'; if ($opt->{stdin}) { @inputs and return $lei->fail("--stdin and @inputs do not mix"); @@ -88,8 +99,8 @@ sub call { # the main "lei convert" method my $input_path = $input; if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { require PublicInbox::NetReader; - $nrd //= PublicInbox::NetReader->new; - $nrd->add_url($input); + $net //= PublicInbox::NetReader->new; + $net->add_url($input); } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { my $ifmt = lc $1; if (($in_fmt // $ifmt) ne $ifmt) { @@ -117,12 +128,12 @@ sub call { # the main "lei convert" method require PublicInbox::MdirReader; } $self->{inputs} = \@inputs; - if ($nrd) { - if (my $err = $nrd->errors) { + if ($net) { + if (my $err = $net->errors) { return $lei->fail($err); } - $nrd->{quiet} = $opt->{quiet}; - $lei->{nrd} = $nrd; + $net->{quiet} = $opt->{quiet}; + $lei->{net} //= $net; } my $op = $lei->workers_start($self, 'lei_convert', 1, { '' => [ $lei->can('dclose'), $lei ] @@ -137,11 +148,15 @@ sub ipc_atfork_child { my $lei = $self->{lei}; $lei->lei_atfork_child; my $l2m = delete $lei->{l2m}; + if (my $net = $lei->{net}) { # may prompt user once + $net->{mics_cached} = $net->imap_common_init($lei); + $net->{nn_cached} = $net->nntp_common_init($lei); + } + $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb(); $l2m->pre_augment($lei); $l2m->do_augment($lei); $l2m->post_augment($lei); $self->{wcb} = $l2m->write_cb($lei); - $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb(); $self->SUPER::ipc_atfork_child; } diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index cbfb3127..13e817d0 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -7,7 +7,6 @@ use strict; use v5.10.1; use parent qw(PublicInbox::IPC); use PublicInbox::Eml; -use PublicInbox::InboxWritable qw(eml_from_path); use PublicInbox::PktOp qw(pkt_do); sub _import_eml { # MboxReader callback diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm index c68b0669..e26e9815 100644 --- a/lib/PublicInbox/NetWriter.pm +++ b/lib/PublicInbox/NetWriter.pm @@ -16,7 +16,8 @@ my %IMAPkw2flags; sub imap_append { my ($mic, $folder, $bref, $smsg, $eml) = @_; $bref //= \($eml->as_string); - $smsg //= bless { }, 'PublicInbox::Smsg'; + $smsg //= bless {}, 'PublicInbox::Smsg'; + bless($smsg, 'PublicInbox::Smsg') if ref($smsg) eq 'HASH'; $smsg->{ts} //= msg_timestamp($eml // PublicInbox::Eml->new($$bref)); my @f = map { $IMAPkw2flags{$_} } @{$smsg->{kw}}; $mic->append_string($folder, $$bref, "@f", $smsg->internaldate) or diff --git a/t/lei-convert.t b/t/lei-convert.t index 2ba62db3..20099f65 100644 --- a/t/lei-convert.t +++ b/t/lei-convert.t @@ -5,6 +5,7 @@ use strict; use v5.10.1; use PublicInbox::TestCommon; use PublicInbox::MboxReader; use PublicInbox::MdirReader; use PublicInbox::NetReader; +use PublicInbox::Eml; require_git 2.6; require_mods(qw(DBD::SQLite Search::Xapian Mail::IMAPClient Net::NNTP)); my ($tmpdir, $for_destroy) = tmpdir; @@ -84,5 +85,19 @@ test_lei({ tmpdir => $tmpdir }, sub { open $fh, '<', "$d/foo.mboxrd" or BAIL_OUT; my $exp = do { local $/; <$fh> }; is($out, $exp, 'stdin => stdout'); + + lei_ok qw(convert -F eml -o mboxcl2:/dev/stdout t/plack-qp.eml); + open $fh, '<', \$lei_out or BAIL_OUT; + @bar = (); + PublicInbox::MboxReader->mboxcl2($fh, sub { + my $eml = shift; + for my $h (qw(Status Content-Length Lines)) { + ok(defined($eml->header_raw($h)), + "$h defined for mboxcl2"); + $eml->header_set($h); + } + push @bar, $eml; + }); + is_deeply(\@bar, [ eml_load('t/plack-qp.eml') ], 'eml => mboxcl2'); }); done_testing; diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t index 64f822cf..da435926 100644 --- a/xt/net_writer-imap.t +++ b/xt/net_writer-imap.t @@ -138,6 +138,10 @@ test_lei(sub { $nwr->imap_each($folder_uri, $imap_slurp_all, my $empty = []); is(scalar(@$empty), 0, 'no results w/o augment'); + lei_ok qw(convert -F eml t/msg_iter-order.eml -o), $$folder_uri; + $nwr->imap_each($folder_uri, $imap_slurp_all, $empty = []); + is_deeply($empty, [ [ [], eml_load('t/msg_iter-order.eml') ] ], + 'converted to IMAP destination'); }); undef $cleanup; # remove temporary folder