unofficial mirror of meta@public-inbox.org
 help / color / mirror / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 04/11] lei import: start rearranging code for IMAP support
Date: Wed, 17 Feb 2021 09:07:00 -0100	[thread overview]
Message-ID: <20210217100707.6796-5-e@80x24.org> (raw)
In-Reply-To: <20210217100707.6796-1-e@80x24.org>

More to come in a later commit; some error handling and failure
modes will be trickier with IMAP due to authentication.
---
 lib/PublicInbox/LeiImport.pm | 74 +++++++++++++++++++++++++-----------
 lib/PublicInbox/NetReader.pm | 19 +++++++++
 2 files changed, 71 insertions(+), 22 deletions(-)

diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 8358d9d4..b25d7e97 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -29,26 +29,21 @@ sub import_done { # EOF callback for main daemon
 	$imp->wq_wait_old(\&import_done_wait, $lei);
 }
 
-sub call { # the main "lei import" method
-	my ($cls, $lei, @argv) = @_;
-	my $sto = $lei->_lei_store(1);
-	$sto->write_prepare($lei);
-	$lei->{opt}->{kw} //= 1;
+sub check_fmt ($;$) {
+	my ($lei, $f) = @_;
 	my $fmt = $lei->{opt}->{'format'};
-	my $self = $lei->{imp} = bless {}, $cls;
-	my @f;
-	for my $x (@argv) {
-		if (-f $x) { push @f, $x }
-		elsif (-d _) { require PublicInbox::MdirReader }
-	}
-	(@f && !$fmt) and
-		return $lei->fail("--format unset for regular file(s):\n@f");
-	if (@f && $fmt ne 'eml') {
-		require PublicInbox::MboxReader;
-		PublicInbox::MboxReader->can($fmt) or
-			return $lei->fail( "--format=$fmt unrecognized\n");
+	if (!$fmt) {
+		my $err = $f ? "regular file(s):\n@$f" : '--stdin';
+		return $lei->fail("--format unset for $err");
 	}
-	$self->{0} = $lei->{0} if $lei->{opt}->{stdin};
+	return 1 if $fmt eq 'eml';
+	require PublicInbox::MboxReader;
+	PublicInbox::MboxReader->can($fmt) ||
+				$lei->fail( "--format=$fmt unrecognized\n");
+}
+
+sub do_import {
+	my ($lei) = @_;
 	my $ops = {
 		'!' => [ $lei->can('fail_handler'), $lei ],
 		'x_it' => [ $lei->can('x_it'), $lei ],
@@ -56,14 +51,19 @@ sub call { # the main "lei import" method
 		'' => [ \&import_done, $lei ],
 	};
 	($lei->{pkt_op_c}, $lei->{pkt_op_p}) = PublicInbox::PktOp->pair($ops);
-	my $j = $lei->{opt}->{jobs} // scalar(@argv) || 1;
-	my $nproc = $self->detect_nproc;
-	$j = $nproc if $j > $nproc;
+	my $self = $lei->{imp};
+	my $j = $lei->{opt}->{jobs} // scalar(@{$self->{argv}}) || 1;
+	if (my $nrd = $lei->{nrd}) {
+		# $j = $nrd->net_concurrency($j); TODO
+	} else {
+		my $nproc = $self->detect_nproc;
+		$j = $nproc if $j > $nproc;
+	}
 	$self->wq_workers_start('lei_import', $j, $lei->oldset, {lei => $lei});
 	my $op = delete $lei->{pkt_op_c};
 	delete $lei->{pkt_op_p};
 	$self->wq_io_do('import_stdin', []) if $self->{0};
-	for my $x (@argv) {
+	for my $x (@{$self->{argv}}) {
 		$self->wq_io_do('import_path_url', [], $x);
 	}
 	$self->wq_close(1);
@@ -73,6 +73,36 @@ sub call { # the main "lei import" method
 	}
 }
 
+sub call { # the main "lei import" method
+	my ($cls, $lei, @argv) = @_;
+	my $sto = $lei->_lei_store(1);
+	$sto->write_prepare($lei);
+	$lei->{opt}->{kw} //= 1;
+	my $self = $lei->{imp} = bless { argv => \@argv }, $cls;
+	if ($lei->{opt}->{stdin}) {
+		@argv and return
+			$lei->fail("--stdin and locations (@argv) do not mix");
+		check_fmt($lei) or return;
+		$self->{0} = $lei->{0};
+	} else {
+		my @f;
+		for my $x (@argv) {
+			if (-f $x) { push @f, $x }
+			elsif (-d _) { require PublicInbox::MdirReader }
+			else {
+				require PublicInbox::NetReader;
+				$lei->{nrd} //= PublicInbox::NetReader->new;
+				$lei->{nrd}->add_url($x);
+			}
+		}
+		if (@f) { check_fmt($lei, \@f) or return }
+		if ($lei->{nrd} && (my @err = $lei->{nrd}->errors)) {
+			return $lei->fail(@err);
+		}
+	}
+	do_import($lei);
+}
+
 sub ipc_atfork_child {
 	my ($self) = @_;
 	$self->{lei}->lei_atfork_child;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index fa337bcd..1d053425 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -280,4 +280,23 @@ sub imap_common_init ($) {
 	$mics;
 }
 
+sub add_url {
+	my ($self, $arg) = @_;
+	if (my $url = imap_url($arg)) {
+		push @{$self->{imap_order}}, $url;
+	} else {
+		push @{$self->{unsupported_url}}, $arg;
+	}
+}
+
+sub errors {
+	my ($self) = @_;
+	if (my $u = $self->{unsupported_url}) {
+		return "Unsupported URL(s): @$u";
+	}
+	undef;
+}
+
+sub new { bless {}, shift };
+
 1;

  parent reply	other threads:[~2021-02-17 10:07 UTC|newest]

Thread overview: 23+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-02-17 10:06 [PATCH 00/11] lei IMAP read support Eric Wong
2021-02-17 10:06 ` [PATCH 01/11] lei: bless config Eric Wong
2021-02-17 10:06 ` [PATCH 02/11] watch: move imap_common_init to NetReader Eric Wong
2021-02-17 10:06 ` [PATCH 03/11] watch: connect to NNTP and IMAP in config order Eric Wong
2021-02-17 10:07 ` Eric Wong [this message]
2021-02-17 10:07 ` [PATCH 05/11] lei import: move check_input_format to lei Eric Wong
2021-02-17 10:07 ` [PATCH 06/11] tests: setup_public_inboxes: use IMAP-friendly newsgroups Eric Wong
2021-02-17 10:07 ` [PATCH 07/11] t/lei_to_mail: remove unnecessary arg passing Eric Wong
2021-02-17 10:07 ` [PATCH 08/11] lei convert: mail format conversion sub-command Eric Wong
2021-02-17 10:53   ` Eric Wong
2021-02-18 11:06     ` [PATCHv2 0/4] lei IMAP support take #2 Eric Wong
2021-02-18 11:06       ` [PATCHv2 1/4] lei convert: mail format conversion sub-command Eric Wong
2021-02-18 20:22         ` [PATCHv3 0/4] lei convert IMAP support Eric Wong
2021-02-18 20:22         ` [PATCHv3 1/4] lei convert: mail format conversion sub-command Eric Wong
2021-02-18 20:22         ` [PATCHv3 2/4] lei import: add IMAP and (maildir|mbox*):$PATHNAME support Eric Wong
2021-02-18 20:22         ` [PATCHv3 3/4] lei: consolidate the bulk of the IPC code Eric Wong
2021-02-18 20:22         ` [PATCHv3 4/4] lei: check for IMAP auth errors Eric Wong
2021-02-18 11:06       ` [PATCHv2 2/4] lei import: add IMAP and (maildir|mbox*):$PATHNAME support Eric Wong
2021-02-18 11:06       ` [PATCH (resend) 3/4] lei: consolidate the bulk of the IPC code Eric Wong
2021-02-18 11:06       ` [PATCHv2 4/4] lei: check for IMAP auth errors Eric Wong
2021-02-17 10:07 ` [PATCH 09/11] lei import: add IMAP, (maildir|mbox*):$PATHNAME support Eric Wong
2021-02-17 10:07 ` [PATCH 10/11] lei: consolidate the bulk of the IPC code Eric Wong
2021-02-17 10:07 ` [PATCH 11/11] lei: check for IMAP auth errors Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210217100707.6796-5-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).