From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 2/3] lei q: fix warning on remote imports
Date: Sun, 21 Mar 2021 15:50:46 +0600 [thread overview]
Message-ID: <20210321095047.13855-3-e@80x24.org> (raw)
In-Reply-To: <20210321095047.13855-1-e@80x24.org>
This will let us tie keywords from remote externals
to those which only exist in local externals.
---
lib/PublicInbox/ContentHash.pm | 15 ++++++++++++---
lib/PublicInbox/LeiDedupe.pm | 9 ++-------
lib/PublicInbox/LeiXSearch.pm | 6 +++++-
t/lei-q-remote-import.t | 3 ++-
4 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/ContentHash.pm b/lib/PublicInbox/ContentHash.pm
index 4dbe7b50..112b1ea6 100644
--- a/lib/PublicInbox/ContentHash.pm
+++ b/lib/PublicInbox/ContentHash.pm
@@ -8,9 +8,9 @@
# See L<public-inbox-v2-format(5)> manpage for more details.
package PublicInbox::ContentHash;
use strict;
-use warnings;
-use base qw/Exporter/;
-our @EXPORT_OK = qw/content_hash content_digest/;
+use v5.10.1;
+use parent qw(Exporter);
+our @EXPORT_OK = qw(content_hash content_digest git_sha);
use PublicInbox::MID qw(mids references);
use PublicInbox::MsgIter;
@@ -94,4 +94,13 @@ sub content_hash ($) {
content_digest($_[0])->digest;
}
+sub git_sha ($$) {
+ my ($n, $eml) = @_;
+ my $dig = Digest::SHA->new($n);
+ my $buf = $eml->as_string;
+ $dig->add('blob '.length($buf)."\0");
+ $dig->add($buf);
+ $dig;
+}
+
1;
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index 5fec9384..a62b3a7c 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -3,7 +3,7 @@
package PublicInbox::LeiDedupe;
use strict;
use v5.10.1;
-use PublicInbox::ContentHash qw(content_hash);
+use PublicInbox::ContentHash qw(content_hash git_sha);
use Digest::SHA ();
# n.b. mutt sets most of these headers not sure about Bytes
@@ -18,12 +18,7 @@ sub _regen_oid ($) {
push @stash, [ $k, \@v ];
$eml->header_set($k); # restore below
}
- my $dig = Digest::SHA->new(1); # XXX SHA256 later
- my $buf = $eml->as_string;
- $dig->add('blob '.length($buf)."\0");
- $dig->add($buf);
- undef $buf;
-
+ my $dig = git_sha(1, $eml);
for my $kv (@stash) { # restore stashed headers
my ($k, @v) = @$kv;
$eml->header_set($k, @v);
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 17171a7f..b6aaf3e1 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -18,6 +18,7 @@ use PublicInbox::MID qw(mids);
use PublicInbox::Smsg;
use PublicInbox::Eml;
use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
+use PublicInbox::ContentHash qw(git_sha);
sub new {
my ($class) = @_;
@@ -207,10 +208,13 @@ sub query_mset { # non-parallel for non-"--threads" users
sub each_remote_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
- if ($self->{import_sto} && !$lei->{ale}->xoids_for($eml, 1)) {
+ my $xoids = $lei->{ale}->xoids_for($eml, 1);
+ if ($self->{import_sto} && !$xoids) {
$self->{import_sto}->ipc_do('add_eml', $eml);
}
my $smsg = bless {}, 'PublicInbox::Smsg';
+ $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
+ : git_sha(1, $eml)->hexdigest;
$smsg->populate($eml);
$smsg->parse_references($eml, mids($eml));
$smsg->{$_} //= '' for qw(from to cc ds subject references mid);
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 25e461ac..93828a24 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -65,8 +65,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
$im->add(eml_load('t/utf8.eml')) or BAIL_OUT '->add';
};
lei_ok(qw(add-external -q), $ibx->{inboxdir});
- lei_ok(qw(q -o), "mboxrd:$o", '--only', $url,
+ lei_ok(qw(q -q -o), "mboxrd:$o", '--only', $url,
'm:testmessage@example.com');
+ is($lei_err, '', 'no warnings or errors');
ok(-s $o, 'got result from remote external');
my $exp = eml_load('t/utf8.eml');
is_deeply($slurp_emls->($o), [$exp], 'got expected result');
next prev parent reply other threads:[~2021-03-21 9:50 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-03-21 9:50 [PATCH 0/3] lei import fix, other fixes Eric Wong
2021-03-21 9:50 ` [PATCH 1/3] lei import: vivify external-only messages Eric Wong
2021-03-21 9:50 ` Eric Wong [this message]
2021-03-21 9:50 ` [PATCH 3/3] lei: fix some warnings in tests Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210321095047.13855-3-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).