Eric Wong (5): address: pairs: enable pure Perl version initialize scalar for `vec' perlop modification t/thread-index-gap: support older DBI t/shared_kv: workaround old File::Spec extindex: fix w/ Xapian 1.2.21..1.2.24 lib/PublicInbox/Address.pm | 1 + lib/PublicInbox/ExtSearchIdx.pm | 1 + lib/PublicInbox/LEI.pm | 2 +- lib/PublicInbox/SearchIdx.pm | 5 ++++- lib/PublicInbox/Syscall.pm | 2 +- lib/PublicInbox/V2Writable.pm | 12 ++++++++++-- t/shared_kv.t | 10 +++------- t/thread-index-gap.t | 7 +++++-- 8 files changed, 26 insertions(+), 14 deletions(-)
Oops, this is needed for systems lacking Email::Address::XS --- lib/PublicInbox/Address.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/PublicInbox/Address.pm b/lib/PublicInbox/Address.pm index a090fa43..2c9c4395 100644 --- a/lib/PublicInbox/Address.pm +++ b/lib/PublicInbox/Address.pm @@ -37,6 +37,7 @@ if ($@) { require PublicInbox::AddressPP; *emails = \&PublicInbox::AddressPP::emails; *names = \&PublicInbox::AddressPP::names; + *pairs = \&PublicInbox::AddressPP::pairs; } 1;
From: Eric Wong <e@yhbt.net> Older Perls (tested 5.16.3) would warn on uninitialized scalars while newer (tested 5.28.1) do not. Just initialize it to an empty string since it'll be filled in by `vec'. --- lib/PublicInbox/LEI.pm | 2 +- lib/PublicInbox/Syscall.pm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 1f4a3082..2784ca6b 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -670,7 +670,7 @@ sub accept_dispatch { # Listener {post_accept} callback my ($sock) = @_; # ignore other $sock->autoflush(1); my $self = bless { sock => $sock }, __PACKAGE__; - vec(my $rvec, fileno($sock), 1) = 1; + vec(my $rvec = '', fileno($sock), 1) = 1; select($rvec, undef, undef, 1) or return send($sock, 'timed out waiting to recv FDs', MSG_EOR); my @fds = $recv_cmd->($sock, my $buf, 4096 * 33); # >MAX_ARG_STRLEN diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm index a1f53235..5ff1d65f 100644 --- a/lib/PublicInbox/Syscall.pm +++ b/lib/PublicInbox/Syscall.pm @@ -224,7 +224,7 @@ sub epoll_ctl_mod8 { # epoll_wait wrapper # ARGS: (epfd, maxevents, timeout (milliseconds), arrayref) # arrayref: values modified to be [$fd, $event] -our $epoll_wait_events; +our $epoll_wait_events = ''; our $epoll_wait_size = 0; sub epoll_wait_mod4 { my ($epfd, $maxevents, $timeout_msec, $events) = @_;
From: Eric Wong <e@yhbt.net> ...by avoiding selectall_array in favor of selectall_arrayref. Tested with DBI 1.627. --- t/thread-index-gap.t | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/t/thread-index-gap.t b/t/thread-index-gap.t index f4e4bd09..83c3707d 100644 --- a/t/thread-index-gap.t +++ b/t/thread-index-gap.t @@ -45,12 +45,15 @@ for my $msgs (['orig', reverse @msgs], ['shuffle', shuffle(@msgs)]) { } $im->done; my $over = $ibx->over; - my @tid = $over->dbh->selectall_array('SELECT DISTINCT(tid) FROM over'); + my $dbh = $over->dbh; + my $tid = $dbh->selectall_arrayref('SELECT DISTINCT(tid) FROM over'); + my @tid = map { $_->[0] } @$tid; is(scalar(@tid), 1, "only one thread initially ($desc)"); $over->dbh_close; run_script([qw(-index --reindex --rethread), $ibx->{inboxdir}]) or BAIL_OUT 'rethread'; - @tid = $over->dbh->selectall_array('SELECT DISTINCT(tid) FROM over'); + $tid = $dbh->selectall_arrayref('SELECT DISTINCT(tid) FROM over'); + @tid = map { $_->[0] } @$tid; is(scalar(@tid), 1, "only one thread after rethread ($desc)"); }
From: Eric Wong <e@yhbt.net> The version of File::Spec shipped with Perl 5.16.3 memoizes the value of File::Spec->tmpdir, causing changes to $ENV{TMPDIR} after-the-fact to be ignored. We'll only work around this in the test since it's innocuous and unlikely to matter in real-world usage (and there's many places where we'd have to workaround this in non-test code). --- t/shared_kv.t | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/t/shared_kv.t b/t/shared_kv.t index e7a8e05e..6f6374f2 100644 --- a/t/shared_kv.t +++ b/t/shared_kv.t @@ -9,9 +9,8 @@ use_ok 'PublicInbox::SharedKV'; my ($tmpdir, $for_destroy) = tmpdir(); local $ENV{TMPDIR} = $tmpdir; my $skv = PublicInbox::SharedKV->new; -opendir(my $dh, $tmpdir) or BAIL_OUT $!; -my @ent = grep(!/\A\.\.?\z/, readdir($dh)); -is(scalar(@ent), 1, 'created a temporary dir'); +my $skv_tmpdir = $skv->{tmpdir}; +ok(-d $skv_tmpdir, 'created a temporary dir'); $skv->dbh; my $dead = "\xde\xad"; my $beef = "\xbe\xef"; @@ -48,10 +47,7 @@ is($skv->delete_by_val($dead), 2, 'delete_by_val hits'); is($skv->delete_by_val($dead), 0, 'delete_by_val misses again'); undef $skv; -rewinddir($dh); -@ent = grep(!/\A\.\.?\z/, readdir($dh)); -is(scalar(@ent), 0, 'temporary dir gone'); -undef $dh; +ok(!-d $skv_tmpdir, 'temporary dir gone'); $skv = PublicInbox::SharedKV->new("$tmpdir/dir", 'base'); ok(-e "$tmpdir/dir/base.sqlite3", 'file created');
From: Eric Wong <e@yhbt.net> Xapian v1.2.21..v1.2.24 failed to set the close-on-exec flag on the flintlock FD, causing "git cat-file" processes to hold onto the lock and prevent subsequent Xapian::WritableDatabase from locking the DB. So cleanup git processes after committing the miscidx transaction. --- lib/PublicInbox/ExtSearchIdx.pm | 1 + lib/PublicInbox/SearchIdx.pm | 5 ++++- lib/PublicInbox/V2Writable.pm | 12 ++++++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm index 85959a95..c782a62a 100644 --- a/lib/PublicInbox/ExtSearchIdx.pm +++ b/lib/PublicInbox/ExtSearchIdx.pm @@ -1003,6 +1003,7 @@ sub idx_init { # similar to V2Writable $self->with_umask(\&_idx_init, $self, $opt); $self->{oidx}->begin_lazy; $self->{oidx}->eidx_prep; + $self->git->batch_prepare; $self->{midx}->begin_txn; } diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index adced076..7f7b980d 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -105,8 +105,11 @@ sub load_xapian_writable () { $DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()'); $DB_OPEN = eval($xap.'::DB_OPEN()'); my $ver = (eval($xap.'::major_version()') << 16) | - (eval($xap.'::minor_version()') << 8); + (eval($xap.'::minor_version()') << 8) | + eval($xap.'::revision()'); $DB_NO_SYNC = 0x4 if $ver >= 0x10400; + # Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks + $X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218; 1; } diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 54004fd7..0104f87a 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -17,6 +17,7 @@ use PublicInbox::InboxWritable; use PublicInbox::OverIdx; use PublicInbox::Msgmap; use PublicInbox::Spawn qw(spawn popen_rd run_die); +use PublicInbox::Search; use PublicInbox::SearchIdx qw(log2stack is_ancestor check_size is_bad_blob); use IO::Handle; # ->autoflush use File::Temp (); @@ -608,7 +609,11 @@ shard[$i] bad echo:$echo != $i waiting for txn commit } my $midx = $self->{midx}; # misc index - $midx->commit_txn if $midx; + if ($midx) { + $midx->commit_txn; + $PublicInbox::Search::X{CLOEXEC_UNSET} and + $self->git->cleanup; + } # last_commit is special, don't commit these until # Xapian shards are done: @@ -618,7 +623,10 @@ shard[$i] bad echo:$echo != $i waiting for txn commit $dbh->commit; $dbh->begin_work; } - $midx->begin_txn if $midx; + if ($midx) { + $self->git->batch_prepare; + $midx->begin_txn; + } } $self->{total_bytes} += $self->{transact_bytes}; $self->{transact_bytes} = 0;