From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF, T_SCC_BODY_TEXT_LINE shortcircuit=no autolearn=ham autolearn_force=no version=3.4.6 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 4EB641F451 for ; Wed, 3 Jan 2024 10:23:15 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1704277395; bh=sjNMmD/zJqFjBocT3mUmnidxOfYWnZL99jxcQLHF/5c=; h=From:To:Subject:Date:From; b=ID73sj/B0znB3bKClz0sIyZOoZ75QD8KrWwlcYnv0sgLDlJwt3BrzRk669J2n1oi8 BifxNEEfObSvYJLfNXLHpVg+tlNEOo0p50pUtiiLMyNV9XOihWYtEfDuHXrOz7c/TO RhYy5DrJLFGaWgbKutMxXI96akJo+fb13RZglijQ= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] lei: MH: support inotify to detect updates Date: Wed, 3 Jan 2024 10:23:15 +0000 Message-ID: <20240103102315.2119260-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This should help us deal with MH sequence number packing and invalidating mail_sync.sqlite3. --- lib/PublicInbox/LEI.pm | 133 +++++++++++++++++--------------- lib/PublicInbox/LeiMailSync.pm | 10 ++- lib/PublicInbox/LeiNoteEvent.pm | 22 +++++- lib/PublicInbox/LeiWatch.pm | 7 +- lib/PublicInbox/MHreader.pm | 2 +- t/lei-watch.t | 12 ++- 6 files changed, 112 insertions(+), 74 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index e0cfd55a..81f940fe 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -28,7 +28,7 @@ use PublicInbox::IPC; use Time::HiRes qw(stat); # ctime comparisons for config cache use File::Path (); use File::Spec; -use Carp (); +use Carp qw(carp); use Sys::Syslog qw(openlog syslog closelog); our $quit = \&CORE::exit; our ($current_lei, $errors_log, $listener, $oldset, $dir_idle); @@ -38,7 +38,7 @@ my $GLP_PASS = Getopt::Long::Parser->new; $GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through)); our (%PATH2CFG, # persistent for socket daemon -$MDIR2CFGPATH, # /path/to/maildir => { /path/to/config => [ ino watches ] } +$MDIR2CFGPATH, # location => { /path/to/config => [ ino watches ] } $OPT, # shared between optparse and opt_dash callback (for Getopt::Long) $daemon_pid ); @@ -606,7 +606,7 @@ sub _lei_atfork_child { $dir_idle->force_close if $dir_idle; undef $dir_idle; %PATH2CFG = (); - $MDIR2CFGPATH = {}; + $MDIR2CFGPATH = undef; eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush'; undef $errors_log; $quit = \&CORE::exit; @@ -1252,32 +1252,43 @@ sub cfg2lei ($) { $lei; } +sub note_event ($@) { # runs lei_note_event for a given config file + my ($cfg_f, @args) = @_; + my $cfg = $PATH2CFG{$cfg_f} // return; + eval { cfg2lei($cfg)->dispatch('note-event', @args) }; + carp "E: note-event $cfg_f: $@\n" if $@; +} + sub dir_idle_handler ($) { # PublicInbox::DirIdle callback my ($ev) = @_; # Linux::Inotify2::Event or duck type my $fn = $ev->fullname; if ($fn =~ m!\A(.+)/(new|cur)/([^/]+)\z!) { # Maildir file - my ($mdir, $nc, $bn) = ($1, $2, $3); - $nc = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM; - for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) { - my $cfg = $PATH2CFG{$f} // next; - eval { - my $lei = cfg2lei($cfg); - $lei->dispatch('note-event', - "maildir:$mdir", $nc, $bn, $fn); - }; - warn "E: note-event $f: $@\n" if $@; + my ($loc, $new_cur, $bn) = ("maildir:$1", $2, $3); + $new_cur = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM; + for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) { + note_event($cfg_f, $loc, $new_cur, $bn, $fn); } - } + } elsif ($fn =~ m!\A(.+)/([0-9]+)\z!) { # MH mail message file + my ($loc, $n, $new_cur) = ("mh:$1", $2, '+'); + $new_cur = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM; + for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) { + note_event($cfg_f, $loc, $new_cur, $n, $fn); + } + } elsif ($fn =~ m!\A(.+)/\.mh_sequences\z!) { # reread flags + my $loc = "mh:$1"; + for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) { + note_event($cfg_f, $loc, '.mh_sequences') + } + } # else we don't care if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) { $ev->cancel; } if ($fn =~ m!\A(.+)/(?:new|cur)\z! && !-e $fn) { - delete $MDIR2CFGPATH->{$1}; + delete $MDIR2CFGPATH->{"maildir:$1"}; } - if (!-e $fn) { # config file or Maildir gone - for my $cfgpaths (values %$MDIR2CFGPATH) { - delete $cfgpaths->{$fn}; - } + if (!-e $fn) { # config file, Maildir, or MH dir gone + delete $_->{$fn} for values %$MDIR2CFGPATH; # config file + delete @$MDIR2CFGPATH{"maildir:$fn", "mh:$fn"}; delete $PATH2CFG{$fn}; } } @@ -1442,19 +1453,22 @@ sub watch_state_ok ($) { $state =~ /\Apause|(?:import|index|tag)-(?:ro|rw)\z/; } -sub cancel_maildir_watch ($$) { - my ($d, $cfg_f) = @_; - my $w = delete $MDIR2CFGPATH->{$d}->{$cfg_f}; - scalar(keys %{$MDIR2CFGPATH->{$d}}) or - delete $MDIR2CFGPATH->{$d}; - for my $x (@{$w // []}) { $x->cancel } +sub cancel_dir_watch ($$$) { + my ($type, $d, $cfg_f) = @_; + my $loc = "$type:".canonpath_harder($d); + my $w = delete $MDIR2CFGPATH->{$loc}->{$cfg_f}; + delete $MDIR2CFGPATH->{$loc} if !(keys %{$MDIR2CFGPATH->{$loc}}); + $_->cancel for @$w; } -sub add_maildir_watch ($$) { - my ($d, $cfg_f) = @_; - if (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) { - my @w = $dir_idle->add_watches(["$d/cur", "$d/new"], 1); - push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w; +sub add_dir_watch ($$$) { + my ($type, $d, $cfg_f) = @_; + $d = canonpath_harder($d); + my $loc = "$type:$d"; + my @dirs = $type eq 'mh' ? ($d) : ("$d/cur", "$d/new"); + if (!exists($MDIR2CFGPATH->{$loc}->{$cfg_f})) { + my @w = $dir_idle->add_watches(\@dirs, 1); + push @{$MDIR2CFGPATH->{$loc}->{$cfg_f}}, @w if @w; } } @@ -1467,24 +1481,20 @@ sub refresh_watches { my %seen; my $cfg_f = $cfg->{'-f'}; for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) { - my $url = substr($w, length('watch.'), -length('.state')); + my $loc = substr($w, length('watch.'), -length('.state')); require PublicInbox::LeiWatch; - $watches->{$url} //= PublicInbox::LeiWatch->new($url); - $seen{$url} = undef; - my $state = $cfg->get_1("watch.$url.state"); + $watches->{$loc} //= PublicInbox::LeiWatch->new($loc); + $seen{$loc} = undef; + my $state = $cfg->get_1("watch.$loc.state"); if (!watch_state_ok($state)) { - warn("watch.$url.state=$state not supported\n"); - next; - } - if ($url =~ /\Amaildir:(.+)/i) { - my $d = canonpath_harder($1); - if ($state eq 'pause') { - cancel_maildir_watch($d, $cfg_f); - } else { - add_maildir_watch($d, $cfg_f); - } + warn("watch.$loc.state=$state not supported\n"); + } elsif ($loc =~ /\A(maildir|mh):(.+)\z/i) { + my ($type, $d) = ($1, $2); + $state eq 'pause' ? + cancel_dir_watch($type, $d, $cfg_f) : + add_dir_watch($type, $d, $cfg_f); } else { # TODO: imap/nntp/jmap - $lei->child_error(0, "E: watch $url not supported, yet") + $lei->child_error(0, "E: watch $loc not supported, yet") } } @@ -1492,29 +1502,28 @@ sub refresh_watches { my $lms = $lei->lms; if ($lms) { $lms->lms_write_prepare; - for my $d ($lms->folders('maildir:')) { - substr($d, 0, length('maildir:')) = ''; - + for my $loc ($lms->folders(qr/\A(?:maildir|mh):/)) { + my $old = $loc; + my ($type, $d) = split /:/, $loc, 2; # fixup old bugs while we're iterating: - my $cd = canonpath_harder($d); - my $f = "maildir:$cd"; - $lms->rename_folder("maildir:$d", $f) if $d ne $cd; - next if $watches->{$f}; # may be set to pause + $d = canonpath_harder($d); + $loc = "$type:$d"; + $lms->rename_folder($old, $loc) if $old ne $loc; + next if $watches->{$loc}; # may be set to pause require PublicInbox::LeiWatch; - $watches->{$f} = PublicInbox::LeiWatch->new($f); - $seen{$f} = undef; - add_maildir_watch($cd, $cfg_f); + $watches->{$loc} = PublicInbox::LeiWatch->new($loc); + $seen{$loc} = undef; + add_dir_watch($type, $d, $cfg_f); } } if ($old) { # cull old non-existent entries - for my $url (keys %$old) { - next if exists $seen{$url}; - delete $old->{$url}; - if ($url =~ /\Amaildir:(.+)/i) { - my $d = canonpath_harder($1); - cancel_maildir_watch($d, $cfg_f); + for my $loc (keys %$old) { + next if exists $seen{$loc}; + delete $old->{$loc}; + if ($loc =~ /\A(maildir|mh):(.+)\z/i) { + cancel_dir_watch($1, $2, $cfg_f); } else { # TODO: imap/nntp/jmap - $lei->child_error(0, "E: watch $url TODO"); + $lei->child_error(0, "E: watch $loc TODO"); } } } diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm index 593715dc..c498421c 100644 --- a/lib/PublicInbox/LeiMailSync.pm +++ b/lib/PublicInbox/LeiMailSync.pm @@ -425,9 +425,13 @@ sub folders { my $re; if (defined($pfx[0])) { $sql .= ' WHERE loc REGEXP ?'; # DBD::SQLite uses perlre - $re = !!$pfx[1] ? '.*' : ''; - $re .= quotemeta($pfx[0]); - $re .= '.*'; + if (ref($pfx[0])) { # assume qr// "Regexp" + $re = $pfx[0]; + } else { + $re = !!$pfx[1] ? '.*' : ''; + $re .= quotemeta($pfx[0]); + $re .= '.*'; + } } my $sth = ($self->{dbh} //= dbh_new($self))->prepare($sql); $sth->bind_param(1, $re) if defined($re); diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm index 8581bd9a..8d900d0c 100644 --- a/lib/PublicInbox/LeiNoteEvent.pm +++ b/lib/PublicInbox/LeiNoteEvent.pm @@ -60,6 +60,18 @@ sub maildir_event { # via wq_nonblock_do } # else: eml_from_path already warns } +sub _mh_cb { # mh_read_one cb + my ($dir, $bn, $kw, $eml, $self, $state) = @_; +} + +sub mh_event { # via wq_nonblock_do + my ($self, $folder, $bn, $state) = @_; + my $dir = substr($folder, 3); + require PublicInbox::MHreader; # if we forked early + my $mhr = PublicInbox::MHreader->new($dir, $self->{lei}->{3}); + $mhr->mh_read_one($bn, \&_mh_cb, $self, $state); +} + sub lei_note_event { my ($lei, $folder, $new_cur, $bn, $fn, @rest) = @_; die "BUG: unexpected: @rest" if @rest; @@ -72,11 +84,14 @@ sub lei_note_event { $lms->arg2folder($lei, [ $folder ]); my $state = $cfg->get_1("watch.$folder.state") // 'tag-rw'; return if $state eq 'pause'; - return $lms->clear_src($folder, \$bn) if $new_cur eq ''; + if ($new_cur eq '') { + my $id = $folder =~ /\Amaildir:/ ? \$bn : $bn + 0; + return $lms->clear_src($folder, $id); + } $lms->lms_pause; $lei->ale; # prepare $sto->write_prepare($lei); - require PublicInbox::MdirReader; + require PublicInbox::MHreader if $folder =~ /\Amh:/; # optimistic my $self = $cfg->{-lei_note_event} //= do { my $wq = bless { lms => $lms }, __PACKAGE__; # MUAs such as mutt can trigger massive rename() storms so @@ -91,12 +106,15 @@ sub lei_note_event { $lei->{lne} = $wq; }; if ($folder =~ /\Amaildir:/i) { + require PublicInbox::MdirReader; my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn) // return; return if index($fl, 'T') >= 0; my $kw = PublicInbox::MdirReader::flags2kw($fl); my $vmd = { kw => $kw, sync_info => [ $folder, \$bn ] }; $self->wq_nonblock_do('maildir_event', $fn, $vmd, $state); + } elsif ($folder =~ /\Amh:/) { + $self->wq_nonblock_do('mh_event', $folder, $bn, $state); } # else: TODO: imap } diff --git a/lib/PublicInbox/LeiWatch.pm b/lib/PublicInbox/LeiWatch.pm index 35267b58..b30e5152 100644 --- a/lib/PublicInbox/LeiWatch.pm +++ b/lib/PublicInbox/LeiWatch.pm @@ -1,13 +1,12 @@ # Copyright all contributors # License: AGPL-3.0+ -# represents a Maildir or IMAP "watch" item +# represents a Maildir, MH or IMAP "watch" item package PublicInbox::LeiWatch; -use strict; -use v5.10.1; +use v5.12; use parent qw(PublicInbox::IPC); -# "url" may be something like "maildir:/path/to/dir" +# "url" may be something like "maildir:/path/to/dir" or "mh:/path/to/dir" sub new { bless { url => $_[1] }, $_[0] } 1; diff --git a/lib/PublicInbox/MHreader.pm b/lib/PublicInbox/MHreader.pm index 673e3e06..033aa740 100644 --- a/lib/PublicInbox/MHreader.pm +++ b/lib/PublicInbox/MHreader.pm @@ -82,7 +82,7 @@ sub kw_for ($$) { \@kw; } -sub _file2eml { # mh_each_file cb +sub _file2eml { # mh_each_file / mh_read_one cb my ($dir, $n, $self, $ucb, @arg) = @_; my $eml = eml_from_path($n); $ucb->($dir, $n, kw_for($self, $n), $eml, @arg) if $eml; diff --git a/t/lei-watch.t b/t/lei-watch.t index 7b357ee0..8ad50d13 100644 --- a/t/lei-watch.t +++ b/t/lei-watch.t @@ -3,6 +3,7 @@ # License: AGPL-3.0+ use strict; use v5.10.1; use PublicInbox::TestCommon; use File::Path qw(make_path remove_tree); +use PublicInbox::IO qw(write_file); plan skip_all => "TEST_FLAKY not enabled for $0" if !$ENV{TEST_FLAKY}; require_mods('lei'); my $have_fast_inotify = eval { require PublicInbox::Inotify } || @@ -13,7 +14,7 @@ $have_fast_inotify or my ($ro_home, $cfg_path) = setup_public_inboxes; test_lei(sub { - my $md = "$ENV{HOME}/md"; + my ($md, $mh1, $mh2) = map { "$ENV{HOME}/$_" } qw(md mh1 mh2); my $cfg_f = "$ENV{HOME}/.config/lei/config"; my $md2 = $md.'2'; lei_ok 'ls-watch'; @@ -45,13 +46,14 @@ test_lei(sub { } # first, make sure tag-ro works - make_path("$md/new", "$md/cur", "$md/tmp"); + make_path("$md/new", "$md/cur", "$md/tmp", $mh1, $mh2); lei_ok qw(add-watch --state=tag-ro), $md; lei_ok 'ls-watch'; like($lei_out, qr/^\Qmaildir:$md\E$/sm, 'maildir shown'); lei_ok qw(q mid:testmessage@example.com -o), $md, '-I', "$ro_home/t1"; my @f = glob("$md/cur/*:2,"); is(scalar(@f), 1, 'got populated maildir with one result'); + rename($f[0], "$f[0]S") or xbail "rename $!"; # set (S)een tick($have_fast_inotify ? 0.2 : 2.2); # always needed for 1 CPU systems lei_ok qw(note-event done); # flushes immediately (instead of 5s) @@ -94,6 +96,12 @@ test_lei(sub { my $cmp = [ <$fh> ]; is_xdeeply($cmp, $ino_contents, 'inotify Maildir watches gone'); }; + + write_file '>', "$mh1/.mh_sequences"; + lei_ok qw(add-watch --state=tag-ro), $mh1, "mh:$mh2"; + lei_ok 'ls-watch', \'refresh watches'; + like $lei_out, qr/^\Qmh:$mh1\E$/sm, 'MH 1 shown'; + like $lei_out, qr/^\Qmh:$mh2\E$/sm, 'MH 2 shown'; }); done_testing;