[PATCH] notmuch new --new-tags=tags...

[PATCH] notmuch new --new-tags=tags...

[Anthony Towns]

Hi *,

The attached patch makes "notmuch new --new-tags=unread,new" set the
"unread" and "new" tags on any new mail it finds rather than "unread"
and "inbox". Or whatever other tags you happen to specify.

Signed-off-by: Anthony Towns <aj@erisian.com.au>
---
 NEWS          |    3 +++
 notmuch-new.c |   49 +++++++++++++++++++++++++++++++++++++++++++++----
 notmuch.1     |   19 ++++++++++++++++++-
 notmuch.c     |    7 ++++++-
 4 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/NEWS b/NEWS
index f29ac27..cbfae5a 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,6 @@
+The "notmuch new" command accepts a "--new-tags=tags..." option to override
+the default tags applied to new mail.
+
 Notmuch 0.1 (2010-04-05)
 ========================
 This is the first release of the notmuch mail system.
diff --git a/notmuch-new.c b/notmuch-new.c
index 44b50aa..77c99e0 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -32,6 +32,11 @@ typedef struct _filename_list {
     _filename_node_t **tail;
 } _filename_list_t;

+typedef struct _tag_list {
+    char *tag;
+    struct _tag_list *next;
+} _tag_list_t;
+
 typedef struct {
     int output_is_a_tty;
     int verbose;
@@ -41,6 +46,8 @@ typedef struct {
     int added_messages;
     struct timeval tv_start;

+    _tag_list_t *new_msg_tags;
+
     _filename_list_t *removed_files;
     _filename_list_t *removed_directories;
 } add_files_state_t;
@@ -93,11 +100,40 @@ _filename_list_add (_filename_list_t *list,
     list->tail = &node->next;
 }

+static _tag_list_t *
+_parse_tags (void *ctx, const char *orig_str)
+{
+    _tag_list_t *tag_head = NULL, **tag_tail = &tag_head;
+    char *dupe_str, *start, *comma;
+
+    dupe_str = talloc_strdup(ctx, orig_str);
+    start = dupe_str;
+
+    do {
+	comma = strchr(start, ',');
+	if (comma)
+	    *(comma++) = '\0';
+
+	if (*start != '\0') {
+	    *tag_tail = talloc(dupe_str, _tag_list_t);
+	    (*tag_tail)->tag = start;
+	    (*tag_tail)->next = NULL;
+	    tag_tail = &(*tag_tail)->next;
+ 	}
+
+	start = comma;
+    } while (start);
+
+    return tag_head;
+}
+
 static void
-tag_inbox_and_unread (notmuch_message_t *message)
+tag_new_message (add_files_state_t *state, notmuch_message_t *message)
 {
-    notmuch_message_add_tag (message, "inbox");
-    notmuch_message_add_tag (message, "unread");
+    _tag_list_t *cur;
+    for (cur = state->new_msg_tags; cur; cur = cur->next) {
+	notmuch_message_add_tag (message, cur->tag);
+    }
 }

 static void
@@ -412,7 +448,7 @@ add_files_recursive (notmuch_database_t *notmuch,
 	/* success */
 	case NOTMUCH_STATUS_SUCCESS:
 	    state->added_messages++;
-	    tag_inbox_and_unread (message);
+	    tag_new_message (state, message);
 	    break;
 	/* Non-fatal issues (go on to next file) */
 	case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
@@ -714,6 +750,7 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
     struct stat st;
     const char *db_path;
     char *dot_notmuch_path;
+    const char *new_msg_tags = "inbox,unread";
     struct sigaction action;
     _filename_node_t *f;
     int renamed_files, removed_files;
@@ -726,12 +763,16 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
     for (i = 0; i < argc && argv[i][0] == '-'; i++) {
 	if (STRNCMP_LITERAL (argv[i], "--verbose") == 0) {
 	    add_files_state.verbose = 1;
+	} else if (STRNCMP_LITERAL (argv[i], "--new-tags=") == 0) {
+	    new_msg_tags = argv[i]+strlen("--new-tags=");
 	} else {
 	    fprintf (stderr, "Unrecognized option: %s\n", argv[i]);
 	    return 1;
 	}
     }

+    add_files_state.new_msg_tags = _parse_tags(ctx, new_msg_tags);
+
     config = notmuch_config_open (ctx, NULL, NULL);
     if (config == NULL)
 	return 1;
diff --git a/notmuch.1 b/notmuch.1
index 86830f4..3bab17c 100644
--- a/notmuch.1
+++ b/notmuch.1
@@ -85,7 +85,7 @@ The
 command is used to incorporate new mail into the notmuch database.
 .RS 4
 .TP 4
-.B new
+.BR new " [options...]"

 Find and import any new messages to the database.

@@ -109,6 +109,23 @@ whenever new mail is delivered and you wish to
incorporate it into the
 database. These subsequent runs will be much quicker than the initial
 run.

+Supported options for
+.B new
+include:
+.RS 4
+.TP 4
+.BR \-\-new\-tags= tags...
+
+Set the listed tags (separated by commas) on new messages
+instead of the default
+.B "inbox"
+and
+.B "unread"
+tags.
+
+.RE
+.RS 4
+
 Invoking
 .B notmuch
 with no command argument will run
diff --git a/notmuch.c b/notmuch.c
index dcfda32..f7b16e3 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -126,7 +126,7 @@ command_t commands[] = {
       "\tInvoking notmuch with no command argument will run setup if\n"
       "\tthe setup command has not previously been completed." },
     { "new", notmuch_new_command,
-      "[--verbose]",
+      "[--verbose] [--new-tags=inbox,unread]",
       "Find and import new messages to the notmuch database.",
       "\tScans all sub-directories of the mail directory, performing\n"
       "\tfull-text indexing on new messages that are found. Each new\n"
@@ -148,6 +148,11 @@ command_t commands[] = {
       "\t\tVerbose operation. Shows paths of message files as\n"
       "\t\tthey are being indexed.\n"
       "\n"
+      "\t--new-tags=tags...\n"
+      "\n"
+      "\t\tSet the listed tags (separated by commas) on new messages\n"
+      "\t\tinstead of the default \"inbox\" and \"unread\" tags.\n"
+      "\n"
       "\tInvoking notmuch with no command argument will run new if\n"
       "\tthe setup command has previously been completed, but new has\n"
       "\tnot previously been run." },
-- 
1.7.0

Re: [PATCH] notmuch new --new-tags=tags...

[Dirk Hohndel]

On Sun, 11 Apr 2010 01:03:04 +1000, Anthony Towns <aj@erisian.com.au> wrote:
> Hi *,
> 
> The attached patch makes "notmuch new --new-tags=unread,new" set the
> "unread" and "new" tags on any new mail it finds rather than "unread"
> and "inbox". Or whatever other tags you happen to specify.

I really like this - I have a similar patch that hardcodes a +new tag
that I use to figure out which emails were imported during the last run
of notmuch new - and that I then clear as the final step in my initial
tagging script.  

But this is much cleaner and more generic.

/D

-- 
Dirk Hohndel
Intel Open Source Technology Center

Re: [PATCH] notmuch new --new-tags=tags...

[Sebastian Spaeth]

On 2010-04-10, Anthony Towns wrote:
> Hi *,
> 
> The attached patch makes "notmuch new --new-tags=unread,new" set the
> "unread" and "new" tags on any new mail it finds rather than "unread"
> and "inbox". Or whatever other tags you happen to specify.

Thanks for the patch. I can't comment on the code quality, but rather
than having to specify the set of new tags on the command line every
time, I think it would make more sense to put them in the notmuch config
file as this patch does:
id:1268432006-24333-2-git-send-email-bgamari.foss@gmail.com

Looking forward to configurable notmuch tags.

Sebastian

Re: [PATCH] notmuch new --new-tags=tags...

[Jameson Rollins]

[-- Attachment #1: Type: text/plain, Size: 2599 bytes --]

On Mon, 12 Apr 2010 10:00:37 +0200, "Sebastian Spaeth" <Sebastian@SSpaeth.de> wrote:
> On 2010-04-10, Anthony Towns wrote:
> > The attached patch makes "notmuch new --new-tags=unread,new" set the
> > "unread" and "new" tags on any new mail it finds rather than "unread"
> > and "inbox". Or whatever other tags you happen to specify.
> 
> Thanks for the patch. I can't comment on the code quality, but rather
> than having to specify the set of new tags on the command line every
> time, I think it would make more sense to put them in the notmuch config
> file as this patch does:
> id:1268432006-24333-2-git-send-email-bgamari.foss@gmail.com

I was thinking about this, and it seems to me that we really need is a
way to just specify which tags should be applied to new messages based
on search terms.  It's becoming pretty clear that most people are doing
some sort of post-notmuch-new tag processing to modify the tags of new
messages to suite their needs.  Why not just integrate this directly
into the notmuch-new processing itself?  It seems like if this was
integrated into notmuch-new directly, the entire processing of new
messages could be sped up considerably, so that one wouldn't have to
call multiple notmuch-new processes in succession.

I'm not sure exactly what the best way to handle it would be, but I can
imagine something like this:

[new-tags]
+sent -new -- from:jrollins@finestructure.net
+drafts -new -- folder:draft
+notmuch -- from:notmuch@notmuchmail.org
+unread +inbox -new -- tag:new

These are all just commands for "notmuch tag" that would be run on all
the new messages as they're processed.  Each new message would be given
"new" tag by default, and then the new tag commands would be run.  So it
would be the equivalent of running the following commands:

notmuch new --new-tags=new
notmuch tag +sent -new -- from:jrollins@finestructure.net
notmuch tag +drafts -new -- folder:draft
notmuch tag +notmuch -- from:notmuch@notmuchmail.org
notmuch tag +unread +inbox -- tag:new

This would make things much easier for everyone who is doing post-new
tag processing, which I think is probably most people.  And I'm sure it
could be made much more efficient (if coded properly) than running all
these notmuch commands in succession, especially for people who have a
lot of post-new tag processing to do.  Keeping the syntax identical to
the notmuch-tag command syntax would keep things simple as well.

Do people who do a lot of post-notmuch-new tag processing think
something like this would suite their needs?

jamie.

[-- Attachment #2: Type: application/pgp-signature, Size: 835 bytes --]

Re: [PATCH] notmuch new --new-tags=tags...

[Tomas Carnecky]

On 4/12/10 1:59 PM, Jameson Rollins wrote:
> On Mon, 12 Apr 2010 10:00:37 +0200, "Sebastian Spaeth"<Sebastian@SSpaeth.de>  wrote:
>> On 2010-04-10, Anthony Towns wrote:
>>> The attached patch makes "notmuch new --new-tags=unread,new" set the
>>> "unread" and "new" tags on any new mail it finds rather than "unread"
>>> and "inbox". Or whatever other tags you happen to specify.
>>
>> Thanks for the patch. I can't comment on the code quality, but rather
>> than having to specify the set of new tags on the command line every
>> time, I think it would make more sense to put them in the notmuch config
>> file as this patch does:
>> id:1268432006-24333-2-git-send-email-bgamari.foss@gmail.com
>
> I was thinking about this, and it seems to me that we really need is a
> way to just specify which tags should be applied to new messages based
> on search terms.  It's becoming pretty clear that most people are doing
> some sort of post-notmuch-new tag processing to modify the tags of new
> messages to suite their needs.  Why not just integrate this directly
> into the notmuch-new processing itself?  It seems like if this was
> integrated into notmuch-new directly, the entire processing of new
> messages could be sped up considerably, so that one wouldn't have to
> call multiple notmuch-new processes in succession.
>
> I'm not sure exactly what the best way to handle it would be, but I can
> imagine something like this:
>
> [new-tags]
> +sent -new -- from:jrollins@finestructure.net
> +drafts -new -- folder:draft
> +notmuch -- from:notmuch@notmuchmail.org
> +unread +inbox -new -- tag:new
>
> These are all just commands for "notmuch tag" that would be run on all
> the new messages as they're processed.  Each new message would be given
> "new" tag by default, and then the new tag commands would be run.  So it
> would be the equivalent of running the following commands:
>
> notmuch new --new-tags=new
> notmuch tag +sent -new -- from:jrollins@finestructure.net
> notmuch tag +drafts -new -- folder:draft
> notmuch tag +notmuch -- from:notmuch@notmuchmail.org
> notmuch tag +unread +inbox -- tag:new
>
> This would make things much easier for everyone who is doing post-new
> tag processing, which I think is probably most people.  And I'm sure it
> could be made much more efficient (if coded properly) than running all
> these notmuch commands in succession, especially for people who have a
> lot of post-new tag processing to do.  Keeping the syntax identical to
> the notmuch-tag command syntax would keep things simple as well.
>
> Do people who do a lot of post-notmuch-new tag processing think
> something like this would suite their needs?

I have a patch which adds support for hooks which are run when tags are 
added, removed or new messages added to notmuch. But perhaps the 
fork/exec overhead of running the hooks would slow the processing down 
too much.
See http://caurea.org/2009/12/22/a-different-approach-to-email-tagging/, 
though that didn't work out quite how I expected. Classifying spam/ham 
is easy (that's what dspam was written for), but patch/not-patch 
resulted in a lot false-positives, especially when people quote emails 
which included patches. Same with the 'notmuch' and 'xorg' tags: dspam 
had trouble figuring out to which mailing list Carl sent the email (he 
sends emails to both lists).

tom

Re: [PATCH] notmuch new --new-tags=tags...

[Scott Robinson]

Excerpts from Tomas Carnecky's message of Mon Apr 12 06:25:23 -0600 2010:
> I have a patch which adds support for hooks which are run when tags are 
> added, removed or new messages added to notmuch. But perhaps the 
> fork/exec overhead of running the hooks would slow the processing down 
> too much.

Rather than a fork/exec for every message, why not have a persistent
subprocess that receives message IDs (+ filenames?) and can then do
arbitrary work and return back the proper tags?

Re: [PATCH] notmuch new --new-tags=tags...

[Michal Sojka]

On Mon, 12 Apr 2010, Jameson Rollins wrote:
> On Mon, 12 Apr 2010 10:00:37 +0200, "Sebastian Spaeth" <Sebastian@SSpaeth.de> wrote:
> > On 2010-04-10, Anthony Towns wrote:
> > > The attached patch makes "notmuch new --new-tags=unread,new" set the
> > > "unread" and "new" tags on any new mail it finds rather than "unread"
> > > and "inbox". Or whatever other tags you happen to specify.
> > 
> > Thanks for the patch. I can't comment on the code quality, but rather
> > than having to specify the set of new tags on the command line every
> > time, I think it would make more sense to put them in the notmuch config
> > file as this patch does:
> > id:1268432006-24333-2-git-send-email-bgamari.foss@gmail.com
> 
> I was thinking about this, and it seems to me that we really need is a
> way to just specify which tags should be applied to new messages based
> on search terms.  It's becoming pretty clear that most people are doing
> some sort of post-notmuch-new tag processing to modify the tags of new
> messages to suite their needs.  Why not just integrate this directly
> into the notmuch-new processing itself?  It seems like if this was
> integrated into notmuch-new directly, the entire processing of new
> messages could be sped up considerably, so that one wouldn't have to
> call multiple notmuch-new processes in succession.

In my mailstore abstraction patches, I suppose that tags for new
messages are derived in some way from mailstore. The idea is, that once
we have a mail store which can store tags (e.g. git-based mails store),
the tags will be assigned on delivery by tools like procmail and notmuch
will only index that tags for faster searching. The problem is the it
will take some time to implement all of this and we want this
functionality now.

> 
> I'm not sure exactly what the best way to handle it would be, but I can
> imagine something like this:
> 
> [new-tags]
> +sent -new -- from:jrollins@finestructure.net
> +drafts -new -- folder:draft
> +notmuch -- from:notmuch@notmuchmail.org
> +unread +inbox -new -- tag:new
> 
> These are all just commands for "notmuch tag" that would be run on all
> the new messages as they're processed.  Each new message would be given
> "new" tag by default, and then the new tag commands would be run.  So it
> would be the equivalent of running the following commands:
> 
> notmuch new --new-tags=new
> notmuch tag +sent -new -- from:jrollins@finestructure.net
> notmuch tag +drafts -new -- folder:draft
> notmuch tag +notmuch -- from:notmuch@notmuchmail.org
> notmuch tag +unread +inbox -- tag:new

The problem I see with this approach is, that all notmuch searches are
build around Xapian. A simple implementation of the above would mean
that whenever you add a new message to the database, you would need to
execute a Xapian query to see if the new message matches your rule or
not. This would be, of course, very expensive.

If we do not want to use xapian for searching, we would have to
implement all the query parsing. building and whatever in notmuch and I
do not think it belongs there.

I do not know much about Xapian internals, so there might be a way of
doing this without reimplementing everything, so correct me if I'm
wrong.

-Michal

Re: [PATCH] notmuch new --new-tags=tags...

[Jameson Rollins]

[-- Attachment #1: Type: text/plain, Size: 1498 bytes --]

On Mon, 12 Apr 2010 17:11:24 +0200, Michal Sojka <sojkam1@fel.cvut.cz> wrote:
> In my mailstore abstraction patches, I suppose that tags for new
> messages are derived in some way from mailstore. The idea is, that once
> we have a mail store which can store tags (e.g. git-based mails store),
> the tags will be assigned on delivery by tools like procmail and notmuch
> will only index that tags for faster searching. The problem is the it
> will take some time to implement all of this and we want this
> functionality now.

This is a big effort, and not one that all notmuch users are really
interested in, so I would like to not see it specifically guide notmuch
development.  There are lots of features that would be nice to have
soon.

In any event, I wasn't suggesting we drop everything to do this.  I was
just curious what people thought of this approach.

> The problem I see with this approach is, that all notmuch searches are
> build around Xapian. A simple implementation of the above would mean
> that whenever you add a new message to the database, you would need to
> execute a Xapian query to see if the new message matches your rule or
> not. This would be, of course, very expensive.

This does sound like a potential issue.  I definitely don't understand
how new messages are added to the database.  I was mostly suggesting a
syntax for adding tag as new messages are added, though, not that an
actual xapian search term.  I don't know if they can be decoupled,
though.

jamie.

[-- Attachment #2: Type: application/pgp-signature, Size: 835 bytes --]

Re: [PATCH] notmuch new --new-tags=tags...

[Anthony Towns]

On Tue, Apr 13, 2010 at 01:55, Jameson Rollins
<jrollins@finestructure.net> wrote:
> On Mon, 12 Apr 2010 17:11:24 +0200, Michal Sojka <sojkam1@fel.cvut.cz> wrote:
>> The problem I see with this approach is, that all notmuch searches are
>> build around Xapian. ...
> This does sound like a potential issue.  I definitely don't understand
> how new messages are added to the database.  I was mostly suggesting a
> syntax for adding tag as new messages are added, though, not that an
> actual xapian search term.  I don't know if they can be decoupled,
> though.

If you say "they're just notmuch tag commands applied at new time" you
expect to have the same search behaviour as Xapian...

Maybe you could do that by temporarily inserting the mail into an
"inmemory" Xapian database, since you're only checking to see if that
particular one matches.

On the other hand, maybe having it be a separate syntax would be good
-- then you could justify using information notmuch doesn't usually
have -- like file/path names, Received or Delivered-To headers, and so
on.

On the gripping hand, maybe "notmuch tag" should simply be fast enough
that running a bunch of them after "notmuch new" isn't an issue.

Cheers,
aj

-- 
Anthony Towns <aj@erisian.com.au>

Re: [PATCH] notmuch new --new-tags=tags...

[Jameson Rollins]

[-- Attachment #1: Type: text/plain, Size: 494 bytes --]

On Tue, 13 Apr 2010 03:15:28 +1000, Anthony Towns <aj@erisian.com.au> wrote:
> If you say "they're just notmuch tag commands applied at new time" you
> expect to have the same search behaviour as Xapian...

True, but that's not exactly what I was saying.  I was just saying to
use the same syntax.  But I expect that things might have to work
differently under the hood.  But it might be too much work to come up
with something to parse the search syntax and apply the tag accordingly.

jamie.

[-- Attachment #2: Type: application/pgp-signature, Size: 835 bytes --]

Re: [PATCH] notmuch new --new-tags=tags...

[Carl Worth]

[-- Attachment #1: Type: text/plain, Size: 2052 bytes --]

On Mon, 12 Apr 2010 07:59:14 -0400, Jameson Rollins <jrollins@finestructure.net> wrote:
> On Mon, 12 Apr 2010 10:00:37 +0200, "Sebastian Spaeth" <Sebastian@SSpaeth.de> wrote:
> > On 2010-04-10, Anthony Towns wrote:
> > > The attached patch makes "notmuch new --new-tags=unread,new" set the
> > > "unread" and "new" tags on any new mail it finds rather than "unread"
> > > and "inbox". Or whatever other tags you happen to specify.

Thanks very much for the patch, AJ. It's definitely a useful feature.

> > Thanks for the patch. I can't comment on the code quality, but rather
> > than having to specify the set of new tags on the command line every
> > time, I think it would make more sense to put them in the notmuch config
> > file as this patch does:
> > id:1268432006-24333-2-git-send-email-bgamari.foss@gmail.com

I ended up taking that version instead, (using a configuration setting
for this instead of a command-line option to "notmuch new".

One could imagine accepting the command-line option as well, but I don't
think that would actually be useful. Let me know if you disagree.

> I was thinking about this, and it seems to me that we really need is a
> way to just specify which tags should be applied to new messages based
> on search terms.

Yes, I agree.

Many of us are (or will soon be) achieving the feature based on the
following recipe:

  * Configuring new.tags to add a "new" tag.

  * Writing a notmuch-poll script that does the following:

      * Performs "notmuch tag" searches to add tags to "new" messages.

      * Clears the "new" tag from all messages.

  * Configuring notmuch-poll-script inside emacs to invoke our
    notmuch-poll script.

If we added support for configuring search-based tagging directly into
notmuch, then the above recipe could be reduced to a single block of
configuration settings, which sounds much nicer.

So I'll look forward to any implementation to provide that. In the
meantime, we can muddle along with the functionality we want, (but a
little more manual effort to achieve it).

-Carl

[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]