[PATCH v2 00/11] Add filesize index, search, sort & emacs UI

[PATCH v2 00/11] Add filesize index, search, sort & emacs UI

[Ioan-Adrian Ratiu]

I'd like to add a feature to quickly work with mail file sizes
because using custom scripts / external programs which parse
maildir contents is slow, and non-intuitive, especially since
notmuch does incremental parsing and has such a nice emacs UI.


Ioan-Adrian Ratiu (11):
  lib: message: index message file sizes
  lib: database: store message filesize & add range processor
  notmuch-search: add filesize based sort order
  emacs: make notmuch-search-oldest-first generic
  emacs: notmuch-search: add filesize sorting
  sprinter: add unsigned_long printer function
  lib: thread: add thread total size function
  notmuch-search: output total_filesize thread result
  notmuch-show: export message filesize
  emacs: notmuch-search: add display thread sizes capability
  emacs: notmuch-show: add filesize to headerline

 devel/schemata         |  1 +
 doc/notmuch-emacs.rst  |  4 ++--
 emacs/notmuch-hello.el | 24 +++++++++++++-----------
 emacs/notmuch-jump.el  | 11 +++++------
 emacs/notmuch-lib.el   |  9 ++++++---
 emacs/notmuch-show.el  |  5 ++++-
 emacs/notmuch-tree.el  |  2 +-
 emacs/notmuch.el       | 48 +++++++++++++++++++++++++++++++++---------------
 lib/database-private.h |  1 +
 lib/database.cc        |  6 ++++++
 lib/index.cc           | 10 ++++++++++
 lib/message-file.c     | 18 +++++++++++++++++-
 lib/message.cc         | 29 +++++++++++++++++++++++++++++
 lib/notmuch-private.h  | 16 ++++++++++++++++
 lib/notmuch.h          | 21 +++++++++++++++++++++
 lib/query.cc           |  6 ++++++
 lib/thread.cc          | 12 ++++++++++++
 notmuch-search.c       |  8 +++++++-
 notmuch-show.c         |  5 +++++
 sprinter-json.c        |  9 +++++++++
 sprinter-sexp.c        |  9 +++++++++
 sprinter-text.c        |  9 +++++++++
 sprinter.h             |  1 +
 23 files changed, 223 insertions(+), 41 deletions(-)

-- 
2.13.0

[PATCH v2 01/11] lib: message: index message file sizes

[Ioan-Adrian Ratiu]

Parse & store the file sizes inside notmuch_message_t objects
while indexing. This is a useful foundation to build upon to
provide per message and per thread size statistics, sorting
and filtering mesages based on their sizes, etc.

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 lib/index.cc          | 10 ++++++++++
 lib/message-file.c    | 18 +++++++++++++++++-
 lib/message.cc        | 29 +++++++++++++++++++++++++++++
 lib/notmuch-private.h | 16 ++++++++++++++++
 lib/notmuch.h         |  6 ++++++
 5 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/lib/index.cc b/lib/index.cc
index 8c145540..e8655bc1 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -441,6 +441,8 @@ _notmuch_message_index_file (notmuch_message_t *message,
     InternetAddressList *addresses;
     const char *from, *subject;
     notmuch_status_t status;
+    unsigned long filesize;
+    char *filesize_str;
 
     status = _notmuch_message_file_get_mime_message (message_file,
 						     &mime_message);
@@ -464,6 +466,14 @@ _notmuch_message_index_file (notmuch_message_t *message,
     subject = g_mime_message_get_subject (mime_message);
     _notmuch_message_gen_terms (message, "subject", subject);
 
+    filesize = _notmuch_message_file_get_size (message_file);
+    filesize_str = talloc_asprintf(NULL, "%lu", filesize);
+    if (! filesize_str)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    _notmuch_message_add_term (message, "filesize", filesize_str);
+    talloc_free (filesize_str);
+
     _index_mime_part (message, g_mime_message_get_mime_part (mime_message));
 
     return NOTMUCH_STATUS_SUCCESS;
diff --git a/lib/message-file.c b/lib/message-file.c
index db18b163..f75593e3 100644
--- a/lib/message-file.c
+++ b/lib/message-file.c
@@ -26,10 +26,13 @@
 
 #include <glib.h> /* GHashTable */
 
+#include <glib/gstdio.h>
+
 struct _notmuch_message_file {
     /* File object */
     FILE *file;
     char *filename;
+    unsigned long filesize; /* in bytes */
 
     /* Cache for decoded headers */
     GHashTable *headers;
@@ -64,7 +67,7 @@ _notmuch_message_file_open_ctx (notmuch_database_t *notmuch,
     if (unlikely (message == NULL))
 	return NULL;
 
-    /* Only needed for error messages during parsing. */
+    /* Only needed during parsing */
     message->filename = talloc_strdup (message, filename);
     if (message->filename == NULL)
 	goto FAIL;
@@ -98,6 +101,12 @@ _notmuch_message_file_close (notmuch_message_file_t *message)
     talloc_free (message);
 }
 
+unsigned long
+_notmuch_message_file_get_size (notmuch_message_file_t *message)
+{
+    return message->filesize;
+}
+
 static notmuch_bool_t
 _is_mbox (FILE *file)
 {
@@ -122,6 +131,8 @@ _notmuch_message_file_parse (notmuch_message_file_t *message)
     notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
     static int initialized = 0;
     notmuch_bool_t is_mbox;
+    GStatBuf statResult;
+    int ret;
 
     if (message->message)
 	return NOTMUCH_STATUS_SUCCESS;
@@ -133,6 +144,11 @@ _notmuch_message_file_parse (notmuch_message_file_t *message)
 	initialized = 1;
     }
 
+    /* filesize defaults to zero which is ignored */
+    ret = g_stat(message->filename, &statResult);
+    if (! ret)
+	message->filesize = statResult.st_size;
+
     message->headers = g_hash_table_new_full (strcase_hash, strcase_equal,
 					      free, g_free);
     if (! message->headers)
diff --git a/lib/message.cc b/lib/message.cc
index b330dcce..c6b6e507 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -988,6 +988,26 @@ notmuch_message_get_date (notmuch_message_t *message)
     return Xapian::sortable_unserialise (value);
 }
 
+unsigned long
+notmuch_message_get_filesize (notmuch_message_t *message)
+{
+    std::string value;
+
+    try {
+	value = message->doc.get_value (NOTMUCH_VALUE_FILESIZE);
+    } catch (Xapian::Error &error) {
+	_notmuch_database_log(_notmuch_message_database (message), "A Xapian exception occurred when reading filesize: %s\n",
+		 error.get_msg().c_str());
+	message->notmuch->exception_reported = TRUE;
+	return 0;
+    }
+
+    if (value.empty ())
+	/* sortable_unserialise is undefined on empty string */
+	return 0;
+    return Xapian::sortable_unserialise (value);
+}
+
 notmuch_tags_t *
 notmuch_message_get_tags (notmuch_message_t *message)
 {
@@ -1208,6 +1228,15 @@ _notmuch_message_close (notmuch_message_t *message)
     }
 }
 
+void
+_notmuch_message_add_filesize (notmuch_message_t *message,
+			       notmuch_message_file_t *message_file)
+{
+    unsigned long filesize = _notmuch_message_file_get_size(message_file);
+    message->doc.add_value (NOTMUCH_VALUE_FILESIZE,
+			    Xapian::sortable_serialise (filesize));
+}
+
 /* Add a name:value term to 'message', (the actual term will be
  * encoded by prefixing the value with a short prefix). See
  * NORMAL_PREFIX and BOOLEAN_PREFIX arrays for the mapping of term
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index ac315e4c..d3428181 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -102,6 +102,7 @@ typedef enum {
     NOTMUCH_VALUE_FROM,
     NOTMUCH_VALUE_SUBJECT,
     NOTMUCH_VALUE_LAST_MOD,
+    NOTMUCH_VALUE_FILESIZE,
 } notmuch_value_t;
 
 /* Xapian (with flint backend) complains if we provide a term longer
@@ -392,6 +393,21 @@ _notmuch_message_file_close (notmuch_message_file_t *message);
 notmuch_status_t
 _notmuch_message_file_parse (notmuch_message_file_t *message);
 
+/*
+ * Get the filesize of a message file
+ *
+ * This filesize member is read during file parsing.
+ */
+unsigned long
+_notmuch_message_file_get_size (notmuch_message_file_t *message);
+
+/*
+ * Set the message filesize to the size of the message_file
+ */
+void
+_notmuch_message_add_filesize (notmuch_message_t *message,
+			       notmuch_message_file_t *message_file);
+
 /* Get the gmime message of a message file.
  *
  * The message file is parsed as necessary.
diff --git a/lib/notmuch.h b/lib/notmuch.h
index e1745444..f90458ce 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1413,6 +1413,12 @@ time_t
 notmuch_message_get_date  (notmuch_message_t *message);
 
 /**
+ * Get the filesize in bytes of 'message'.
+ */
+unsigned long
+notmuch_message_get_filesize  (notmuch_message_t *message);
+
+/**
  * Get the value of the specified header from 'message' as a UTF-8 string.
  *
  * Common headers are stored in the database when the message is
-- 
2.13.0

Re: [PATCH v2 01/11] lib: message: index message file sizes

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

>      notmuch_status_t status;
> +    unsigned long filesize;
> +    char *filesize_str;
>  
>      status = _notmuch_message_file_get_mime_message (message_file,
>  						     &mime_message);
> @@ -464,6 +466,14 @@ _notmuch_message_index_file (notmuch_message_t *message,
>      subject = g_mime_message_get_subject (mime_message);
>      _notmuch_message_gen_terms (message, "subject", subject);
>  
> +    filesize = _notmuch_message_file_get_size (message_file);
> +    filesize_str = talloc_asprintf(NULL, "%lu", filesize);
> +    if (! filesize_str)
> +	return NOTMUCH_STATUS_OUT_OF_MEMORY;
> +
> +    _notmuch_message_add_term (message, "filesize", filesize_str);
> +    talloc_free (filesize_str);

I wanted to see what the performance impact of this commit was, but
unfortunately this breaks "notmuch new" (and most other things) as the
prefix "filesize" is not defined yet.

A handy way to see that this doesn't happen is to run

  % git rebase -i --exec "make clean test" origin/master

before posting a series.

Another preliminary question is why are you adding both a prefixed term
and a value slot for file size? You can use the value slot to query.

d

Re: [PATCH v2 01/11] lib: message: index message file sizes

[David Bremner]

As a preliminary note, I think this series will most likely need to
adapt to the reindexing series
id:20170604123235.24466-2-david@tethera.net as I think they are touching
the same parts of the code.  You might want to wait for that to go in
(or for it to be cancelled) before reworking your series.

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> Parse & store the file sizes inside notmuch_message_t objects
> while indexing.

That seems not actually to be true, since there is no member of
notmuch_message_t which stores the filesize. It's also a bit confusing,
since indexing is about updating the database, not the in-memory data
structures.

> +    filesize = _notmuch_message_file_get_size (message_file);
> +    filesize_str = talloc_asprintf(NULL, "%lu", filesize);
> +    if (! filesize_str)
> +	return NOTMUCH_STATUS_OUT_OF_MEMORY;
> +
> +    _notmuch_message_add_term (message, "filesize", filesize_str);
> +    talloc_free (filesize_str);
> +

As I mentioned in a previous message,
   1) this crashes, because you have no prefix for filesize yet.
   2) there seems to be no point in adding this term, since you search
   on the value slot anyway.
   Presumably you want to replace it with a call to _notmuch_message_add_filesize.

I did manage to do a little benchmarking after applying the next patch,
and database size and initial indexing time both increase by about
0.5% with the notmuch performance test suite (large version). This seems
acceptable to me, and I would hope it only improves (or at least doesn't
get worse) when the redundant terms are dropped.

> +    /* filesize defaults to zero which is ignored */

Which filesize do you refer to here? I'm a bit on the fence about
pervasively assuming a zero filesize is an error.

> +    ret = g_stat(message->filename, &statResult);
> +    if (! ret)
> +	message->filesize = statResult.st_size;
> +

Why are you using g_stat instead of plain stat? g_stat seems to mainly
add windows compatibility (and confusion, since it's less familiar).

> +unsigned long
> +notmuch_message_get_filesize (notmuch_message_t *message)
> +{
> +    std::string value;
> +
> +    try {
> +	value = message->doc.get_value (NOTMUCH_VALUE_FILESIZE);

I wondered if this was wasteful going straight to the database without
caching, but apparently we do it already for from, subject, and
message-id.

> +    } catch (Xapian::Error &error) {
> +	_notmuch_database_log(_notmuch_message_database (message), "A Xapian exception occurred when reading filesize: %s\n",
> +		 error.get_msg().c_str());
> +	message->notmuch->exception_reported = TRUE;
> +	return 0;
> +    }
> +    if (value.empty ())
> +	/* sortable_unserialise is undefined on empty string */
> +	return 0;
> +    return Xapian::sortable_unserialise (value);
> +}

I'm not sure about this error handling. Do we want an API where we can't
tell the difference between a missing value, an empty file, and a
transient Xapian exception? OTOH, I do see that it's a bit clunky to use
a status return and output pointer here.
>  
> +void
> +_notmuch_message_add_filesize (notmuch_message_t *message,
> +			       notmuch_message_file_t *message_file)
> +{
> +    unsigned long filesize = _notmuch_message_file_get_size(message_file);
> +    message->doc.add_value (NOTMUCH_VALUE_FILESIZE,
> +			    Xapian::sortable_serialise (filesize));
> +}

Shouldn't this have some exception handling (and probably an error
return)? basically any xapian operation can throw an exception.

>  /**
> + * Get the filesize in bytes of 'message'.
> + */
> +unsigned long
> +notmuch_message_get_filesize  (notmuch_message_t *message);
> +
> +/**

Please document the error conditions and returns of any public API call added.

[PATCH v2 02/11] lib: database: store message filesize & add range processor

[Ioan-Adrian Ratiu]

The filesize and range processor are very useful to do search queries
like "filesize:1000..10000". All sizes are in bytes for now because
that's what I'm interested in.

I think the database needs to be re-created for this to work so that
all newly indexed messages have the new xapian filesize value, I did
not have the time to test a "hybrid" database where just some of the
messages have the value.

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 lib/database-private.h | 1 +
 lib/database.cc        | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/lib/database-private.h b/lib/database-private.h
index 727b1d61..d4d5ab6b 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -215,6 +215,7 @@ struct _notmuch_database {
     Xapian::ValueRangeProcessor *value_range_processor;
     Xapian::ValueRangeProcessor *date_range_processor;
     Xapian::ValueRangeProcessor *last_mod_range_processor;
+    Xapian::ValueRangeProcessor *filesize_range_processor;
 };
 
 /* Prior to database version 3, features were implied by the database
diff --git a/lib/database.cc b/lib/database.cc
index 5b13f541..84e616cf 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -295,6 +295,7 @@ prefix_t prefix_table[] = {
     { "subject",		"XSUBJECT",	NOTMUCH_FIELD_EXTERNAL |
 						NOTMUCH_FIELD_PROBABILISTIC |
 						NOTMUCH_FIELD_PROCESSOR},
+    { "filesize",		"XFILESIZE",	NOTMUCH_FIELD_EXTERNAL },
 };
 
 static void
@@ -1082,6 +1083,7 @@ notmuch_database_open_verbose (const char *path,
 	notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
 	notmuch->date_range_processor = new ParseTimeValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
 	notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
+	notmuch->filesize_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_FILESIZE, "filesize:");
 
 	notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
 	notmuch->query_parser->set_database (*notmuch->xapian_db);
@@ -1090,6 +1092,7 @@ notmuch_database_open_verbose (const char *path,
 	notmuch->query_parser->add_valuerangeprocessor (notmuch->value_range_processor);
 	notmuch->query_parser->add_valuerangeprocessor (notmuch->date_range_processor);
 	notmuch->query_parser->add_valuerangeprocessor (notmuch->last_mod_range_processor);
+	notmuch->query_parser->add_valuerangeprocessor (notmuch->filesize_range_processor);
 
 	for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
 	    const prefix_t *prefix = &prefix_table[i];
@@ -1166,6 +1169,8 @@ notmuch_database_close (notmuch_database_t *notmuch)
     notmuch->date_range_processor = NULL;
     delete notmuch->last_mod_range_processor;
     notmuch->last_mod_range_processor = NULL;
+    delete notmuch->filesize_range_processor;
+    notmuch->filesize_range_processor = NULL;
 
     return status;
 }
@@ -2563,6 +2568,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 	}
 
 	_notmuch_message_add_filename (message, filename);
+	_notmuch_message_add_filesize (message, message_file);
 
 	/* Is this a newly created message object or a ghost
 	 * message?  We have to be slightly careful: if this is a
-- 
2.13.0

Re: [PATCH v2 02/11] lib: database: store message filesize & add range processor

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> The filesize and range processor are very useful to do search queries
> like "filesize:1000..10000". All sizes are in bytes for now because
> that's what I'm interested in.

Some points to mention in the commit message

- you're adding a term prefix (although as discussed in the previous
  message, and below, I don't think you want to be)

- you're extending the query language.

- it's fine that all sizes are in bytes, but the main point is not your
  motivation but that it can be extended to other units in the future by
  preprocessing the query.
  
>
> I think the database needs to be re-created for this to work so that
> all newly indexed messages have the new xapian filesize value, I did
> not have the time to test a "hybrid" database where just some of the
> messages have the value.

see commit 0de999aab5bd4cd44bc4ea76fd1d25172bd839ae for how to handle
this.

After this commit, you have enough new features to test indexing and
searching by file size, so please do, in particular testing the case you
mention above. The new syntax should work with e.g. "notmuch search" and
"notmuch show".

>
> +    { "filesize",		"XFILESIZE",	NOTMUCH_FIELD_EXTERNAL },
>  };

This is a bit subtle, but I don't think you should add filesize to this
table (compare with "lastmod").

[PATCH v2 03/11] notmuch-search: add filesize based sort order

[Ioan-Adrian Ratiu]

With this it now becomes possible to order the search results by
filesize using the --sort=biggest-first/smallest-first args.

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 lib/notmuch.h    | 8 ++++++++
 lib/query.cc     | 6 ++++++
 notmuch-search.c | 2 ++
 3 files changed, 16 insertions(+)

diff --git a/lib/notmuch.h b/lib/notmuch.h
index f90458ce..b7bf3526 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -743,6 +743,14 @@ typedef enum {
      */
     NOTMUCH_SORT_MESSAGE_ID,
     /**
+     * Smallest first.
+     */
+    NOTMUCH_SORT_SMALLEST_FIRST,
+    /**
+     * Biggest first
+     */
+    NOTMUCH_SORT_BIGGEST_FIRST,
+    /**
      * Do not sort.
      */
     NOTMUCH_SORT_UNSORTED
diff --git a/lib/query.cc b/lib/query.cc
index 9c6ecc8d..72b725e3 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -330,6 +330,12 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 	case NOTMUCH_SORT_MESSAGE_ID:
 	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE);
 	    break;
+	case NOTMUCH_SORT_SMALLEST_FIRST:
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_FILESIZE, FALSE);
+	    break;
+	case NOTMUCH_SORT_BIGGEST_FIRST:
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_FILESIZE, TRUE);
+	    break;
 	case NOTMUCH_SORT_UNSORTED:
 	    break;
 	}
diff --git a/notmuch-search.c b/notmuch-search.c
index 019e14ee..65ecfaab 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -778,6 +778,8 @@ static const notmuch_opt_desc_t common_options[] = {
     { NOTMUCH_OPT_KEYWORD, &search_context.sort, "sort", 's',
       (notmuch_keyword_t []){ { "oldest-first", NOTMUCH_SORT_OLDEST_FIRST },
 			      { "newest-first", NOTMUCH_SORT_NEWEST_FIRST },
+			      { "smallest-first", NOTMUCH_SORT_SMALLEST_FIRST },
+			      { "biggest-first", NOTMUCH_SORT_BIGGEST_FIRST },
 			      { 0, 0 } } },
     { NOTMUCH_OPT_KEYWORD, &search_context.format_sel, "format", 'f',
       (notmuch_keyword_t []){ { "json", NOTMUCH_FORMAT_JSON },
-- 
2.13.0

Re: [PATCH v2 03/11] notmuch-search: add filesize based sort order

[Tomi Ollila]

On Fri, May 19 2017, Ioan-Adrian Ratiu wrote:

> With this it now becomes possible to order the search results by
> filesize using the --sort=biggest-first/smallest-first args.

Quick drive-by comment (for now): In many places elsewhere we're talking
about 'large' files (e.g. LARGEFILE_SOURCE...) so should we use that
terminology here too (?) instead of 'big*'

Tomi


>
> Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
> ---
>  lib/notmuch.h    | 8 ++++++++
>  lib/query.cc     | 6 ++++++
>  notmuch-search.c | 2 ++
>  3 files changed, 16 insertions(+)
>
> diff --git a/lib/notmuch.h b/lib/notmuch.h
> index f90458ce..b7bf3526 100644
> --- a/lib/notmuch.h
> +++ b/lib/notmuch.h
> @@ -743,6 +743,14 @@ typedef enum {
>       */
>      NOTMUCH_SORT_MESSAGE_ID,
>      /**
> +     * Smallest first.
> +     */
> +    NOTMUCH_SORT_SMALLEST_FIRST,
> +    /**
> +     * Biggest first
> +     */
> +    NOTMUCH_SORT_BIGGEST_FIRST,
> +    /**
>       * Do not sort.
>       */
>      NOTMUCH_SORT_UNSORTED
> diff --git a/lib/query.cc b/lib/query.cc
> index 9c6ecc8d..72b725e3 100644
> --- a/lib/query.cc
> +++ b/lib/query.cc
> @@ -330,6 +330,12 @@ _notmuch_query_search_documents (notmuch_query_t *query,
>  	case NOTMUCH_SORT_MESSAGE_ID:
>  	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE);
>  	    break;
> +	case NOTMUCH_SORT_SMALLEST_FIRST:
> +	    enquire.set_sort_by_value (NOTMUCH_VALUE_FILESIZE, FALSE);
> +	    break;
> +	case NOTMUCH_SORT_BIGGEST_FIRST:
> +	    enquire.set_sort_by_value (NOTMUCH_VALUE_FILESIZE, TRUE);
> +	    break;
>  	case NOTMUCH_SORT_UNSORTED:
>  	    break;
>  	}
> diff --git a/notmuch-search.c b/notmuch-search.c
> index 019e14ee..65ecfaab 100644
> --- a/notmuch-search.c
> +++ b/notmuch-search.c
> @@ -778,6 +778,8 @@ static const notmuch_opt_desc_t common_options[] = {
>      { NOTMUCH_OPT_KEYWORD, &search_context.sort, "sort", 's',
>        (notmuch_keyword_t []){ { "oldest-first", NOTMUCH_SORT_OLDEST_FIRST },
>  			      { "newest-first", NOTMUCH_SORT_NEWEST_FIRST },
> +			      { "smallest-first", NOTMUCH_SORT_SMALLEST_FIRST },
> +			      { "biggest-first", NOTMUCH_SORT_BIGGEST_FIRST },
>  			      { 0, 0 } } },
>      { NOTMUCH_OPT_KEYWORD, &search_context.format_sel, "format", 'f',
>        (notmuch_keyword_t []){ { "json", NOTMUCH_FORMAT_JSON },
> -- 
> 2.13.0
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH v2 03/11] notmuch-search: add filesize based sort order

[Ioan-Adrian Ratiu]

On Fri, 19 May 2017, Tomi Ollila <tomi.ollila@iki.fi> wrote:
> On Fri, May 19 2017, Ioan-Adrian Ratiu wrote:
>
>> With this it now becomes possible to order the search results by
>> filesize using the --sort=biggest-first/smallest-first args.
>
> Quick drive-by comment (for now): In many places elsewhere we're talking
> about 'large' files (e.g. LARGEFILE_SOURCE...) so should we use that
> terminology here too (?) instead of 'big*'

Sure. I'll update to use 'large' in v3, but I'll wait a little longer to
get more feedback on v2 before sending it because it's a small change.

Ionel

>
> Tomi
>
>
>>
>> Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
>> ---
>>  lib/notmuch.h    | 8 ++++++++
>>  lib/query.cc     | 6 ++++++
>>  notmuch-search.c | 2 ++
>>  3 files changed, 16 insertions(+)
>>
>> diff --git a/lib/notmuch.h b/lib/notmuch.h
>> index f90458ce..b7bf3526 100644
>> --- a/lib/notmuch.h
>> +++ b/lib/notmuch.h
>> @@ -743,6 +743,14 @@ typedef enum {
>>       */
>>      NOTMUCH_SORT_MESSAGE_ID,
>>      /**
>> +     * Smallest first.
>> +     */
>> +    NOTMUCH_SORT_SMALLEST_FIRST,
>> +    /**
>> +     * Biggest first
>> +     */
>> +    NOTMUCH_SORT_BIGGEST_FIRST,
>> +    /**
>>       * Do not sort.
>>       */
>>      NOTMUCH_SORT_UNSORTED
>> diff --git a/lib/query.cc b/lib/query.cc
>> index 9c6ecc8d..72b725e3 100644
>> --- a/lib/query.cc
>> +++ b/lib/query.cc
>> @@ -330,6 +330,12 @@ _notmuch_query_search_documents (notmuch_query_t *query,
>>  	case NOTMUCH_SORT_MESSAGE_ID:
>>  	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE);
>>  	    break;
>> +	case NOTMUCH_SORT_SMALLEST_FIRST:
>> +	    enquire.set_sort_by_value (NOTMUCH_VALUE_FILESIZE, FALSE);
>> +	    break;
>> +	case NOTMUCH_SORT_BIGGEST_FIRST:
>> +	    enquire.set_sort_by_value (NOTMUCH_VALUE_FILESIZE, TRUE);
>> +	    break;
>>  	case NOTMUCH_SORT_UNSORTED:
>>  	    break;
>>  	}
>> diff --git a/notmuch-search.c b/notmuch-search.c
>> index 019e14ee..65ecfaab 100644
>> --- a/notmuch-search.c
>> +++ b/notmuch-search.c
>> @@ -778,6 +778,8 @@ static const notmuch_opt_desc_t common_options[] = {
>>      { NOTMUCH_OPT_KEYWORD, &search_context.sort, "sort", 's',
>>        (notmuch_keyword_t []){ { "oldest-first", NOTMUCH_SORT_OLDEST_FIRST },
>>  			      { "newest-first", NOTMUCH_SORT_NEWEST_FIRST },
>> +			      { "smallest-first", NOTMUCH_SORT_SMALLEST_FIRST },
>> +			      { "biggest-first", NOTMUCH_SORT_BIGGEST_FIRST },
>>  			      { 0, 0 } } },
>>      { NOTMUCH_OPT_KEYWORD, &search_context.format_sel, "format", 'f',
>>        (notmuch_keyword_t []){ { "json", NOTMUCH_FORMAT_JSON },
>> -- 
>> 2.13.0
>>
>> _______________________________________________
>> notmuch mailing list
>> notmuch@notmuchmail.org
>> https://notmuchmail.org/mailman/listinfo/notmuch
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH v2 03/11] notmuch-search: add filesize based sort order

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> With this it now becomes possible to order the search results by
> filesize using the --sort=biggest-first/smallest-first args.

You should mention that you are also changing the library API, as well
as adding new search options to notmuch-show  and notmuch-search.
>
> diff --git a/lib/notmuch.h b/lib/notmuch.h
> index f90458ce..b7bf3526 100644
> --- a/lib/notmuch.h
> +++ b/lib/notmuch.h
> @@ -743,6 +743,14 @@ typedef enum {
>       */
>      NOTMUCH_SORT_MESSAGE_ID,
>      /**
> +     * Smallest first.
> +     */
> +    NOTMUCH_SORT_SMALLEST_FIRST,
> +    /**
> +     * Biggest first
> +     */
> +    NOTMUCH_SORT_BIGGEST_FIRST,
> +    /**
>       * Do not sort.
>       */

At the moment the ABI is unstable on master, but that will change with
the next release.  Depending on when this is merged, it may need to bump
the library version. I think just bumping LIBNOTMUCH_MINOR should be ok.


> +++ b/lib/query.cc
> @@ -330,6 +330,12 @@ _notmuch_query_search_documents (notmuch_query_t *query,
>  	case NOTMUCH_SORT_MESSAGE_ID:
>  	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE);
>  	    break;
> +	case NOTMUCH_SORT_SMALLEST_FIRST:
> +	    enquire.set_sort_by_value (NOTMUCH_VALUE_FILESIZE, FALSE);
> +	    break;
> +	case NOTMUCH_SORT_BIGGEST_FIRST:
> +	    enquire.set_sort_by_value (NOTMUCH_VALUE_FILESIZE, TRUE);
> +	    break;
>  	case NOTMUCH_SORT_UNSORTED:
>  	    break;
>  	}
> diff --git a/notmuch-search.c b/notmuch-search.c
> index 019e14ee..65ecfaab 100644
> --- a/notmuch-search.c
> +++ b/notmuch-search.c
> @@ -778,6 +778,8 @@ static const notmuch_opt_desc_t common_options[] = {
>      { NOTMUCH_OPT_KEYWORD, &search_context.sort, "sort", 's',
>        (notmuch_keyword_t []){ { "oldest-first", NOTMUCH_SORT_OLDEST_FIRST },
>  			      { "newest-first", NOTMUCH_SORT_NEWEST_FIRST },
> +			      { "smallest-first", NOTMUCH_SORT_SMALLEST_FIRST },
> +			      { "biggest-first", NOTMUCH_SORT_BIGGEST_FIRST },
>  			      { 0, 0 } } },
>      { NOTMUCH_OPT_KEYWORD, &search_context.format_sel, "format", 'f',
>        (notmuch_keyword_t []){ { "json", NOTMUCH_FORMAT_JSON },

These new options should be documented, and tested.

[PATCH v2 04/11] emacs: make notmuch-search-oldest-first generic

[Ioan-Adrian Ratiu]

The current search result order logic assumes results are always
sorted by date and thus uses a boolean switch for oldest/newest
ordering specifications.

This is problematic if I want to introduce other result orderings,
like for example based on the mail-file size with smallest/biggest
ordering specifications.

In the interest of keeping all current logic intact and reusable,
while at the same time supporting multiple search result orderings,
change the defcustom configuration notmuch-search-oldest-first to
notmuch-search-default-sort-order which takes values 'oldest-first
and 'newest-first (for now).

Implementing new result orderings thus becomes a simple matter of
adding more possible entries for notmuch-search-default-sort-order.

Aside from the UI variable rename change, this commit should be
totally transparent for the user, it does not modify or add any new
result sorting logic.

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 doc/notmuch-emacs.rst  |  4 ++--
 emacs/notmuch-hello.el | 15 +++++++--------
 emacs/notmuch-jump.el  | 11 +++++------
 emacs/notmuch-lib.el   |  7 ++++---
 emacs/notmuch-tree.el  |  2 +-
 emacs/notmuch.el       | 29 +++++++++++++++--------------
 6 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst
index 5e25996f..66a69bb8 100644
--- a/doc/notmuch-emacs.rst
+++ b/doc/notmuch-emacs.rst
@@ -169,8 +169,8 @@ variables.
     Control how each thread of messages is presented in the
     ``notmuch-show-mode`` buffer
 
-:index:`notmuch-search-oldest-first`
-    Display the oldest threads at the top of the buffer
+:index:`notmuch-search-default-sort-order`
+    Control the default search result method
 
 .. _notmuch-show:
 
diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
index c858a20b..3ba2a16b 100644
--- a/emacs/notmuch-hello.el
+++ b/emacs/notmuch-hello.el
@@ -28,7 +28,7 @@
 (require 'notmuch-lib)
 (require 'notmuch-mua)
 
-(declare-function notmuch-search "notmuch" (&optional query oldest-first target-thread target-line continuation))
+(declare-function notmuch-search "notmuch" (&optional query sort-order target-thread target-line continuation))
 (declare-function notmuch-poll "notmuch" ())
 (declare-function notmuch-tree "notmuch-tree"
                   (&optional query query-context target buffer-name open-target))
@@ -381,7 +381,7 @@ afterwards.")
     (setq search (notmuch-hello-trim search))
     (let ((history-delete-duplicates t))
       (add-to-history 'notmuch-search-history search)))
-  (notmuch-search search notmuch-search-oldest-first))
+  (notmuch-search search notmuch-search-default-sort-order))
 
 (defun notmuch-hello-add-saved-search (widget)
   (interactive)
@@ -443,7 +443,7 @@ diagonal."
     (notmuch-search (widget-get widget
 				:notmuch-search-terms)
 		    (widget-get widget
-				:notmuch-search-oldest-first))))
+				:notmuch-search-sort-order))))
 
 (defun notmuch-saved-search-count (search)
   (car (process-lines notmuch-command "count" search)))
@@ -575,10 +575,9 @@ with `notmuch-hello-query-counts'."
 		  (widget-insert (make-string column-indent ? )))
 	      (let* ((name (plist-get elem :name))
 		     (query (plist-get elem :query))
-		     (oldest-first (case (plist-get elem :sort-order)
-				     (newest-first nil)
-				     (oldest-first t)
-				     (otherwise notmuch-search-oldest-first)))
+		     (sort-order (if (plist-get elem :sort-order)
+				     (plist-get elem :sort-order)
+				   notmuch-search-default-sort-order))
 		     (search-type (eq (plist-get elem :search-type) 'tree))
 		     (msg-count (plist-get elem :count)))
 		(widget-insert (format "%8s "
@@ -586,7 +585,7 @@ with `notmuch-hello-query-counts'."
 		(widget-create 'push-button
 			       :notify #'notmuch-hello-widget-search
 			       :notmuch-search-terms query
-			       :notmuch-search-oldest-first oldest-first
+			       :notmuch-search-sort-order sort-order
 			       :notmuch-search-type search-type
 			       name)
 		(setq column-indent
diff --git a/emacs/notmuch-jump.el b/emacs/notmuch-jump.el
index 3e20b8c7..716d39b8 100644
--- a/emacs/notmuch-jump.el
+++ b/emacs/notmuch-jump.el
@@ -50,15 +50,14 @@ fast way to jump to a saved search from anywhere in Notmuch."
 	(when key
 	  (let ((name (plist-get saved-search :name))
 		(query (plist-get saved-search :query))
-		(oldest-first
-		 (case (plist-get saved-search :sort-order)
-		   (newest-first nil)
-		   (oldest-first t)
-		   (otherwise (default-value 'notmuch-search-oldest-first)))))
+		(sort-order
+		 (if (plist-get saved-search :sort-order)
+		     (plist-get saved-search :sort-order)
+		   (notmuch-search-default-sort-order))))
 	    (push (list key name
 			(if (eq (plist-get saved-search :search-type) 'tree)
 			    `(lambda () (notmuch-tree ',query))
-			  `(lambda () (notmuch-search ',query ',oldest-first))))
+			  `(lambda () (notmuch-search ',query ',sort-order))))
 		  action-map)))))
     (setq action-map (nreverse action-map))
 
diff --git a/emacs/notmuch-lib.el b/emacs/notmuch-lib.el
index 337b20ac..34ffa712 100644
--- a/emacs/notmuch-lib.el
+++ b/emacs/notmuch-lib.el
@@ -92,14 +92,15 @@ $PATH)."
   :type 'string
   :group 'notmuch-external)
 
-(defcustom notmuch-search-oldest-first t
-  "Show the oldest mail first when searching.
+(defcustom notmuch-search-default-sort-order 'oldest-first
+  "Default result sorting to use when searching.
 
 This variable defines the default sort order for displaying
 search results. Note that any filtered searches created by
 `notmuch-search-filter' retain the search order of the parent
 search."
-  :type 'boolean
+  :type '(choice (const :tag "oldest-first" oldest-first)
+		 (const :tag "newest-first" newest-first))
   :group 'notmuch-search)
 
 (defcustom notmuch-poll-script nil
diff --git a/emacs/notmuch-tree.el b/emacs/notmuch-tree.el
index d4d40761..99cb098c 100644
--- a/emacs/notmuch-tree.el
+++ b/emacs/notmuch-tree.el
@@ -33,7 +33,7 @@
 (require 'notmuch-parser)
 
 (eval-when-compile (require 'cl))
-(declare-function notmuch-search "notmuch" (&optional query oldest-first target-thread target-line))
+(declare-function notmuch-search "notmuch" (&optional query sort-order target-thread target-line))
 (declare-function notmuch-call-notmuch-process "notmuch" (&rest args))
 (declare-function notmuch-read-query "notmuch" (prompt))
 (declare-function notmuch-search-find-thread-id "notmuch" (&optional bare))
diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index 40b9fabd..248b97c7 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -367,7 +367,7 @@ Complete list of currently available key bindings:
 
 \\{notmuch-search-mode-map}"
   (make-local-variable 'notmuch-search-query-string)
-  (make-local-variable 'notmuch-search-oldest-first)
+  (make-local-variable 'notmuch-search-default-sort-order)
   (make-local-variable 'notmuch-search-target-thread)
   (make-local-variable 'notmuch-search-target-line)
   (setq notmuch-buffer-refresh-function #'notmuch-search-refresh-view)
@@ -942,13 +942,13 @@ PROMPT is the string to prompt with."
 
 (put 'notmuch-search 'notmuch-doc "Search for messages.")
 ;;;###autoload
-(defun notmuch-search (&optional query oldest-first target-thread target-line no-display)
+(defun notmuch-search (&optional query sort-order target-thread target-line no-display)
   "Display threads matching QUERY in a notmuch-search buffer.
 
 If QUERY is nil, it is read interactively from the minibuffer.
 Other optional parameters are used as follows:
 
-  OLDEST-FIRST: A Boolean controlling the sort order of returned threads
+  SORT-ORDER: Sort order of the returned threads
   TARGET-THREAD: A thread ID (without the thread: prefix) that will be made
                  current if it appears in the search results.
   TARGET-LINE: The line number to move to if the target thread does not
@@ -965,7 +965,7 @@ the configured default sort order."
     nil
     ;; Use the default search order (if we're doing a search from a
     ;; search buffer, ignore any buffer-local overrides)
-    (default-value 'notmuch-search-oldest-first)))
+    (default-value 'notmuch-search-default-sort-order)))
 
   (let* ((query (or query (notmuch-read-query "Notmuch search: ")))
 	 (buffer (get-buffer-create (notmuch-search-buffer-title query))))
@@ -976,7 +976,7 @@ the configured default sort order."
     ;; Don't track undo information for this buffer
     (set 'buffer-undo-list t)
     (set 'notmuch-search-query-string query)
-    (set 'notmuch-search-oldest-first oldest-first)
+    (set 'notmuch-search-default-sort-order sort-order)
     (set 'notmuch-search-target-thread target-thread)
     (set 'notmuch-search-target-line target-line)
     (notmuch-tag-clear-cache)
@@ -991,9 +991,7 @@ the configured default sort order."
 	(let ((proc (notmuch-start-notmuch
 		     "notmuch-search" buffer #'notmuch-search-process-sentinel
 		     "search" "--format=sexp" "--format-version=2"
-		     (if oldest-first
-			 "--sort=oldest-first"
-		       "--sort=newest-first")
+		     (concat "--sort=" (symbol-name sort-order))
 		     query))
 	      ;; Use a scratch buffer to accumulate partial output.
 	      ;; This buffer will be killed by the sentinel, which
@@ -1014,20 +1012,23 @@ thread. Otherwise, point will be moved to attempt to be in the
 same relative position within the new buffer."
   (interactive)
   (let ((target-line (line-number-at-pos))
-	(oldest-first notmuch-search-oldest-first)
+	(sort-order notmuch-search-default-sort-order)
 	(target-thread (notmuch-search-find-thread-id 'bare))
 	(query notmuch-search-query-string))
     ;; notmuch-search erases the current buffer.
-    (notmuch-search query oldest-first target-thread target-line t)
+    (notmuch-search query sort-order target-thread target-line t)
     (goto-char (point-min))))
 
 (defun notmuch-search-toggle-order ()
   "Toggle the current search order.
 
 This command toggles the sort order for the current search. The
-default sort order is defined by `notmuch-search-oldest-first'."
+default sort order is defined by `notmuch-search-default-sort-order'."
   (interactive)
-  (set 'notmuch-search-oldest-first (not notmuch-search-oldest-first))
+  (setq notmuch-search-default-sort-order
+	(case notmuch-search-default-sort-order
+	  ('oldest-first 'newest-first)
+	  (otherwise 'oldest-first)))
   (notmuch-search-refresh-view))
 
 (defun notmuch-group-disjunctive-query-string (query-string)
@@ -1051,7 +1052,7 @@ current search results AND the additional query string provided."
     (notmuch-search (if (string= grouped-original-query "*")
 			grouped-query
 		      (concat grouped-original-query " and " grouped-query))
-		    notmuch-search-oldest-first)))
+		    notmuch-search-default-sort-order)))
 
 (defun notmuch-search-filter-by-tag (tag)
   "Filter the current search results based on a single tag.
@@ -1060,7 +1061,7 @@ Runs a new search matching only messages that match both the
 current search results AND that are tagged with the given tag."
   (interactive
    (list (notmuch-select-tag-with-completion "Filter by tag: ")))
-  (notmuch-search (concat notmuch-search-query-string " and tag:" tag) notmuch-search-oldest-first))
+  (notmuch-search (concat notmuch-search-query-string " and tag:" tag) notmuch-search-default-sort-order))
 
 ;;;###autoload
 (defun notmuch ()
-- 
2.13.0

Re: [PATCH v2 04/11] emacs: make notmuch-search-oldest-first generic

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> The current search result order logic assumes results are always
> sorted by date and thus uses a boolean switch for oldest/newest
> ordering specifications.
>
> This is problematic if I want to introduce other result orderings,
> like for example based on the mail-file size with smallest/biggest
> ordering specifications.
It's a minor point, but I would write "if we want" here

[PATCH v2 05/11] emacs: notmuch-search: add filesize sorting

[Ioan-Adrian Ratiu]

Besides the previous date-based result orderings (oldest-first and
newest-first) add two more filesize-based orderings: biggest-first
smallest-first.

The orderings are interchangeable, you can specify any one as the
default via notmuch-search-default-sort-order or as the preffered
ordering for a saved search (via the :sort-order property).

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 emacs/notmuch-hello.el |  9 ++++++---
 emacs/notmuch-lib.el   |  4 +++-
 emacs/notmuch.el       | 12 ++++++++++--
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
index 3ba2a16b..51117577 100644
--- a/emacs/notmuch-hello.el
+++ b/emacs/notmuch-hello.el
@@ -95,7 +95,9 @@ searches so they still work in customize."
 			    (choice :tag " Sort Order"
 				    (const :tag "Default" nil)
 				    (const :tag "Oldest-first" oldest-first)
-				    (const :tag "Newest-first" newest-first)))
+				    (const :tag "Newest-first" newest-first)
+				    (const :tag "Biggest-first" biggest-first)
+				    (const :tag "Smallest-first" smallest-first)))
 		     (group :format "%v" :inline t (const :format "" :search-type)
 			    (choice :tag " Search Type"
 				    (const :tag "Search mode" nil)
@@ -120,8 +122,9 @@ a plist. Supported properties are
                    shown. If not present then the :query property
                    is used.
   :sort-order      Specify the sort order to be used for the search.
-                   Possible values are 'oldest-first 'newest-first or
-                   nil. Nil means use the default sort order.
+                   Possible values are 'oldest-first 'newest-first
+                   'biggest-first 'smallest-first or nil.
+                   Nil means use the default sort order.
   :search-type     Specify whether to run the search in search-mode
                    or tree mode. Set to 'tree to specify tree
                    mode, set to nil (or anything except tree) to
diff --git a/emacs/notmuch-lib.el b/emacs/notmuch-lib.el
index 34ffa712..ded75c2e 100644
--- a/emacs/notmuch-lib.el
+++ b/emacs/notmuch-lib.el
@@ -100,7 +100,9 @@ search results. Note that any filtered searches created by
 `notmuch-search-filter' retain the search order of the parent
 search."
   :type '(choice (const :tag "oldest-first" oldest-first)
-		 (const :tag "newest-first" newest-first))
+		 (const :tag "newest-first" newest-first)
+		 (const :tag "biggest-first" biggest-first)
+		 (const :tag "smallest-first" smallest-first))
   :group 'notmuch-search)
 
 (defcustom notmuch-poll-script nil
diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index 248b97c7..5b9c1d07 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -1023,12 +1023,20 @@ same relative position within the new buffer."
   "Toggle the current search order.
 
 This command toggles the sort order for the current search. The
-default sort order is defined by `notmuch-search-default-sort-order'."
+default sort order is defined by `notmuch-search-default-sort-order'.
+
+There are two types of orderings: by date and by filesize. Toggling is
+permitted only within a specific type (for ex. you can't toggle from
+oldest-first to smallest-first, only from oldest-first to newest-first)
+to avoid confusion."
   (interactive)
   (setq notmuch-search-default-sort-order
 	(case notmuch-search-default-sort-order
 	  ('oldest-first 'newest-first)
-	  (otherwise 'oldest-first)))
+	  ('newest-first 'biggest-first)
+	  ('biggest-first 'smallest-first)
+	  ('smallest-first 'oldest-first)
+	  (otherwise notmuch-search-default-sort-order)))
   (notmuch-search-refresh-view))
 
 (defun notmuch-group-disjunctive-query-string (query-string)
-- 
2.13.0

Re: [PATCH v2 05/11] emacs: notmuch-search: add filesize sorting

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> Besides the previous date-based result orderings (oldest-first and
> newest-first) add two more filesize-based orderings: biggest-first
> smallest-first.
>
> The orderings are interchangeable, you can specify any one as the
> default via notmuch-search-default-sort-order or as the preffered
                                                    spelling ^
> ordering for a saved search (via the :sort-order property).
>
> Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
> ---
>  emacs/notmuch-hello.el |  9 ++++++---
>  emacs/notmuch-lib.el   |  4 +++-
>  emacs/notmuch.el       | 12 ++++++++++--
>  3 files changed, 19 insertions(+), 6 deletions(-)
>
> diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
> index 3ba2a16b..51117577 100644
> --- a/emacs/notmuch-hello.el
> +++ b/emacs/notmuch-hello.el
> @@ -95,7 +95,9 @@ searches so they still work in customize."
>  			    (choice :tag " Sort Order"
>  				    (const :tag "Default" nil)
>  				    (const :tag "Oldest-first" oldest-first)
> -				    (const :tag "Newest-first" newest-first)))
> +				    (const :tag "Newest-first" newest-first)
> +				    (const :tag "Biggest-first" biggest-first)
> +				    (const :tag "Smallest-first" smallest-first)))

I agree with Tomi that Largest sounds better than Biggest here.

[PATCH v2 06/11] sprinter: add unsigned_long printer function

[Ioan-Adrian Ratiu]

We need to output unsigned long values for message and thread
(sum of all message's) file sizes.

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 sprinter-json.c | 9 +++++++++
 sprinter-sexp.c | 9 +++++++++
 sprinter-text.c | 9 +++++++++
 sprinter.h      | 1 +
 4 files changed, 28 insertions(+)

diff --git a/sprinter-json.c b/sprinter-json.c
index 0a077907..de1dbec2 100644
--- a/sprinter-json.c
+++ b/sprinter-json.c
@@ -132,6 +132,14 @@ json_integer (struct sprinter *sp, int val)
 }
 
 static void
+json_unsigned_long (struct sprinter *sp, unsigned long val)
+{
+    struct sprinter_json *spj = json_begin_value (sp);
+
+    fprintf (spj->stream, "%lu", val);
+}
+
+static void
 json_boolean (struct sprinter *sp, notmuch_bool_t val)
 {
     struct sprinter_json *spj = json_begin_value (sp);
@@ -181,6 +189,7 @@ sprinter_json_create (const void *ctx, FILE *stream)
 	    .string = json_string,
 	    .string_len = json_string_len,
 	    .integer = json_integer,
+	    .unsigned_long = json_unsigned_long,
 	    .boolean = json_boolean,
 	    .null = json_null,
 	    .map_key = json_map_key,
diff --git a/sprinter-sexp.c b/sprinter-sexp.c
index 08783e11..3162ad9c 100644
--- a/sprinter-sexp.c
+++ b/sprinter-sexp.c
@@ -169,6 +169,14 @@ sexp_integer (struct sprinter *sp, int val)
 }
 
 static void
+sexp_unsigned_long (struct sprinter *sp, unsigned long val)
+{
+    struct sprinter_sexp *sps = sexp_begin_value (sp);
+
+    fprintf (sps->stream, "%lu", val);
+}
+
+static void
 sexp_boolean (struct sprinter *sp, notmuch_bool_t val)
 {
     struct sprinter_sexp *sps = sexp_begin_value (sp);
@@ -216,6 +224,7 @@ sprinter_sexp_create (const void *ctx, FILE *stream)
 	    .string = sexp_string,
 	    .string_len = sexp_string_len,
 	    .integer = sexp_integer,
+	    .unsigned_long = sexp_unsigned_long,
 	    .boolean = sexp_boolean,
 	    .null = sexp_null,
 	    .map_key = sexp_map_key,
diff --git a/sprinter-text.c b/sprinter-text.c
index 7779488f..5d1607e9 100644
--- a/sprinter-text.c
+++ b/sprinter-text.c
@@ -52,6 +52,14 @@ text_integer (struct sprinter *sp, int val)
 }
 
 static void
+text_unsigned_long (struct sprinter *sp, unsigned long val)
+{
+    struct sprinter_text *sptxt = (struct sprinter_text *) sp;
+
+    fprintf (sptxt->stream, "%lu", val);
+}
+
+static void
 text_boolean (struct sprinter *sp, notmuch_bool_t val)
 {
     struct sprinter_text *sptxt = (struct sprinter_text *) sp;
@@ -123,6 +131,7 @@ sprinter_text_create (const void *ctx, FILE *stream)
 	    .string = text_string,
 	    .string_len = text_string_len,
 	    .integer = text_integer,
+	    .unsigned_long = text_unsigned_long,
 	    .boolean = text_boolean,
 	    .null = text_null,
 	    .map_key = text_map_key,
diff --git a/sprinter.h b/sprinter.h
index f859672f..2495a7d1 100644
--- a/sprinter.h
+++ b/sprinter.h
@@ -34,6 +34,7 @@ typedef struct sprinter {
     void (*string) (struct sprinter *, const char *);
     void (*string_len) (struct sprinter *, const char *, size_t);
     void (*integer) (struct sprinter *, int);
+    void (*unsigned_long) (struct sprinter *, unsigned long);
     void (*boolean) (struct sprinter *, notmuch_bool_t);
     void (*null) (struct sprinter *);
 
-- 
2.13.0

Re: [PATCH v2 06/11] sprinter: add unsigned_long printer function

[Jani Nikula]

On Fri, 19 May 2017, Ioan-Adrian Ratiu <adi@adirat.com> wrote:
> We need to output unsigned long values for message and thread
> (sum of all message's) file sizes.

The sprinter types should be about the types that can be represented
using the format being output, i.e. JSON and S-Expressions, *not* about
the C types. I'd rather see the integer sprinter functions expanded to a
bigger type.

BR,
Jani.


>
> Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
> ---
>  sprinter-json.c | 9 +++++++++
>  sprinter-sexp.c | 9 +++++++++
>  sprinter-text.c | 9 +++++++++
>  sprinter.h      | 1 +
>  4 files changed, 28 insertions(+)
>
> diff --git a/sprinter-json.c b/sprinter-json.c
> index 0a077907..de1dbec2 100644
> --- a/sprinter-json.c
> +++ b/sprinter-json.c
> @@ -132,6 +132,14 @@ json_integer (struct sprinter *sp, int val)
>  }
>  
>  static void
> +json_unsigned_long (struct sprinter *sp, unsigned long val)
> +{
> +    struct sprinter_json *spj = json_begin_value (sp);
> +
> +    fprintf (spj->stream, "%lu", val);
> +}
> +
> +static void
>  json_boolean (struct sprinter *sp, notmuch_bool_t val)
>  {
>      struct sprinter_json *spj = json_begin_value (sp);
> @@ -181,6 +189,7 @@ sprinter_json_create (const void *ctx, FILE *stream)
>  	    .string = json_string,
>  	    .string_len = json_string_len,
>  	    .integer = json_integer,
> +	    .unsigned_long = json_unsigned_long,
>  	    .boolean = json_boolean,
>  	    .null = json_null,
>  	    .map_key = json_map_key,
> diff --git a/sprinter-sexp.c b/sprinter-sexp.c
> index 08783e11..3162ad9c 100644
> --- a/sprinter-sexp.c
> +++ b/sprinter-sexp.c
> @@ -169,6 +169,14 @@ sexp_integer (struct sprinter *sp, int val)
>  }
>  
>  static void
> +sexp_unsigned_long (struct sprinter *sp, unsigned long val)
> +{
> +    struct sprinter_sexp *sps = sexp_begin_value (sp);
> +
> +    fprintf (sps->stream, "%lu", val);
> +}
> +
> +static void
>  sexp_boolean (struct sprinter *sp, notmuch_bool_t val)
>  {
>      struct sprinter_sexp *sps = sexp_begin_value (sp);
> @@ -216,6 +224,7 @@ sprinter_sexp_create (const void *ctx, FILE *stream)
>  	    .string = sexp_string,
>  	    .string_len = sexp_string_len,
>  	    .integer = sexp_integer,
> +	    .unsigned_long = sexp_unsigned_long,
>  	    .boolean = sexp_boolean,
>  	    .null = sexp_null,
>  	    .map_key = sexp_map_key,
> diff --git a/sprinter-text.c b/sprinter-text.c
> index 7779488f..5d1607e9 100644
> --- a/sprinter-text.c
> +++ b/sprinter-text.c
> @@ -52,6 +52,14 @@ text_integer (struct sprinter *sp, int val)
>  }
>  
>  static void
> +text_unsigned_long (struct sprinter *sp, unsigned long val)
> +{
> +    struct sprinter_text *sptxt = (struct sprinter_text *) sp;
> +
> +    fprintf (sptxt->stream, "%lu", val);
> +}
> +
> +static void
>  text_boolean (struct sprinter *sp, notmuch_bool_t val)
>  {
>      struct sprinter_text *sptxt = (struct sprinter_text *) sp;
> @@ -123,6 +131,7 @@ sprinter_text_create (const void *ctx, FILE *stream)
>  	    .string = text_string,
>  	    .string_len = text_string_len,
>  	    .integer = text_integer,
> +	    .unsigned_long = text_unsigned_long,
>  	    .boolean = text_boolean,
>  	    .null = text_null,
>  	    .map_key = text_map_key,
> diff --git a/sprinter.h b/sprinter.h
> index f859672f..2495a7d1 100644
> --- a/sprinter.h
> +++ b/sprinter.h
> @@ -34,6 +34,7 @@ typedef struct sprinter {
>      void (*string) (struct sprinter *, const char *);
>      void (*string_len) (struct sprinter *, const char *, size_t);
>      void (*integer) (struct sprinter *, int);
> +    void (*unsigned_long) (struct sprinter *, unsigned long);
>      void (*boolean) (struct sprinter *, notmuch_bool_t);
>      void (*null) (struct sprinter *);
>  
> -- 
> 2.13.0
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 07/11] lib: thread: add thread total size function

[Ioan-Adrian Ratiu]

Given a thread object this function computes the total disk space used
by all messages contained in the thread.

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 lib/notmuch.h |  7 +++++++
 lib/thread.cc | 12 ++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/lib/notmuch.h b/lib/notmuch.h
index b7bf3526..4f2afbe4 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1185,6 +1185,13 @@ time_t
 notmuch_thread_get_newest_date (notmuch_thread_t *thread);
 
 /**
+ * Get the total disk space used by the thread i.e. a sum of the
+ * thread messages filesizes.
+ */
+unsigned long
+notmuch_thread_get_total_filesize (notmuch_thread_t *thread);
+
+/**
  * Get the tags for 'thread', returning a notmuch_tags_t object which
  * can be used to iterate over all tags.
  *
diff --git a/lib/thread.cc b/lib/thread.cc
index 1a1ecfa5..72878b62 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -596,6 +596,18 @@ notmuch_thread_get_newest_date (notmuch_thread_t *thread)
     return thread->newest;
 }
 
+unsigned long
+notmuch_thread_get_total_filesize (notmuch_thread_t *thread)
+{
+    notmuch_message_node_t *node;
+    unsigned long total_filesize = 0;
+
+    for (node = thread->message_list->head; node; node = node->next)
+	total_filesize += notmuch_message_get_filesize (node->message);
+
+    return total_filesize;
+}
+
 notmuch_tags_t *
 notmuch_thread_get_tags (notmuch_thread_t *thread)
 {
-- 
2.13.0

Re: [PATCH v2 07/11] lib: thread: add thread total size function

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> Given a thread object this function computes the total disk space used
> by all messages contained in the thread.
>

Can you take the approach of id:20170604123235.24466-9-david@tethera.net
and compute the total filesize as the thread is built?

d

[PATCH v2 08/11] notmuch-search: output total_filesize thread result

[Ioan-Adrian Ratiu]

This works for all the search output formats (sexp, json, text).

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 devel/schemata   | 1 +
 notmuch-search.c | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/devel/schemata b/devel/schemata
index 00ebb7a6..c9e05623 100644
--- a/devel/schemata
+++ b/devel/schemata
@@ -153,6 +153,7 @@ search_tags = [string*]
 
 thread_summary = {
     thread:         threadid,
+    total_filesize: unsigned long,
     timestamp:      unix_time,
     date_relative:  string,   # user-friendly timestamp
     matched:        int,      # number of matched messages
diff --git a/notmuch-search.c b/notmuch-search.c
index 65ecfaab..3cf2a65a 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -161,6 +161,7 @@ do_search_threads (search_context_t *ctx)
 	    const char *thread_id = notmuch_thread_get_thread_id (thread);
 	    int matched = notmuch_thread_get_matched_messages (thread);
 	    int total = notmuch_thread_get_total_messages (thread);
+	    unsigned long  total_filesize = notmuch_thread_get_total_filesize (thread);
 	    const char *relative_date = NULL;
 	    notmuch_bool_t first_tag = TRUE;
 
@@ -175,8 +176,9 @@ do_search_threads (search_context_t *ctx)
 
 	    if (format->is_text_printer) {
                 /* Special case for the text formatter */
-		printf ("thread:%s %12s [%d/%d] %s; %s (",
+		printf ("thread:%s %lu %12s [%d/%d] %s; %s (",
 			thread_id,
+			total_filesize,
 			relative_date,
 			matched,
 			total,
@@ -185,6 +187,8 @@ do_search_threads (search_context_t *ctx)
 	    } else { /* Structured Output */
 		format->map_key (format, "thread");
 		format->string (format, thread_id);
+		format->map_key (format, "total_filesize");
+		format->unsigned_long (format, total_filesize);
 		format->map_key (format, "timestamp");
 		format->integer (format, date);
 		format->map_key (format, "date_relative");
-- 
2.13.0

Re: [PATCH v2 08/11] notmuch-search: output total_filesize thread result

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> This works for all the search output formats (sexp, json, text).
>
>  	    if (format->is_text_printer) {
>                  /* Special case for the text formatter */
> -		printf ("thread:%s %12s [%d/%d] %s; %s (",
> +		printf ("thread:%s %lu %12s [%d/%d] %s; %s (",

I'm not convinced about this change. I don't think that everyone finds it
as important as you to know the total filesize in a thread. It also
breaks many tests. One thing I noticed when running the tests is that in
quite a few cases, it prints 0. So I suspect there is some bugs left.

[PATCH v2 09/11] notmuch-show: export message filesize

[Ioan-Adrian Ratiu]

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 notmuch-show.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/notmuch-show.c b/notmuch-show.c
index 7021008e..8229c85c 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -91,6 +91,7 @@ format_message_sprinter (sprinter_t *sp, notmuch_message_t *message)
     notmuch_tags_t *tags;
     time_t date;
     const char *relative_date;
+    unsigned long filesize;
 
     sp->map_key (sp, "id");
     sp->string (sp, notmuch_message_get_message_id (message));
@@ -117,6 +118,10 @@ format_message_sprinter (sprinter_t *sp, notmuch_message_t *message)
 	sp->string (sp, notmuch_message_get_filename (message));
     }
 
+    sp->map_key (sp, "filesize");
+    filesize = notmuch_message_get_filesize (message);
+    sp->unsigned_long (sp, filesize);
+
     sp->map_key (sp, "timestamp");
     date = notmuch_message_get_date (message);
     sp->integer (sp, date);
-- 
2.13.0

Re: [PATCH v2 09/11] notmuch-show: export message filesize

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
> ---
>  notmuch-show.c | 5 +++++
>  1 file changed, 5 insertions(+)

This should update schemata and several tests.

[PATCH v2 10/11] emacs: notmuch-search: add display thread sizes capability

[Ioan-Adrian Ratiu]

By default this is off because it's tiresome to look at all those
numbers in every search view. It's much more pleasant to have it
enabled by default in notmuch-show even if you apply searches and sort
results based on file size.

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 emacs/notmuch.el | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index 5b9c1d07..dbcd67eb 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -73,7 +73,7 @@
     ("subject" . "%s ")
     ("tags" . "(%s)"))
   "Search result formatting. Supported fields are:
-	date, count, authors, subject, tags
+	date, count, total_filesize, authors, subject, tags
 For example:
 	(setq notmuch-search-result-format \(\(\"authors\" . \"%-40s\"\)
 					     \(\"subject\" . \"%s\"\)\)\)
@@ -262,6 +262,12 @@ there will be called at other points of notmuch execution."
  :group 'notmuch-show
  :group 'notmuch-faces)
 
+(defface notmuch-search-thread-total-filesize
+  '((t :inherit default))
+  "Face used in search mode for total thread filesizes."
+  :group 'notmuch-search
+  :group 'notmuch-faces)
+
 (defface notmuch-search-date
   '((t :inherit default))
   "Face used in search mode for dates."
@@ -801,6 +807,9 @@ non-authors is found, assume that all of the authors match."
 
 (defun notmuch-search-insert-field (field format-string result)
   (cond
+   ((string-equal field "total-filesize")
+    (insert (propertize (format format-string (file-size-human-readable (plist-get result :total_filesize)))
+			'face 'notmuch-search-thread-total-filesize)))
    ((string-equal field "date")
     (insert (propertize (format format-string (plist-get result :date_relative))
 			'face 'notmuch-search-date)))
-- 
2.13.0

Re: [PATCH v2 10/11] emacs: notmuch-search: add display thread sizes capability

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> By default this is off because it's tiresome to look at all those
> numbers in every search view. It's much more pleasant to have it
> enabled by default in notmuch-show even if you apply searches and sort
> results based on file size.

this comment is a bit confusing, since the commit doesn't actually do
anything with notmuch-show. Maybe you can clarify that.

Re: [PATCH v2 10/11] emacs: notmuch-search: add display thread sizes capability

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> By default this is off because it's tiresome to look at all those
> numbers in every search view. It's much more pleasant to have it
> enabled by default in notmuch-show even if you apply searches and sort
> results based on file size.
>
> Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
> ---
>  emacs/notmuch.el | 11 ++++++++++-
>  1 file changed, 10 insertions(+), 1 deletion(-)
>
> diff --git a/emacs/notmuch.el b/emacs/notmuch.el
> index 5b9c1d07..dbcd67eb 100644
> --- a/emacs/notmuch.el
> +++ b/emacs/notmuch.el
> @@ -73,7 +73,7 @@
>      ("subject" . "%s ")
>      ("tags" . "(%s)"))
>    "Search result formatting. Supported fields are:
> -	date, count, authors, subject, tags
> +	date, count, total_filesize, authors, subject, tags
>  For example:

>  
>  (defun notmuch-search-insert-field (field format-string result)
>    (cond
> +   ((string-equal field "total-filesize")
> +    (insert (propertize (format format-string (file-size-human-readable (plist-get result :total_filesize)))
> +			'face 'notmuch-search-thread-total-filesize)))

there seems to be some confusion here about total-filesize and
total_filesize; I guess the docstring for the variable should match the
string-equal test.

[PATCH v2 11/11] emacs: notmuch-show: add filesize to headerline

[Ioan-Adrian Ratiu]

Signed-off-by: Ioan-Adrian Ratiu <adi@adirat.com>
---
 emacs/notmuch-show.el | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el
index c670160d..f2cb09d2 100644
--- a/emacs/notmuch-show.el
+++ b/emacs/notmuch-show.el
@@ -446,7 +446,7 @@ unchanged ADDRESS if parsing fails."
       ;; Otherwise format the name and address together.
       (concat p-name " <" p-address ">"))))
 
-(defun notmuch-show-insert-headerline (headers date tags depth)
+(defun notmuch-show-insert-headerline (headers date filesize tags depth)
   "Insert a notmuch style headerline based on HEADERS for a
 message at DEPTH in the current thread."
   (let ((start (point)))
@@ -456,6 +456,8 @@ message at DEPTH in the current thread."
 	    " ("
 	    date
 	    ") ("
+	    (file-size-human-readable filesize)
+	    ") ("
 	    (notmuch-tag-format-tags tags tags)
 	    ")\n")
     (overlay-put (make-overlay start (point)) 'face 'notmuch-message-summary-face)))
@@ -1038,6 +1040,7 @@ is t, hide the part initially and show the button."
 					    (plist-get msg :date_relative)
 					  nil)
 					(plist-get headers :Date))
+				    (plist-get msg :filesize)
 				    (plist-get msg :tags) depth)
 
     (setq content-start (point-marker))
-- 
2.13.0

Re: [PATCH v2 11/11] emacs: notmuch-show: add filesize to headerline

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

>  	    date
>  	    ") ("
> +	    (file-size-human-readable filesize)
> +	    ") ("
>  	    (notmuch-tag-format-tags tags tags)


I think this should probably be optional.  I don't know that everyone
will want to give up the screen space for this. I'm not sure how hard it
would be to have a fully customizable format like notmuch-search-result-format

Another thing I noticed is that your emacs code crashes when used
against an older version of notmuch.  You can either make the emacs code
more robust, or use the format-version machinery to ensure that the
emacs binary version is new enough.  The former is probably nicer since
you most likely want to gracefully handle databases without the filesize
value slot.

Finally, with the current patch series, there are over 200 failing
tests. That's obviously something to improve ;).

d

Re: [PATCH v2 00/11] Add filesize index, search, sort & emacs UI

[Jani Nikula]

On Fri, 19 May 2017, Ioan-Adrian Ratiu <adi@adirat.com> wrote:
> I'd like to add a feature to quickly work with mail file sizes
> because using custom scripts / external programs which parse
> maildir contents is slow, and non-intuitive, especially since
> notmuch does incremental parsing and has such a nice emacs UI.

Before we dive into the details, I guess at least I'd like to see better
rationale for the feature. I don't see myself searching or sorting based
on message size. What's the use case? (I wouldn't mind adding the
message size to the formatted output, but that doesn't require indexing
the size or adding the search support for it.)

One fundamental issue David noted on IRC, it'll be hard to decide what
to index for "message" size when there are duplicates that typically
have different sizes. For other things, we've indexed whatever comes
first, but I suppose we'd like to fix that.

BR,
Jani.

>
>
> Ioan-Adrian Ratiu (11):
>   lib: message: index message file sizes
>   lib: database: store message filesize & add range processor
>   notmuch-search: add filesize based sort order
>   emacs: make notmuch-search-oldest-first generic
>   emacs: notmuch-search: add filesize sorting
>   sprinter: add unsigned_long printer function
>   lib: thread: add thread total size function
>   notmuch-search: output total_filesize thread result
>   notmuch-show: export message filesize
>   emacs: notmuch-search: add display thread sizes capability
>   emacs: notmuch-show: add filesize to headerline
>
>  devel/schemata         |  1 +
>  doc/notmuch-emacs.rst  |  4 ++--
>  emacs/notmuch-hello.el | 24 +++++++++++++-----------
>  emacs/notmuch-jump.el  | 11 +++++------
>  emacs/notmuch-lib.el   |  9 ++++++---
>  emacs/notmuch-show.el  |  5 ++++-
>  emacs/notmuch-tree.el  |  2 +-
>  emacs/notmuch.el       | 48 +++++++++++++++++++++++++++++++++---------------
>  lib/database-private.h |  1 +
>  lib/database.cc        |  6 ++++++
>  lib/index.cc           | 10 ++++++++++
>  lib/message-file.c     | 18 +++++++++++++++++-
>  lib/message.cc         | 29 +++++++++++++++++++++++++++++
>  lib/notmuch-private.h  | 16 ++++++++++++++++
>  lib/notmuch.h          | 21 +++++++++++++++++++++
>  lib/query.cc           |  6 ++++++
>  lib/thread.cc          | 12 ++++++++++++
>  notmuch-search.c       |  8 +++++++-
>  notmuch-show.c         |  5 +++++
>  sprinter-json.c        |  9 +++++++++
>  sprinter-sexp.c        |  9 +++++++++
>  sprinter-text.c        |  9 +++++++++
>  sprinter.h             |  1 +
>  23 files changed, 223 insertions(+), 41 deletions(-)
>
> -- 
> 2.13.0
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH v2 00/11] Add filesize index, search, sort & emacs UI

[Ioan-Adrian Ratiu]

On Tue, 23 May 2017, Jani Nikula <jani@nikula.org> wrote:
> On Fri, 19 May 2017, Ioan-Adrian Ratiu <adi@adirat.com> wrote:
>> I'd like to add a feature to quickly work with mail file sizes
>> because using custom scripts / external programs which parse
>> maildir contents is slow, and non-intuitive, especially since
>> notmuch does incremental parsing and has such a nice emacs UI.
>
> Before we dive into the details, I guess at least I'd like to see better
> rationale for the feature. I don't see myself searching or sorting based
> on message size. What's the use case? (I wouldn't mind adding the
> message size to the formatted output, but that doesn't require indexing
> the size or adding the search support for it.)

The use case is very simple: I routinely get big mail and want to know
about it to take various actions (extract attach, forward, archive etc).

Of course I can always write a script to search the maildir but that
takes too long, I'd very much rather have incremental indexing,
instant search and a very nice GUI front-end (emacs saved searches in
my case).

With this series I now have saved searches based on filesize which tell
me exactly instantly how many messages I have in a certain size range.

>
> One fundamental issue David noted on IRC, it'll be hard to decide what
> to index for "message" size when there are duplicates that typically
> have different sizes. For other things, we've indexed whatever comes
> first, but I suppose we'd like to fix that.

Yes, agreed, and I'll go even further: notmuch search shows thread
results - what I did in this patch series is sum up the total messages
file sizes in a thread for the search results.

We could do something similar, i.e. add all duplicate file sizes to the
"message" size and those in turn get added to a thread size. I'm not
saying we should necessarily do this, I'm just pointing it out as an
alternative which would work very well for me.

Ionel

>
> BR,
> Jani.
>
>>
>>
>> Ioan-Adrian Ratiu (11):
>>   lib: message: index message file sizes
>>   lib: database: store message filesize & add range processor
>>   notmuch-search: add filesize based sort order
>>   emacs: make notmuch-search-oldest-first generic
>>   emacs: notmuch-search: add filesize sorting
>>   sprinter: add unsigned_long printer function
>>   lib: thread: add thread total size function
>>   notmuch-search: output total_filesize thread result
>>   notmuch-show: export message filesize
>>   emacs: notmuch-search: add display thread sizes capability
>>   emacs: notmuch-show: add filesize to headerline
>>
>>  devel/schemata         |  1 +
>>  doc/notmuch-emacs.rst  |  4 ++--
>>  emacs/notmuch-hello.el | 24 +++++++++++++-----------
>>  emacs/notmuch-jump.el  | 11 +++++------
>>  emacs/notmuch-lib.el   |  9 ++++++---
>>  emacs/notmuch-show.el  |  5 ++++-
>>  emacs/notmuch-tree.el  |  2 +-
>>  emacs/notmuch.el       | 48 +++++++++++++++++++++++++++++++++---------------
>>  lib/database-private.h |  1 +
>>  lib/database.cc        |  6 ++++++
>>  lib/index.cc           | 10 ++++++++++
>>  lib/message-file.c     | 18 +++++++++++++++++-
>>  lib/message.cc         | 29 +++++++++++++++++++++++++++++
>>  lib/notmuch-private.h  | 16 ++++++++++++++++
>>  lib/notmuch.h          | 21 +++++++++++++++++++++
>>  lib/query.cc           |  6 ++++++
>>  lib/thread.cc          | 12 ++++++++++++
>>  notmuch-search.c       |  8 +++++++-
>>  notmuch-show.c         |  5 +++++
>>  sprinter-json.c        |  9 +++++++++
>>  sprinter-sexp.c        |  9 +++++++++
>>  sprinter-text.c        |  9 +++++++++
>>  sprinter.h             |  1 +
>>  23 files changed, 223 insertions(+), 41 deletions(-)
>>
>> -- 
>> 2.13.0
>>
>> _______________________________________________
>> notmuch mailing list
>> notmuch@notmuchmail.org
>> https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH v2 00/11] Add filesize index, search, sort & emacs UI

[David Bremner]

Ioan-Adrian Ratiu <adi@adirat.com> writes:

> I'd like to add a feature to quickly work with mail file sizes
> because using custom scripts / external programs which parse
> maildir contents is slow, and non-intuitive, especially since
> notmuch does incremental parsing and has such a nice emacs UI.

I just remembered (and olly confirmed) that there is some work in
progress on adding file sizes to the xapian query parser. Finishing that
would benefit more people, and allow the feature you want in notmuch
with less code in notmuch. Maybe you could have a look and see if it's
practical to finish this off (or consult with the original student who
seems to have abandonded it, but maybe not).

     https://github.com/xapian/xapian/pull/135

Re: [PATCH v2 00/11] Add filesize index, search, sort & emacs UI

[David Bremner]

David Bremner <david@tethera.net> writes:

> Ioan-Adrian Ratiu <adi@adirat.com> writes:
>
>> I'd like to add a feature to quickly work with mail file sizes
>> because using custom scripts / external programs which parse
>> maildir contents is slow, and non-intuitive, especially since
>> notmuch does incremental parsing and has such a nice emacs UI.
>
> I just remembered (and olly confirmed) that there is some work in
> progress on adding file sizes to the xapian query parser. Finishing that
> would benefit more people, and allow the feature you want in notmuch
> with less code in notmuch. Maybe you could have a look and see if it's
> practical to finish this off (or consult with the original student who
> seems to have abandonded it, but maybe not).
>
>      https://github.com/xapian/xapian/pull/135

When I take a second look at that pull request, it's more a layer on top
of the series proposed for notmuch, to provide human readable units.
It (and the omega file size indexing it extends) nonetheless might
be interesting for you.