/* mailstore-files.c - Original notmuch mail store - a collection of * plain-text email messages (one message per file). * * Copyright © 2009 Carl Worth * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see http://www.gnu.org/licenses/ . * * Authors: Carl Worth * Michal Sojka */ #define _GNU_SOURCE /* For asprintf() */ #include "notmuch.h" #include "mailstore-private.h" #include #include #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) struct mailstore_priv { void (*tag_new)(notmuch_message_t *message, const char *filename); int (*tag_renamed)(notmuch_message_t *message, const char *filename); }; struct maildir_flag_tag { char flag; const char *tag; bool inverse; }; /* ASCII ordered table of Maildir flags and assiciated tags */ struct maildir_flag_tag flag2tag[] = { { 'D', "draft", false}, { 'F', "flagged", false}, { 'P', "passed", false}, { 'R', "replied", false}, { 'S', "unread", true }, { 'T', "delete", false}, }; typedef struct _filename_node { char *filename; struct _filename_node *next; } _filename_node_t; typedef struct _filename_list { _filename_node_t *head; _filename_node_t **tail; } _filename_list_t; typedef struct _indexing_context_priv { _filename_list_t *removed_files; _filename_list_t *removed_directories; } _indexing_context_priv_t; static _filename_list_t * _filename_list_create (const void *ctx) { _filename_list_t *list; list = talloc (ctx, _filename_list_t); if (list == NULL) return NULL; list->head = NULL; list->tail = &list->head; return list; } static void _filename_list_add (_filename_list_t *list, const char *filename) { _filename_node_t *node = talloc (list, _filename_node_t); node->filename = talloc_strdup (list, filename); node->next = NULL; *(list->tail) = node; list->tail = &node->next; } static void tag_inbox_and_unread (notmuch_message_t *message, const char *filename) { (void)filename; notmuch_message_add_tag (message, "inbox"); notmuch_message_add_tag (message, "unread"); } static int dirent_sort_inode (const struct dirent **a, const struct dirent **b) { return ((*a)->d_ino < (*b)->d_ino) ? -1 : 1; } static int dirent_sort_strcmp_name (const struct dirent **a, const struct dirent **b) { return strcmp ((*a)->d_name, (*b)->d_name); } /* Test if the directory looks like a Maildir directory. * * Search through the array of directory entries to see if we can find all * three subdirectories typical for Maildir, that is "new", "cur", and "tmp". * * Return 1 if the directory looks like a Maildir and 0 otherwise. */ static int _entries_resemble_maildir (struct dirent **entries, int count) { int i, found = 0; for (i = 0; i < count; i++) { if (entries[i]->d_type != DT_DIR && entries[i]->d_type != DT_UNKNOWN) continue; if (strcmp(entries[i]->d_name, "new") == 0 || strcmp(entries[i]->d_name, "cur") == 0 || strcmp(entries[i]->d_name, "tmp") == 0) { found++; if (found == 3) return 1; } } return 0; } static int tag_from_maildir_flags (notmuch_message_t *message, const char *filename) { const char *flags, *p; char f; bool valid, unread; unsigned i; flags = strstr(filename, ":2,"); if (!flags) return -1; flags += 3; /* Check that the letters are valid Maildir flags */ f = 0; valid = true; for (p=flags; valid && *p; p++) { switch (*p) { case 'P': case 'R': case 'S': case 'T': case 'D': case 'F': if (*p > f) f=*p; else valid = false; break; default: valid = false; } } if (!valid) { fprintf (stderr, "Warning: Invalid maildir flags in filename %s\n", filename); return -1; } notmuch_message_freeze(message); unread = true; for (i = 0; i < ARRAY_SIZE(flag2tag); i++) { if ((strchr(flags, flag2tag[i].flag) != NULL) ^ flag2tag[i].inverse) { notmuch_message_add_tag (message, flag2tag[i].tag); } else { notmuch_message_remove_tag (message, flag2tag[i].tag); } } notmuch_message_thaw(message); /* From now on, we can synchronize the tags from the database to * the mailstore. */ notmuch_message_set_flag(message, NOTMUCH_MESSAGE_FLAG_TAGS_INVALID, FALSE); return 0; } static void get_new_flags(notmuch_message_t *message, char *flags) { notmuch_tags_t *tags; const char *tag; unsigned i; char *p; for (i = 0; i < ARRAY_SIZE(flag2tag); i++) flags[i] = flag2tag[i].inverse ? flag2tag[i].flag : '\0'; for (tags = notmuch_message_get_tags (message); notmuch_tags_valid (tags); notmuch_tags_move_to_next (tags)) { tag = notmuch_tags_get (tags); for (i = 0; i < ARRAY_SIZE(flag2tag); i++) { if (strcmp(tag, flag2tag[i].tag) == 0) flags[i] = flag2tag[i].inverse ? '\0' : flag2tag[i].flag; } } p = flags; for (i = 0; i < ARRAY_SIZE(flag2tag); i++) { if (flags[i]) *p++ = flags[i]; } *p = '\0'; } static char * get_subdir (char *filename) { char *p, *subdir = NULL; p = filename + strlen (filename) - 1; while (p > filename + 3 && *p != '/') p--; if (*p == '/') { subdir = p - 3; if (subdir > filename && *(subdir - 1) != '/') subdir = NULL; } return subdir; } /* Store maildir-related tags as maildir flags */ static notmuch_private_status_t maildir_sync_tags (notmuch_mailstore_t *mailstore, notmuch_message_t *message) { char flags[ARRAY_SIZE(flag2tag)+1]; const char *filename, *p; char *filename_new, *subdir = NULL; int ret; (void)mailstore; get_new_flags (message, flags); filename = notmuch_message_get_filename (message); p = strstr(filename, ":2,"); if (!p) p = filename + strlen(filename); filename_new = talloc_size(message, (p-filename) + 3 + sizeof(flags)); if (unlikely (filename_new == NULL)) return NOTMUCH_STATUS_OUT_OF_MEMORY; memcpy(filename_new, filename, p-filename); filename_new[p-filename] = '\0'; /* If message is in new/ move it under cur/. */ subdir = get_subdir (filename_new); if (subdir && memcmp (subdir, "new/", 4) == 0) memcpy (subdir, "cur/", 4); strcpy (filename_new+(p-filename), ":2,"); strcpy (filename_new+(p-filename)+3, flags); if (strcmp (filename, filename_new) != 0) { ret = rename (filename, filename_new); if (ret == -1) { perror (talloc_asprintf (message, "rename of %s to %s failed", filename, filename_new)); exit (1); } return _notmuch_message_rename (message, filename_new); /* _notmuch_message_sync is our caller. Do not call it here. */ } return NOTMUCH_STATUS_SUCCESS; } static void tag_inbox_and_maildir_flags (notmuch_message_t *message, const char *filename) { notmuch_message_add_tag (message, "inbox"); if (tag_from_maildir_flags(message, filename) != 0) notmuch_message_add_tag (message, "unread"); } /* Examine 'path' recursively as follows: * * o Ask the filesystem for the mtime of 'path' (fs_mtime) * o Ask the database for its timestamp of 'path' (db_mtime) * * o Ask the filesystem for files and directories within 'path' * (via scandir and stored in fs_entries) * o Ask the database for files and directories within 'path' * (db_files and db_subdirs) * * o Pass 1: For each directory in fs_entries, recursively call into * this same function. * * o Pass 2: If 'fs_mtime' > 'db_mtime', then walk fs_entries * simultaneously with db_files and db_subdirs. Look for one of * three interesting cases: * * 1. Regular file in fs_entries and not in db_files * This is a new file to add_message into the database. * * 2. Filename in db_files not in fs_entries. * This is a file that has been removed from the mail store. * * 3. Directory in db_subdirs not in fs_entries * This is a directory that has been removed from the mail store. * * Note that the addition of a directory is not interesting here, * since that will have been taken care of in pass 1. Also, we * don't immediately act on file/directory removal since we must * ensure that in the case of a rename that the new filename is * added before the old filename is removed, (so that no * information is lost from the database). * * o Tell the database to update its time of 'path' to 'fs_mtime' */ static notmuch_status_t add_files_recursive (notmuch_mailstore_t *mailstore, const char *path, notmuch_indexing_context_t *state) { DIR *dir = NULL; struct dirent *entry = NULL; char *next = NULL; time_t fs_mtime, db_mtime; notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS; notmuch_message_t *message = NULL; struct dirent **fs_entries = NULL; int i, num_fs_entries; notmuch_directory_t *directory; notmuch_filenames_t *db_files = NULL; notmuch_filenames_t *db_subdirs = NULL; struct stat st; notmuch_bool_t is_maildir, new_directory; _indexing_context_priv_t *priv = state->priv; struct mailstore_priv *mailstore_priv = mailstore->priv; notmuch_database_t *notmuch = mailstore->notmuch; if (stat (path, &st)) { fprintf (stderr, "Error reading directory %s: %s\n", path, strerror (errno)); return NOTMUCH_STATUS_FILE_ERROR; } /* This is not an error since we may have recursed based on a * symlink to a regular file, not a directory, and we don't know * that until this stat. */ if (! S_ISDIR (st.st_mode)) return NOTMUCH_STATUS_SUCCESS; fs_mtime = st.st_mtime; directory = notmuch_database_get_directory (notmuch, path); db_mtime = notmuch_directory_get_mtime (directory); if (db_mtime == 0) { new_directory = TRUE; db_files = NULL; db_subdirs = NULL; } else { new_directory = FALSE; db_files = notmuch_directory_get_child_files (directory); db_subdirs = notmuch_directory_get_child_directories (directory); } /* If the database knows about this directory, then we sort based * on strcmp to match the database sorting. Otherwise, we can do * inode-based sorting for faster filesystem operation. */ num_fs_entries = scandir (path, &fs_entries, 0, new_directory ? dirent_sort_inode : dirent_sort_strcmp_name); if (num_fs_entries == -1) { fprintf (stderr, "Error opening directory %s: %s\n", path, strerror (errno)); ret = NOTMUCH_STATUS_FILE_ERROR; goto DONE; } /* Pass 1: Recurse into all sub-directories. */ is_maildir = _entries_resemble_maildir (fs_entries, num_fs_entries); for (i = 0; i < num_fs_entries; i++) { if (state->interrupted) break; entry = fs_entries[i]; /* We only want to descend into directories. * But symlinks can be to directories too, of course. * * And if the filesystem doesn't tell us the file type in the * scandir results, then it might be a directory (and if not, * then we'll stat and return immediately in the next level of * recursion). */ if (entry->d_type != DT_DIR && entry->d_type != DT_LNK && entry->d_type != DT_UNKNOWN) { continue; } /* Ignore special directories to avoid infinite recursion. * Also ignore the .notmuch directory and any "tmp" directory * that appears within a maildir. */ /* XXX: Eventually we'll want more sophistication to let the * user specify files to be ignored. */ if (strcmp (entry->d_name, ".") == 0 || strcmp (entry->d_name, "..") == 0 || (is_maildir && strcmp (entry->d_name, "tmp") == 0) || strcmp (entry->d_name, ".notmuch") ==0) { continue; } next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); status = add_files_recursive (mailstore, next, state); if (status && ret == NOTMUCH_STATUS_SUCCESS) ret = status; talloc_free (next); next = NULL; } /* If this directory hasn't been modified since the last * "notmuch new", then we can skip the second pass entirely. */ if (fs_mtime <= db_mtime) goto DONE; /* Pass 2: Scan for new files, removed files, and removed directories. */ for (i = 0; i < num_fs_entries; i++) { if (state->interrupted) break; entry = fs_entries[i]; /* Check if we've walked past any names in db_files or * db_subdirs. If so, these have been deleted. */ while (notmuch_filenames_valid (db_files) && strcmp (notmuch_filenames_get (db_files), entry->d_name) < 0) { char *absolute = talloc_asprintf (priv->removed_files, "%s/%s", path, notmuch_filenames_get (db_files)); _filename_list_add (priv->removed_files, absolute); notmuch_filenames_move_to_next (db_files); } while (notmuch_filenames_valid (db_subdirs) && strcmp (notmuch_filenames_get (db_subdirs), entry->d_name) <= 0) { const char *filename = notmuch_filenames_get (db_subdirs); if (strcmp (filename, entry->d_name) < 0) { char *absolute = talloc_asprintf (priv->removed_directories, "%s/%s", path, filename); _filename_list_add (priv->removed_directories, absolute); } notmuch_filenames_move_to_next (db_subdirs); } /* If we're looking at a symlink, we only want to add it if it * links to a regular file, (and not to a directory, say). * * Similarly, if the file is of unknown type (due to filesytem * limitations), then we also need to look closer. * * In either case, a stat does the trick. */ if (entry->d_type == DT_LNK || entry->d_type == DT_UNKNOWN) { int err; next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); err = stat (next, &st); talloc_free (next); next = NULL; /* Don't emit an error for a link pointing nowhere, since * the directory-traversal pass will have already done * that. */ if (err) continue; if (! S_ISREG (st.st_mode)) continue; } else if (entry->d_type != DT_REG) { continue; } /* Don't add a file that we've added before. */ if (notmuch_filenames_valid (db_files) && strcmp (notmuch_filenames_get (db_files), entry->d_name) == 0) { notmuch_filenames_move_to_next (db_files); continue; } /* We're now looking at a regular file that doesn't yet exist * in the database, so add it. */ next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); state->processed_files++; if (state->verbose && state->print_verbose_cb) state->print_verbose_cb(state, next); status = notmuch_database_add_message (notmuch, next, &message); switch (status) { /* success */ case NOTMUCH_STATUS_SUCCESS: state->added_messages++; mailstore_priv->tag_new (message, next); break; /* Non-fatal issues (go on to next file) */ case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: if (mailstore_priv->tag_renamed) mailstore_priv->tag_renamed (message, next); break; case NOTMUCH_STATUS_FILE_NOT_EMAIL: fprintf (stderr, "Note: Ignoring non-mail file: %s\n", next); break; /* Fatal issues. Don't process anymore. */ case NOTMUCH_STATUS_READ_ONLY_DATABASE: case NOTMUCH_STATUS_XAPIAN_EXCEPTION: case NOTMUCH_STATUS_OUT_OF_MEMORY: fprintf (stderr, "Error: %s. Halting processing.\n", notmuch_status_to_string (status)); ret = status; goto DONE; default: case NOTMUCH_STATUS_FILE_ERROR: case NOTMUCH_STATUS_NULL_POINTER: case NOTMUCH_STATUS_TAG_TOO_LONG: case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: case NOTMUCH_STATUS_LAST_STATUS: INTERNAL_ERROR ("add_message returned unexpected value: %d", status); goto DONE; } if (message) { notmuch_message_destroy (message); message = NULL; } if (state->print_progress && state->print_progress_cb) { state->print_progress = 0; state->print_progress_cb (state); } talloc_free (next); next = NULL; } if (state->interrupted) goto DONE; /* Now that we've walked the whole filesystem list, anything left * over in the database lists has been deleted. */ while (notmuch_filenames_valid (db_files)) { char *absolute = talloc_asprintf (priv->removed_files, "%s/%s", path, notmuch_filenames_get (db_files)); _filename_list_add (priv->removed_files, absolute); notmuch_filenames_move_to_next (db_files); } while (notmuch_filenames_valid (db_subdirs)) { char *absolute = talloc_asprintf (priv->removed_directories, "%s/%s", path, notmuch_filenames_get (db_subdirs)); _filename_list_add (priv->removed_directories, absolute); notmuch_filenames_move_to_next (db_subdirs); } if (! state->interrupted) { status = notmuch_directory_set_mtime (directory, fs_mtime); if (status && ret == NOTMUCH_STATUS_SUCCESS) ret = status; } DONE: if (next) talloc_free (next); if (entry) free (entry); if (dir) closedir (dir); if (fs_entries) free (fs_entries); if (db_subdirs) notmuch_filenames_destroy (db_subdirs); if (db_files) notmuch_filenames_destroy (db_files); if (directory) notmuch_directory_destroy (directory); return ret; } /* XXX: This should be merged with the add_files function since it * shares a lot of logic with it. */ /* Recursively count all regular files in path and all sub-directories * of path. The result is added to *count (which should be * initialized to zero by the top-level caller before calling * count_files). */ static void count_files (notmuch_mailstore_t *mailstore, const char *path, int *count, volatile sig_atomic_t *interrupted) { struct dirent *entry = NULL; char *next; struct stat st; struct dirent **fs_entries = NULL; int num_fs_entries = scandir (path, &fs_entries, 0, dirent_sort_inode); int i = 0; (void)mailstore; if (num_fs_entries == -1) { fprintf (stderr, "Warning: failed to open directory %s: %s\n", path, strerror (errno)); goto DONE; } while (!*interrupted) { if (i == num_fs_entries) break; entry = fs_entries[i++]; /* Ignore special directories to avoid infinite recursion. * Also ignore the .notmuch directory. */ /* XXX: Eventually we'll want more sophistication to let the * user specify files to be ignored. */ if (strcmp (entry->d_name, ".") == 0 || strcmp (entry->d_name, "..") == 0 || strcmp (entry->d_name, ".notmuch") == 0) { continue; } if (asprintf (&next, "%s/%s", path, entry->d_name) == -1) { next = NULL; fprintf (stderr, "Error descending from %s to %s: Out of memory\n", path, entry->d_name); continue; } stat (next, &st); if (S_ISREG (st.st_mode)) { *count = *count + 1; if (*count % 1000 == 0) { printf ("Found %d files so far.\r", *count); fflush (stdout); } } else if (S_ISDIR (st.st_mode)) { count_files (mailstore, next, count, interrupted); } free (next); } DONE: if (entry) free (entry); if (fs_entries) free (fs_entries); } /* Recursively remove all filenames from the database referring to * 'path' (or to any of its children). */ static void _remove_directory (void *ctx, notmuch_database_t *notmuch, const char *path, int *renamed_files, int *removed_files) { notmuch_directory_t *directory; notmuch_filenames_t *files, *subdirs; notmuch_status_t status; char *absolute; directory = notmuch_database_get_directory (notmuch, path); for (files = notmuch_directory_get_child_files (directory); notmuch_filenames_valid (files); notmuch_filenames_move_to_next (files)) { absolute = talloc_asprintf (ctx, "%s/%s", path, notmuch_filenames_get (files)); status = notmuch_database_remove_message (notmuch, absolute); if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) *renamed_files = *renamed_files + 1; else *removed_files = *removed_files + 1; talloc_free (absolute); } for (subdirs = notmuch_directory_get_child_directories (directory); notmuch_filenames_valid (subdirs); notmuch_filenames_move_to_next (subdirs)) { absolute = talloc_asprintf (ctx, "%s/%s", path, notmuch_filenames_get (subdirs)); _remove_directory (ctx, notmuch, absolute, renamed_files, removed_files); talloc_free (absolute); } notmuch_directory_destroy (directory); } static notmuch_private_status_t index_new(notmuch_mailstore_t *mailstore, const char* path, notmuch_indexing_context_t *indexing_ctx) { _indexing_context_priv_t *priv; _filename_node_t *f; notmuch_status_t status, ret; notmuch_database_t *notmuch = mailstore->notmuch; priv = talloc(NULL, _indexing_context_priv_t); indexing_ctx->priv = priv; if (priv == NULL) return NOTMUCH_STATUS_OUT_OF_MEMORY; priv->removed_files = _filename_list_create (priv); priv->removed_directories = _filename_list_create (priv); ret = add_files_recursive(mailstore, path, indexing_ctx); indexing_ctx->removed_files = 0; indexing_ctx->renamed_files = 0; for (f = priv->removed_files->head; f; f = f->next) { status = notmuch_database_remove_message (notmuch, f->filename); if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) indexing_ctx->renamed_files++; else indexing_ctx->removed_files++; } for (f = priv->removed_directories->head; f; f = f->next) { _remove_directory (priv, notmuch, f->filename, &indexing_ctx->renamed_files, &indexing_ctx->removed_files); } talloc_free(priv); return ret; } static FILE * open_file(notmuch_mailstore_t *mailstore, const char *filename) { const char *db_path, *abs_filename; db_path = notmuch_database_get_path(mailstore->notmuch); abs_filename = talloc_asprintf(mailstore, "%s/%s", db_path, filename); if (unlikely(abs_filename == NULL)) return NULL; return fopen (abs_filename, "r"); } struct mailstore_priv files_priv = { .tag_new = tag_inbox_and_unread, }; /* Original notmuch mail store */ struct _notmuch_mailstore mailstore_files = { .type = "files", .count_files = count_files, .index_new = index_new, .sync_tags = NULL, .open_file = open_file, .priv = &files_priv, }; struct mailstore_priv maildir_priv = { .tag_new = tag_inbox_and_maildir_flags, .tag_renamed = tag_from_maildir_flags, }; /* Similar to mailstore_files, but does bi-directional synchronization between certain tags and maildir flags */ struct _notmuch_mailstore mailstore_maildir = { .type = "maildir", .count_files = count_files, .index_new = index_new, .sync_tags = maildir_sync_tags, .open_file = open_file, .priv = &maildir_priv, };