From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by olra.theworths.org (Postfix) with ESMTP id 33790431FB6 for ; Tue, 4 Sep 2012 11:53:12 -0700 (PDT) X-Virus-Scanned: Debian amavisd-new at olra.theworths.org X-Spam-Flag: NO X-Spam-Score: -0.799 X-Spam-Level: X-Spam-Status: No, score=-0.799 tagged_above=-999 required=5 tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, DKIM_VALID_AU=-0.1, FREEMAIL_FROM=0.001, RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled Received: from olra.theworths.org ([127.0.0.1]) by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id A2Pzhqsv8xA6 for ; Tue, 4 Sep 2012 11:53:11 -0700 (PDT) Received: from mail-ee0-f53.google.com (mail-ee0-f53.google.com [74.125.83.53]) (using TLSv1 with cipher RC4-SHA (128/128 bits)) (No client certificate requested) by olra.theworths.org (Postfix) with ESMTPS id 4B2D0431FAF for ; Tue, 4 Sep 2012 11:53:11 -0700 (PDT) Received: by eekb47 with SMTP id b47so2947704eek.26 for ; Tue, 04 Sep 2012 11:53:08 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=from:to:subject:date:message-id:x-mailer; bh=dozgH3mWp36ddi1pgxlbLiGB1CaZxhWjD8Nva5iDvAc=; b=mA8A+Gnzud/GFb9xDcnbG27IpaeEWC4DvYRj9DEIZbpASSVp4Medlim0oZbjXHM37f nmFHfPmz26nHqklfz1rR8OyxM9ssCZGaCxKAqaQdWYuKZZfFwo3TKIWDOUWzeFxI2u2N gEH5pIC2G1LyihObX9omjUr+E/AtsWcgW+Pvk44Mv1eM6AykJtCrxql0JhOGzV4LrJq0 X2Ik17BOtWq6AXtsDGNB9ZEbdT7m2Ft+3JmIiBimQ5VsYhi3GFARroq8a4LtOBWFAzK8 z1qF7b7N+sPeVsue23xzVnQfm/hptZVFhbT2o4JNf3Y3vCgLazvLPl9ZU9lZ1FMdeOtK y+Fg== Received: by 10.14.204.72 with SMTP id g48mr27528941eeo.45.1346784788676; Tue, 04 Sep 2012 11:53:08 -0700 (PDT) Received: from localhost ([2001:470:1f0b:14dd:224:d7ff:fee2:c588]) by mx.google.com with ESMTPS id e7sm47648765eep.2.2012.09.04.11.53.07 (version=TLSv1/SSLv3 cipher=OTHER); Tue, 04 Sep 2012 11:53:07 -0700 (PDT) From: Dmitry Kurochkin To: notmuch@notmuchmail.org Subject: [PATCH] Add notmuch-remove-duplicates.py script to contrib. Date: Tue, 4 Sep 2012 22:53:05 +0400 Message-Id: <1346784785-19746-1-git-send-email-dmitry.kurochkin@gmail.com> X-Mailer: git-send-email 1.7.10.4 X-BeenThere: notmuch@notmuchmail.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: "Use and development of the notmuch mail system." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 04 Sep 2012 18:53:12 -0000 The script removes duplicate message files. It takes no options. Files are assumed duplicates if their content is the same except for ignored headers. Currently, the only ignored header is Received:. --- contrib/notmuch-remove-duplicates.py | 95 ++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100755 contrib/notmuch-remove-duplicates.py diff --git a/contrib/notmuch-remove-duplicates.py b/contrib/notmuch-remove-duplicates.py new file mode 100755 index 0000000..dbe2e25 --- /dev/null +++ b/contrib/notmuch-remove-duplicates.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +import sys + +IGNORED_HEADERS = [ "Received:" ] + +if len(sys.argv) != 1: + print "Usage: %s" % sys.argv[0] + print + print "The script removes duplicate message files. Takes no options." + print "Requires notmuch python module." + print + print "Files are assumed duplicates if their content is the same" + print "except for the following headers: %s." % ", ".join(IGNORED_HEADERS) + exit(1) + +import notmuch +import os +import time + +class MailComparator: + """Checks if mail files are duplicates.""" + def __init__(self, filename): + self.filename = filename + self.mail = self.readFile(self.filename) + + def isDuplicate(self, filename): + return self.mail == self.readFile(filename) + + @staticmethod + def readFile(filename): + with open(filename) as f: + data = "" + while True: + line = f.readline() + for header in IGNORED_HEADERS: + if line.startswith(header): + # skip header continuation lines + while True: + line = f.readline() + if len(line) == 0 or line[0] not in [" ", "\t"]: + break + break + else: + data += line + if line == "\n": + break + data += f.read() + return data + +db = notmuch.Database() +query = db.create_query('*') +print "Number of messages: %s" % query.count_messages() + +files_count = 0 +for root, dirs, files in os.walk(db.get_path()): + if not root.startswith(os.path.join(db.get_path(), ".notmuch/")): + files_count += len(files) +print "Number of files: %s" % files_count +print "Estimated number of duplicates: %s" % (files_count - query.count_messages()) + +msgs = query.search_messages() +msg_count = 0 +suspected_duplicates_count = 0 +duplicates_count = 0 +timestamp = time.time() +for msg in msgs: + msg_count += 1 + if len(msg.get_filenames()) > 1: + filenames = msg.get_filenames() + comparator = MailComparator(filenames.next()) + for filename in filenames: + if os.path.realpath(comparator.filename) == os.path.realpath(filename): + print "Message '%s' has filenames pointing to the same file: '%s' '%s'" % (msg.get_message_id(), comparator.filename, filename) + elif comparator.isDuplicate(filename): + os.remove(filename) + duplicates_count += 1 + else: + #print "Potential duplicates: %s" % msg.get_message_id() + suspected_duplicates_count += 1 + + new_timestamp = time.time() + if new_timestamp - timestamp > 1: + timestamp = new_timestamp + sys.stdout.write("\rProcessed %s messages, removed %s duplicates..." % (msg_count, duplicates_count)) + sys.stdout.flush() + +print "\rFinished. Processed %s messages, removed %s duplicates." % (msg_count, duplicates_count) +if duplicates_count > 0: + print "You might want to run 'notmuch new' now." + +if suspected_duplicates_count > 0: + print + print "Found %s messages with duplicate IDs but different content." % suspected_duplicates_count + print "Perhaps we should ignore more headers." -- 1.7.10.4