[PATCH] introducing xattrMaildir: a Maildir repository where message files have all IMAP flags in an extended attribute
Erik Quaeghebeur
offlineimap at equaeghe.nospammail.net
Sun Mar 10 20:48:25 UTC 2013
Dear maintainers & other interested subscribers,
To be applied on top of the previous patch I sent to this mailing list,
this patch adds a new repository type, xattrMaildir, a Maildir
repository where message files have all IMAP flags in an extended
attribute 'user.org.offlineimap.flags'. (It is perfectly backwards
compatible with the Maildir specification.)
As before I have done a (successful) test that is minimal: for one
message, I synced from an IMAP server to an xattrMaildir repository and
verified (using the getfattr command-line tool) that all IMAP flags were
indeed synced.
The code uses the xattr module (http://pyxattr.k1024.org/module.html) as
an extra dependency. Actually, very little has changed:
* in folder/__init__.py and repository/__init__.py only boilerplate
needed to be added
* the new repository/xattrMaildir.py is also essentially boilerplate;
it is identical to repository/Maildir.py except for a replacement of
reference to folder.Maildir.Maildir to folder.xattrMaildir.xattrMaildir
* the new folder/xattrMaildir.py modifies three methods from
folder/Maildir.py, and in a minimal fashion:
- in quickchanged the check whether the flags have been changed as
reverted to an equality check (back from the subset check introduced in
my previous patch)
- _scanfolder and savemessageflags we actually read and write the
extended attributes (look for 'xattr.get' and 'xattr.set'), but these
are essentially one-line changes/additions.
Based on the realization that the changes needed are so small, it may be
considered that no new repository is introduced, but that the
functionality is activated by an option that controls the then three
necessary conditionals.
Again, your testing and critique is very much welcome. The code is a
proof-of-concept and therefore very rough.
Best,
Erik
P.S.: As with my previous patch, I've also added it in attachment to
deal with copy-paste introduced linebreaks.
P.P.S: Why '\\Seen' instead of '\Seen'?
---
offlineimap/folder/__init__.py | 2 +-
offlineimap/folder/xattrMaildir.py | 117
+++++++++++++++++++++++++++++++++
offlineimap/repository/__init__.py | 4 +-
offlineimap/repository/xattrMaildir.py | 63 ++++++++++++++++++
4 files changed, 184 insertions(+), 2 deletions(-)
create mode 100644 offlineimap/folder/xattrMaildir.py
create mode 100644 offlineimap/repository/xattrMaildir.py
diff --git a/offlineimap/folder/__init__.py b/offlineimap/folder/__init__.py
index 2b54a71..39f4c89 100644
--- a/offlineimap/folder/__init__.py
+++ b/offlineimap/folder/__init__.py
@@ -1,2 +1,2 @@
-from . import Base, Gmail, IMAP, Maildir, LocalStatus
+from . import Base, Gmail, IMAP, Maildir, xattrMaildir, LocalStatus
diff --git a/offlineimap/folder/xattrMaildir.py
b/offlineimap/folder/xattrMaildir.py
new file mode 100644
index 0000000..17fc8b4
--- /dev/null
+++ b/offlineimap/folder/xattrMaildir.py
@@ -0,0 +1,117 @@
+# xattrMaildir folder support
+
+from offlineimap.folder.Maildir import re_uidmatch, MaildirFolder
+
+import os
+import xattr
+from offlineimap import imaputil
+
+try: # python 2.6 has set() built in
+ set
+except NameError:
+ from sets import Set as set
+
+from offlineimap import OfflineImapError
+
+class xattrMaildirFolder(MaildirFolder):
+ def _scanfolder(self):
+ """Cache the message list from a Maildir.
+
+ Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F
+ (flagged).
+ :returns: dict that can be used as self.messagelist"""
+ maxage = self.config.getdefaultint("Account " + self.accountname,
+ "maxage", None)
+ maxsize = self.config.getdefaultint("Account " + self.accountname,
+ "maxsize", None)
+ retval = {}
+ files = []
+ nouidcounter = -1 # Messages without UIDs get negative
UIDs.
+ for dirannex in ['new', 'cur']:
+ fulldirname = os.path.join(self.getfullname(), dirannex)
+ files.extend((dirannex, filename) for
+ filename in os.listdir(fulldirname))
+
+ for dirannex, filename in files:
+ # We store just dirannex and filename, ie 'cur/123...'
+ filepath = os.path.join(dirannex, filename)
+ # check maxage/maxsize if this message should be considered
+ if maxage and not self._iswithinmaxage(filename, maxage):
+ continue
+ if maxsize and (os.path.getsize(os.path.join(
+ self.getfullname(), filepath)) > maxsize):
+ continue
+
+ (prefix, uid, fmd5, maildirflags) =
self._parse_filename(filename)
+ if uid is None: # assign negative uid to upload it.
+ uid = nouidcounter
+ nouidcounter -= 1
+ else: # It comes from our folder.
+ uidmatch = re_uidmatch.search(filename)
+ uid = None
+ if not uidmatch:
+ uid = nouidcounter
+ nouidcounter -= 1
+ else:
+ uid = long(uidmatch.group(1))
+ # 'filename' is 'dirannex/filename', e.g.
cur/123,U=1,FMD5=1:2,S
+ retval[uid] = {'flags':
set((xattr.get(os.path.join(self.getfullname(), newfilename),
+ 'org.offlineimap.flags',
+
namespace=xattr.NS_USER)).split()),
+ 'filename': filepath}
+ return retval
+
+ def quickchanged(self, statusfolder):
+ """Returns True if the Maildir has changed"""
+ self.cachemessagelist()
+ # Folder has different uids than statusfolder => TRUE
+ if sorted(self.getmessageuidlist()) != \
+ sorted(statusfolder.getmessageuidlist()):
+ return True
+ # Also check for flag changes, it's quick on a Maildir
+ for (uid, message) in self.getmessagelist().iteritems():
+ if message['flags'] != statusfolder.getmessageflags(uid):
+ return True
+ return False #Nope, nothing changed
+
+ def savemessageflags(self, uid, flags):
+ """Sets the specified message's flags to the given set.
+
+ This function moves the message to the cur or new subdir,
+ depending on the 'S'een flag.
+
+ Note that this function does not check against dryrun settings,
+ so you need to ensure that it is never called in a
+ dryrun mode."""
+ oldfilename = self.messagelist[uid]['filename']
+ dir_prefix, filename = os.path.split(oldfilename)
+ # If a message has been seen, it goes into 'cur'
+ dir_prefix = 'cur' if '\\Seen' in flags else 'new'
+
+ if flags != self.messagelist[uid]['flags']:
+ # Flags have actually changed, construct new filename Strip
+ # off existing infostring (possibly discarding small letter
+ # flags that dovecot uses TODO)
+ infomatch = self.re_flagmatch.search(filename)
+ if infomatch:
+ filename = filename[:-len(infomatch.group())] #strip off
+ infostr = '%s2,%s' % (self.infosep,
+
''.join(sorted(imaputil.flagsimap2maildir(flags))))
+ filename += infostr
+
+ newfilename = os.path.join(dir_prefix, filename)
+ if (newfilename != oldfilename):
+ try:
+ os.rename(os.path.join(self.getfullname(), oldfilename),
+ os.path.join(self.getfullname(), newfilename))
+ except OSError as e:
+ raise OfflineImapError("Can't rename file '%s' to '%s':
%s" % (
+ oldfilename, newfilename, e[1]),
+ OfflineImapError.ERROR.FOLDER)
+
+ self.messagelist[uid]['flags'] = flags
+ self.messagelist[uid]['filename'] = newfilename
+
+ xattr.set(os.path.join(self.getfullname(), newfilename),
+ 'org.offlineimap.flags', ' '.join(flags),
+ namespace=xattr.NS_USER)
diff --git a/offlineimap/repository/__init__.py
b/offlineimap/repository/__init__.py
index 22cd128..f9b7b1b 100644
--- a/offlineimap/repository/__init__.py
+++ b/offlineimap/repository/__init__.py
@@ -23,6 +23,7 @@ except ImportError: #python2
from offlineimap.repository.IMAP import IMAPRepository,
MappedIMAPRepository
from offlineimap.repository.Gmail import GmailRepository
from offlineimap.repository.Maildir import MaildirRepository
+from offlineimap.repository.xattrMaildir import xattrMaildirRepository
from offlineimap.repository.LocalStatus import LocalStatusRepository
from offlineimap.error import OfflineImapError
@@ -46,7 +47,8 @@ class Repository(object):
elif reqtype == 'local':
name = account.getconf('localrepository')
typemap = {'IMAP': MappedIMAPRepository,
- 'Maildir': MaildirRepository}
+ 'Maildir': MaildirRepository,
+ 'xattrMaildir': xattrMaildirRepository}
elif reqtype == 'status':
# create and return a LocalStatusRepository
diff --git a/offlineimap/repository/xattrMaildir.py
b/offlineimap/repository/xattrMaildir.py
new file mode 100644
index 0000000..4a95380
--- /dev/null
+++ b/offlineimap/repository/xattrMaildir.py
@@ -0,0 +1,63 @@
+# xattrMaildir repository support
+
+from offlineimap import folder
+from offlineimap.repository.Maildir import MaildirRepository
+import os
+
+class xattrMaildirRepository(MaildirRepository):
+ def _getfolders_scandir(self, root, extension = None):
+ """Recursively scan folder 'root'; return a list of MailDirFolder
+
+ :param root: (absolute) path to Maildir root
+ :param extension: (relative) subfolder to examine within root"""
+ self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension
= %s" \
+ % (root, extension))
+ retval = []
+
+ # Configure the full path to this repository -- "toppath"
+ if extension:
+ toppath = os.path.join(root, extension)
+ else:
+ toppath = root
+ self.debug(" toppath = %s" % toppath)
+
+ # Iterate over directories in top & top itself.
+ for dirname in os.listdir(toppath) + ['']:
+ self.debug(" dirname = %s" % dirname)
+ if dirname == '' and extension is not None:
+ self.debug(' skip this entry (already scanned)')
+ continue
+ if dirname in ['cur', 'new', 'tmp']:
+ self.debug(" skip this entry (Maildir special)")
+ # Bypass special files.
+ continue
+ fullname = os.path.join(toppath, dirname)
+ if not os.path.isdir(fullname):
+ self.debug(" skip this entry (not a directory)")
+ # Not a directory -- not a folder.
+ continue
+ if extension:
+ # extension can be None which fails.
+ foldername = os.path.join(extension, dirname)
+ else:
+ foldername = dirname
+
+ if (os.path.isdir(os.path.join(fullname, 'cur')) and
+ os.path.isdir(os.path.join(fullname, 'new')) and
+ os.path.isdir(os.path.join(fullname, 'tmp'))):
+ # This directory has maildir stuff -- process
+ self.debug(" This is maildir folder '%s'." % foldername)
+ if self.getconfboolean('restoreatime', False):
+ self._append_folder_atimes(foldername)
+
retval.append(folder.xattrMaildir.xattrMaildirFolder(self.root,
+
foldername,
+
self.getsep(),
+ self))
+
+ if self.getsep() == '/' and dirname != '':
+ # Recursively check sub-directories for folders too.
+ retval.extend(self._getfolders_scandir(root, foldername))
+ self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \
+ repr([x.getname() for x in retval]))
+ return retval
+
--
1.8.1.5
-------------- next part --------------
A non-text attachment was scrubbed...
Name: introducing-xattrMaildir-a-Maildir-repository-where-.patch
Type: text/x-patch
Size: 10763 bytes
Desc: not available
URL: <http://lists.alioth.debian.org/pipermail/offlineimap-project/attachments/20130310/ac2c48d0/attachment.bin>
More information about the OfflineIMAP-project
mailing list