[Sugar-devel] [PATCH] Add support for JEBs containing multiple entries

[Sascha Silbe]

By allowing Journal Entry Bundles (JEBs) to carry more than one entry,
we can use them to back up an entire data store.

Signed-off-by: Sascha Silbe <sascha-pgp at silbe.org>

---
 src/jarabe/journal/journalentrybundle.py |  139 ++++++++++++++++++++----------
 src/jarabe/model/bundleregistry.py       |    6 +-
 2 files changed, 97 insertions(+), 48 deletions(-)

Since even after repeating asking I couldn't get hold of any existing JEB,
I could only test against the JEBs created by my backup implementation.


diff --git a/src/jarabe/journal/journalentrybundle.py b/src/jarabe/journal/journalentrybundle.py
index 41777c7..a3735ca 100644
--- a/src/jarabe/journal/journalentrybundle.py
+++ b/src/jarabe/journal/journalentrybundle.py
@@ -14,24 +14,30 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
+import logging
 import os
 import tempfile
-import shutil
 
-import simplejson
 import dbus
 
-from sugar.bundle.bundle import Bundle, MalformedBundleException
+try:
+    import json
+except ImportError:
+    import simplejson as json
 
+from sugar.bundle.bundle import Bundle, MalformedBundleException
 from jarabe.journal import model
 
+
 class JournalEntryBundle(Bundle):
     """A Journal entry bundle
 
     See http://wiki.laptop.org/go/Journal_entry_bundles for details
     """
+    # TODO: migrate to SL wiki
 
     MIME_TYPE = 'application/vnd.olpc-journal-entry'
+    _METADATA_JSON_NAME = '_metadata.json'
 
     _zipped_extension = '.xoj'
     _unzipped_extension = None
@@ -40,55 +46,98 @@ class JournalEntryBundle(Bundle):
     def __init__(self, path):
         Bundle.__init__(self, path)
 
-    def install(self, uid=''):
-        if os.environ.has_key('SUGAR_ACTIVITY_ROOT'):
-            install_dir = os.path.join(os.environ['SUGAR_ACTIVITY_ROOT'],
-                                       'data')
-        else:
-            install_dir = tempfile.gettempdir()
-        bundle_dir = os.path.join(install_dir, self._zip_root_dir)
-        temp_uid = self._zip_root_dir
-        self._unzip(install_dir)
-        try:
-            metadata = self._read_metadata(bundle_dir)
-            metadata['uid'] = uid
-
-            preview = self._read_preview(temp_uid, bundle_dir)
-            if preview is not None:
-                metadata['preview'] = dbus.ByteArray(preview)
-
-            file_path = os.path.join(bundle_dir, temp_uid)
-            model.write(metadata, file_path)
-        finally:
-            shutil.rmtree(bundle_dir, ignore_errors=True)
+    def _check_zip_bundle(self):
+        # potentially expensive, but avoids trouble during unpacking
+        if self._zip_file.testzip() is not None:
+            raise MalformedBundleException('Corrupt zip file')
+
+        file_names = self._zip_file.namelist()
+        if len(file_names) == 0:
+            raise MalformedBundleException('Empty zip file')
+
+        metadata_seen = False
+        for name in file_names:
+            for part in name.split('/'):
+                if part.startswith('.'):
+                    raise MalformedBundleException(
+                        'Path component starts with dot: %r', name)
+
+            if name.split('/')[-1] == self._METADATA_JSON_NAME:
+                metadata_seen = True
+
+        if not metadata_seen:
+            raise MalformedBundleException('No metadata file found')
+
+    def install(self):
+        for object_id, file_paths in self._get_directories().items():
+            if len(object_id) < 36:
+                logging.warning('Ignoring unknown directory %r', object_id)
+                continue
+
+            if self._METADATA_JSON_NAME not in file_paths:
+                logging.warning('Ignoring directory %r without %s',
+                    object_id, self._METADATA_JSON_NAME)
+                continue
+
+            try:
+                self._install_entry(object_id, file_paths)
+            except Exception:
+                logging.exception('Error installing Journal entry %r:',
+                    object_id)
+
+    def _install_entry(self, object_id, file_paths):
+        file_paths.remove(self._METADATA_JSON_NAME)
+        metadata = self._read_metadata(object_id)
+
+        data_file_name = ''
+        if object_id in file_paths:
+            file_paths.remove(object_id)
+            data_file_name = self._read_data(object_id)
+
+        for path in file_paths:
+            components = path.split('/')
+            if len(components) != 2 or components[1] != object_id:
+                logging.warning('Ignoring unknown file %r', path)
+
+            name = components[0]
+            value = self._zip_file.read(os.path.join(object_id, path))
+            metadata[name] = dbus.ByteArray(value)
+
+        model.write(metadata, data_file_name, transfer_ownership=True)
 
     def get_bundle_id(self):
         return None
 
-    def _read_metadata(self, bundle_dir):
-        metadata_path = os.path.join(bundle_dir, '_metadata.json')
-        if not os.path.exists(metadata_path):
-            raise MalformedBundleException(
-                    'Bundle must contain the file "_metadata.json"')
-        f = open(metadata_path, 'r')
+    def _read_data(self, object_id):
+        data_fd, data_file_name = tempfile.mkstemp(prefix='JEB')
+        data_file = os.fdopen(data_fd, 'w')
         try:
-            json_data = f.read()
+            # TODO: handle large files better
+            # TODO: predict disk-full
+            data_file.write(self._zip_file.read(
+                os.path.join(object_id, object_id)))
+            return data_file_name
         finally:
-            f.close()
-        return simplejson.loads(json_data)
-
-    def _read_preview(self, uid, bundle_dir):
-        preview_path = os.path.join(bundle_dir, 'preview', uid)
-        if not os.path.exists(preview_path):
-            return ''
-        f = open(preview_path, 'r')
-        try:
-            preview_data = f.read()
-        finally:
-            f.close()
-        return preview_data
+            data_file.close()
+
+    def _read_metadata(self, object_id):
+        metadata_path = os.path.join(object_id, self._METADATA_JSON_NAME)
+        json_data = self._zip_file.read(metadata_path)
+        return json.loads(json_data)
+
+    def _get_directories(self):
+        """Return the names of all top-level directories and their contents.
+        """
+        contents = {}
+        for path in self._zip_file.namelist():
+            if path.endswith('/'):
+                continue
+
+            directory, file_name = path.lstrip('/').split('/', 1)
+            contents.setdefault(directory, []).append(file_name)
+
+        return contents
 
     def is_installed(self):
         # These bundles can be reinstalled as many times as desired.
         return False
-
diff --git a/src/jarabe/model/bundleregistry.py b/src/jarabe/model/bundleregistry.py
index 86a2738..f4c7275 100644
--- a/src/jarabe/model/bundleregistry.py
+++ b/src/jarabe/model/bundleregistry.py
@@ -164,7 +164,7 @@ class BundleRegistry(gobject.GObject):
             if bundle.get_bundle_id() == bundle_id:
                 return bundle
         return None
-    
+
     def __iter__(self):
         return self._bundles.__iter__()
 
@@ -364,7 +364,7 @@ class BundleRegistry(gobject.GObject):
 
         install_dir = env.get_user_activities_path()
         if isinstance(bundle, JournalEntryBundle):
-            install_path = bundle.install(uid)
+            install_path = bundle.install()
         else:
             install_path = bundle.install(install_dir)
 
@@ -376,7 +376,7 @@ class BundleRegistry(gobject.GObject):
         elif not self.add_bundle(install_path):
             raise RegistrationException
 
-    def uninstall(self, bundle, force=False):        
+    def uninstall(self, bundle, force=False):
         # TODO treat ContentBundle in special way
         # needs rethinking while fixing ContentBundle support
         if isinstance(bundle, ContentBundle) or \
-- 
tg: (6b92d37..) t/jeb-multi (depends on: upstream/master)