Note: this patch does not expose the fix_manifest function when running bundlebuilder as a script. That would be a 2 or 3 line patch, seperately. From 9ab619888a9d4ef40ca3e984d4485c5a22558105 Mon Sep 17 00:00:00 2001 From: chema <chema@chema-laptop.(none)> Date: Fri, 6 Jun 2008 10:01:21 -0600 Subject: [PATCH] Bundlebuilder to use MANIFEST, fix_manifest function, some functions pushed upstream to bundle.py and activitybundle.py ---  src/sugar/activity/bundlebuilder.py |   69 ++++++++++--------  src/sugar/bundle/activitybundle.py  |  140 ++++++++++++++++++++++++++++++++---  src/sugar/bundle/bundle.py          |   23 ++++++  3 files changed, 189 insertions(+), 43 deletions(-) diff --git a/src/sugar/activity/bundlebuilder.py b/src/sugar/activity/bundlebuilder.py index 1063f72..72c246c 100644 --- a/src/sugar/activity/bundlebuilder.py +++ b/src/sugar/activity/bundlebuilder.py @@ -19,28 +19,16 @@ import os  import zipfile  import tarfile  import shutil -import subprocess +from subprocess import Popen, PIPE  import re  import gettext  from optparse import OptionParser +import warnings +import subprocess    from sugar import env -from sugar.bundle.activitybundle import ActivityBundle - -def list_files(base_dir, ignore_dirs=None, ignore_files=None): -    result = [] +from sugar.bundle.activitybundle import ActivityBundle,MANIFEST,list_files   -    for root, dirs, files in os.walk(base_dir): -        for f in files: -            if ignore_files and f not in ignore_files: -                rel_path = root[len(base_dir) + 1:] -                result.append(os.path.join(rel_path, f)) -        if ignore_dirs and root == base_dir: -            for ignore in ignore_dirs: -                if ignore in dirs: -                    dirs.remove(ignore) - -    return result    class Config(object):      def __init__(self, bundle_name, source_dir=None): @@ -53,6 +41,7 @@ class Config(object):          bundle = ActivityBundle(self.source_dir)          version = bundle.get_activity_version()   +        self.bundle = bundle          self.bundle_name = bundle_name          self.xo_name = '%s-%d.xo' % (self.bundle_name, version)          self.tarball_name = '%s-%d.tar.bz2' % (self.bundle_name, version) @@ -103,9 +92,9 @@ class Builder(object):              f.close()    class Packager(object): -    def __init__(self, config): +    def __init__(self, config, dist_dir = None):          self.config = config -        self.dist_dir = os.path.join(self.config.source_dir, 'dist') +        self.dist_dir = dist_dir or os.path.join(self.config.source_dir, 'dist')          self.package_path = None            if not os.path.exists(self.dist_dir): @@ -113,19 +102,28 @@ class Packager(object):                 class BuildPackager(Packager): -    def __init__(self, config): -        Packager.__init__(self, config) +    def __init__(self, config,dist_dir = None): +        Packager.__init__(self, config,dist_dir )          self.build_dir = self.config.source_dir        def get_files(self): -        return list_files(self.build_dir, -                          ignore_dirs=['po', 'dist', '.git'], -                          ignore_files=['.gitignore']) +        return self.config.bundle.get_files() +    +    def fix_manifest(self): +        allfiles =  list_files(self.build_dir, +                          ignore_dirs=['dist', '.git'], +                          ignore_files=['.gitignore', 'MANIFEST', '*.pyc', '*~', '*.bak']) +        for file in allfiles: +            if file not in self.config.bundle.manifest: +                self.config.bundle.manifest.append(file) +        manifestfile = open(os.path.join(self.config.source_dir,MANIFEST),"wb") +        for line in self.config.bundle.manifest: +            manifestfile.write(line + "\n")    class XOPackager(BuildPackager): -    def __init__(self, config): +    def __init__(self, config,dist_dir = None,dist_name = None):          BuildPackager.__init__(self, config) -        self.package_path = os.path.join(self.dist_dir, self.config.xo_name) +        self.package_path = os.path.join(self.dist_dir, dist_name or self.config.xo_name)        def package(self):          bundle_zip = zipfile.ZipFile(self.package_path, 'w', @@ -137,16 +135,25 @@ class XOPackager(BuildPackager):            bundle_zip.close()   -class SourcePackager(Packager): -    def __init__(self, config): -        Packager.__init__(self, config) +class SourcePackager(BuildPackager): +    def __init__(self, config,dist_dir = None): +        BuildPackager.__init__(self, config,dist_dir)          self.package_path = os.path.join(self.dist_dir,                                           self.config.tarball_name)        def get_files(self): -        return list_files(self.config.source_dir, -                          ignore_dirs=['locale', 'dist', '.git'], -                          ignore_files=['.gitignore']) +        git_ls = Popen('git-ls-files',stdout=PIPE,cwd=self.config.source_dir) +        if git_ls.wait(): +            #non-0 return code - failed +            return Packager.get_files(self) +        f = git_ls.stdout +        files = [] +        for line in f.readlines(): +            filename = line.strip() +            if not filename.startswith('.'): +                files.append(filename) +        f.close() +        return files        def package(self):          tar = tarfile.open(self.package_path, "w") diff --git a/src/sugar/bundle/activitybundle.py b/src/sugar/bundle/activitybundle.py index db30555..1ff7d39 100644 --- a/src/sugar/bundle/activitybundle.py +++ b/src/sugar/bundle/activitybundle.py @@ -21,15 +21,31 @@ from ConfigParser import ConfigParser  import locale  import os  import tempfile +from fnmatch import fnmatch    from sugar.bundle.bundle import Bundle, MalformedBundleException, \      AlreadyInstalledException, RegistrationException, \      NotInstalledException   -from sugar import activity -from sugar import env -  import logging +import warnings + +MANIFEST = "MANIFEST" + +def list_files(base_dir, ignore_dirs=None, ignore_files=None): +    result = [] +    +    for root, dirs, files in os.walk(base_dir): +        for f in files: +            if not (ignore_files and [True for pat in ignore_files if fnmatch(f,pat)]): +                #result matches a pattern in ignore_files, ignore it +                rel_path = root[len(base_dir) + 1:] +                result.append(os.path.join(rel_path, f)) +        if ignore_dirs and root == base_dir: +            for ignore in ignore_dirs: +                if ignore in dirs: +                    dirs.remove(ignore) +    return result    class ActivityBundle(Bundle):      """A Sugar activity bundle @@ -64,7 +80,62 @@ class ActivityBundle(Bundle):          linfo_file = self._get_linfo_file()          if linfo_file:              self._parse_linfo(linfo_file) - +        +        self.read_manifest() + +    def _raw_manifest(self): +        try: +            f = self._get_file(MANIFEST) +        except IOError: +            f = none +        if not f: +            warnings.warn(MalformedBundleException("Activity directory lacks a MANIFEST file.")) +            return [] +        ret = map(lambda x:x.strip(),f.readlines()) #strip trailing \n and other whitespace +        f.close() +        return ret +        +    def read_manifest(self): +        """read_manifest: sets self.manifest to list of lines in MANIFEST, +        with invalid lines replaced by empty lines. +        +        Since absolute order carries information on file history, it should be preserved. +        For instance, when renaming a file, you should leave the new name on the same line +        as the old one. +        """ +        manifestlines = self._raw_manifest() +        for num,line in enumerate(manifestlines): +            if line: +                if line in manifestlines[0:num]: +                    manifestlines[num] = "" +                    warnings.warn(MalformedBundleException("Bundle %s: duplicate entry in MANIFEST: %s" +                                                  %(self._name,line))) +                    continue +                if line == MANIFEST: +                    manifestlines[num] = "" +                    warnings.warn(MalformedBundleException("Bundle %s: MANIFEST includes itself: %s" +                                                  %(self._name,line))) +                if line.endswith("/"): +                    if not self._is_dir(line): +                        manifestlines[num] = "" +                        warnings.warn(MalformedBundleException("Bundle %s: invalid dir entry in MANIFEST: %s" +                                                      %(self._name,line))) +                else: +                    if not self._is_file(line): +                        manifestlines[num] = "" +                        warnings.warn(MalformedBundleException("Bundle %s: invalid entry in MANIFEST: %s" +                                                      %(self._name,line))) +        #remove trailing newlines - unlike internal newlines, they do not help keep absolute position +        while manifestlines[-1]=="": +            manifestlines = manifestlines[:-1] +        self.manifest = manifestlines +    +    def get_files(self,manifest = None): +        return [MANIFEST] + filter(lambda line: line and not line.endswith("/"),manifest or self.manifest) +    +    def get_dirs(self): +        return filter(lambda line: line and line.endswith("/"),self.manifest) +           def _parse_info(self, info_file):          cp = ConfigParser()          cp.readfp(info_file) @@ -116,6 +187,7 @@ class ActivityBundle(Bundle):                  raise MalformedBundleException(                      'Activity bundle %s has invalid version number %s' %                      (self._path, version)) +           def _get_linfo_file(self):          lang = locale.getdefaultlocale()[0] @@ -209,28 +281,53 @@ class ActivityBundle(Bundle):          return self._show_launcher        def is_installed(self): +        from sugar import activity +                 if activity.get_registry().get_activity(self._bundle_id):              return True          else:              return False        def need_upgrade(self): +        from sugar import activity +                 act = activity.get_registry().get_activity(self._bundle_id)          if act is None or act.version != self._activity_version:              return True          else:              return False - -    def install(self): -        activities_path = env.get_user_activities_path() -        act = activity.get_registry().get_activity(self._bundle_id) -        if act is not None and act.path.startswith(activities_path): -            raise AlreadyInstalledException - -        install_dir = env.get_user_activities_path() +    +    def unpack(self,install_dir,strict_manifest=False):          self._unzip(install_dir)            install_path = os.path.join(install_dir, self._zip_root_dir) +        +        #check installed files against the MANIFEST +        manifestfiles = self.get_files(self._raw_manifest()) +        for file in list_files(install_path): +            if file in manifestfiles: +                manifestfiles.remove(file) +            elif file != MANIFEST: +                #warnings.warn(MalformedBundleException("Bundle %s: %s not in MANIFEST"% +                #                                       (self._name,file))) +                if strict_manifest: +                    os.remove(os.path.join(install_path,file)) +        if manifestfiles: +            err = MalformedBundleException("Bundle %s: files in MANIFEST not included: %s"% +                                                       (self._name,str(manifestfiles))) +            if strict_manifest: +                raise err +            else: +                warnings.warn(err) +        +        #create empty directories +        for dir in self.get_dirs(): +            dirpath = os.path.join(install_path,dir) +            if os.path.isdir(dirpath): +                warnings.warn(MalformedBundleException("Bunldle %s: non-empty dir %s in MANIFEST"% +                                                       (self._name,dirpath))) +            else: +                os.makedirs(os.path.join(install_path,dir))            xdg_data_home = os.getenv('XDG_DATA_HOME',                                    os.path.expanduser('~/.local/share')) @@ -267,11 +364,27 @@ class ActivityBundle(Bundle):                      os.symlink(info_file,                                 os.path.join(installed_icons_dir,                                              os.path.basename(info_file))) +        return install_path   +    def install(self): +        from sugar import activity +        from sugar import env +        +        activities_path = env.get_user_activities_path() +        act = activity.get_registry().get_activity(self._bundle_id) +        if act is not None and act.path.startswith(activities_path): +            raise AlreadyInstalledException + +        install_dir = env.get_user_activities_path() +        install_path = self.unpack(install_dir) +                 if not activity.get_registry().add_bundle(install_path):              raise RegistrationException        def uninstall(self, force=False): +        from sugar import activity +        from sugar import env +                 if self._unpacked:              install_path = self._path          else: @@ -316,6 +429,9 @@ class ActivityBundle(Bundle):              raise RegistrationException        def upgrade(self): +        from sugar import activity +        from sugar import env +                 act = activity.get_registry().get_activity(self._bundle_id)          if act is None:              logging.warning('Activity not installed') diff --git a/src/sugar/bundle/bundle.py b/src/sugar/bundle/bundle.py index 47d08b2..44e30b0 100644 --- a/src/sugar/bundle/bundle.py +++ b/src/sugar/bundle/bundle.py @@ -114,6 +114,29 @@ class Bundle:              zip_file.close()            return f +    +    def _is_dir(self, filename): +        if self._unpacked: +            path = os.path.join(self._path, filename) +            return os.path.isdir(path) +        else: +            return True #zip files contain all dirs you care about! +            +    def _is_file(self, filename): +        if self._unpacked: +            path = os.path.join(self._path, filename) +            return os.path.isfile(path) +        else: +            zip_file = zipfile.ZipFile(self._path) +            path = os.path.join(self._zip_root_dir, filename) +            try: +                data = zip_file.getinfo(path) +                return True +            except KeyError: +                return False +            finally: +                zip_file.close() +                       def get_path(self):          """Get the bundle path.""" -- <a href="http://1.5.2.5">1.5.2.5</a>