The full script is available at <a href="https://bugzilla.redhat.com/show_bug.cgi?id=448030#c41">https://bugzilla.redhat.com/show_bug.cgi?id=448030#c41</a> [Patch 2/5] Author: Frederick Grose <<a href="mailto:fgrose@sugarlabs.org">fgrose@sugarlabs.org</a>> Date: Thu Apr 14 09:48:41 2011 -0400 Support refreshing of attached LiveOS devices. Provide a new feature to refresh an attached LiveOS source image with a renewed rootfs and freshed overlay and, optionally, skip building a new .iso file. Updated base_on() to support selinux contexts, provide a liveos_mount() function, new functions to check disc space for building and refreshing (with an extra space option to test or adjust staging space). Provide an option to bypass the break to shell or script to permit uninterrupted processing. Include the kernel architecture of the source in the .iso image name, support F15-style rd.luks=0 in the boot configuration adjustment, and provide updated usage statements. diff --git a/tools/edit-livecd b/tools/edit-livecd index 659cfae..e156895 100755 --- a/tools/edit-livecd +++ b/tools/edit-livecd @@ -69,8 +69,24 @@ class LiveImageEditor(LiveImageCreator): self.clone = False """Signals when to copy a running LiveOS image as base.""" - self._include = None - """A string of file or directory paths to include in __copy_img_root.""" + self.refresh_only = False + """Signals that a filesystem refresh only is requested. No iso9660 + installation image will be produced. If the source has never been + configured, this value is set to None.""" + + self._overlay = None + """Signals the existence of an filesystem overlay file in base_on.""" + + self._include = [] + """A string of file or directory paths in the outer device filesystem + to include in __copy_src_root().""" + + self._exclude = [] + """A string of file or directory paths in the outer device filesystem + to exclude from __copy_src_root().""" + + self._releasefile = [] + """A string of file or directory paths to include in _brand().""" self._builder = os.getlogin() """The name of the Remix builder for _branding. @@ -82,10 +98,10 @@ class LiveImageEditor(LiveImageCreator): current compression or lack thereof. Compression type options vary with the version of the kernel and SquashFS used.""" - self.skip_compression = False + self.skip_compression = None """Controls whether to use squashfs to compress the image.""" - self.skip_minimize = False + self.skip_minimize = None """Controls whether an image minimizing snapshot should be created.""" self._isofstype = "iso9660" @@ -110,6 +126,15 @@ class LiveImageEditor(LiveImageCreator): self._LiveImageCreatorBase__isodir = None """directory where the iso is staged""" + self.squashmnt = None + """Mount object for the source LiveOS SquashFS.""" + + self.liveosmnt = None + """Mount object for the source LiveOS image.""" + + self.extra_loops = [] + """List of extra loop devices created for block device sources.""" + # properties def __get_image(self): if self._LoopImageCreator__imagedir is None: @@ -166,6 +191,7 @@ class LiveImageEditor(LiveImageCreator): finally: os.unlink(path) + def mount(self, base_on, cachedir = None): """mount existing file system. @@ -176,7 +202,9 @@ class LiveImageEditor(LiveImageCreator): We also need to get some info about the image before we can mount it. base_on -- the <LIVEIMG.src> a LiveOS.iso file or an attached LiveOS - device, such as, /dev/live for a currently running image. + device, such as, /dev/live for a currently running image, + or <base_on>/mnt/live for a live-mounted (overlay-mounted) + LiveOS device. cachedir -- a directory in which to store a Yum cache; Not used in edit-liveos. @@ -196,13 +224,56 @@ class LiveImageEditor(LiveImageCreator): makedirs(self._LoopImageCreator__imagedir) makedirs(self._ImageCreator_outdir) - if self.clone: + self._LiveImageCreatorBase__isodir = os.path.join( + self._ImageCreator__builddir, 'iso') + + if self._src_type == 'blk': + srcmnt = DiskMount(RawDisk(None, base_on), self._mkdtemp('blk-')) + elif self._src_type == 'iso': + srcmnt = DiskMount(LoopbackDisk(base_on, None), + self._mkdtemp('iso-')) + elif self._src_type == 'live': + srcmnt = Mount(os.path.join('/mnt', 'live')) + + self.srcmnt = srcmnt + + try: + srcmnt.mount() + except MountError, e: + raise CreatorError("Failed to mount '%s' : %s" % (base_on, e)) + else: + srcmntdir = srcmnt.mountdir + liveosdir = os.path.join(srcmntdir, 'LiveOS') + if not os.path.exists(liveosdir): + raise CreatorError("No LiveOS directory on %s" % base_on) + + files = os.listdir(liveosdir) + for f in files: + if f.find('overlay-') == 0: + self._overlay = f + break + if os.path.exists(os.path.join(liveosdir, 'ext3fs.img')): + self.squashmnt = False + + if self._space_to_proceed(srcmntdir, base_on): + self.__copy_src_root(srcmntdir, base_on) + else: + raise CreatorError('''Insufficient space to stage a new build. + Exiting...''') + finally: + srcmnt.cleanup() + + if self.clone and self._overlay: # Need to clone base_on into ext3fs.img at this point self._LoopImageCreator__fslabel = <a href="http://self.name">self.name</a> self._base_on(base_on) else: + self.liveosmnt.cleanup() + if self.squashmnt.mounted: + self.squashmnt.cleanup() LiveImageCreator._base_on(self, base_on) - self._LoopImageCreator__fstype = get_fsvalue(self._image, 'TYPE') + self._LoopImageCreator__fstype = get_fsvalue('value', 'TYPE', None, + self._image) self._get_fslabel() self.fslabel = self._LoopImageCreator__fslabel @@ -210,140 +281,377 @@ class LiveImageEditor(LiveImageCreator): self._LoopImageCreator__instloop = ExtDiskMount( ExistingSparseLoopbackDisk(self._image, - self._LoopImageCreator__image_size), + self._LoopImageCreator__image_size), self._ImageCreator_instroot, self._fstype, self._LoopImageCreator__blocksize, self.fslabel, - self.tmpdir) + tmpdir=self.tmpdir) try: self._LoopImageCreator__instloop.mount() except MountError, e: raise CreatorError("Failed to loopback mount '%s' : %s" % (self._image, e)) - cachesrc = cachedir or (self._ImageCreator__builddir + "/yum-cache") - makedirs(cachesrc) + if self._script or self._shell: + if self._src_type == 'iso': + cachesrc = cachedir or (self._ImageCreator__builddir + "/yum-cache") + makedirs(cachesrc) - for (f, dest) in [("/sys", None), ("/proc", None), - ("/dev/pts", None), ("/dev/shm", None), - (cachesrc, "/var/cache/yum")]: - self._ImageCreator__bindmounts.append(BindChrootMount(f, self._instroot, dest)) + for (f, dest) in [("/sys", None), ("/proc", None), + ("/dev/pts", None), ("/dev/shm", None), + (cachesrc, "/var/cache/yum")]: + self._ImageCreator__bindmounts.append(BindChrootMount( + f, self._instroot, dest)) - self._do_bindmounts() + self._do_bindmounts() - os.symlink("../proc/mounts", self._instroot + "/etc/mtab") + os.symlink("/proc/self/mounts", self._instroot + "/etc/mtab") - self.__copy_img_root(base_on) - self._brand(self._builder) - def _base_on(self, base_on): - """Clone the running LiveOS image as the basis for the new image.""" + def liveos_mount(self, srcmntdir, base_on, ops=None): + """Mount the LiveOS image in the iso, or installed device.""" - self.__fstype = 'ext4' - self.__image_size = 4096L * 1024 * 1024 - self.__blocksize = 4096 + if self.liveosmnt and self.liveosmnt.mounted: + return self.liveosmnt + + liveosdir = os.path.join(srcmntdir, 'LiveOS') + squash_img = os.path.join(liveosdir, 'squashfs.img') + rootfs_img = os.path.join(liveosdir, 'ext3fs.img') + + if self._src_type == 'live': + return + + self.extra_loops.extend([add_loop(8)]) + self.extra_loops.extend([add_loop(9)]) + self.extra_loops.extend([add_loop(10)]) + if self._src_type == 'blk' and self._overlay: + if not os.path.exists(rootfs_img): + self.liveosmnt = LiveImageMount(RawDisk(None, base_on), + self._mkdtemp('live-'), self._overlay, + self._ImageCreator__builddir) + elif os.path.exists(squash_img): + self.squashmnt = DiskMount(LoopbackDisk(squash_img, None), + self._mkdtemp('squash-'), ops=ops) + rootfs_img = os.path.join(self.squashmnt.mountdir, + 'LiveOS', 'ext3fs.img') + try: + self.squashmnt.mount() + except MountError, e: + raise CreatorError("Failed to mount '%s' : %s" % + (squash_img, e)) + + if os.path.exists(rootfs_img): + self.liveosmnt = DiskMount(LoopbackDisk(rootfs_img, None), + self._mkdtemp('ext3-'), ops=ops) + elif not self.liveosmnt: + raise CreatorError("No LiveOS image found on '%s', Exiting..." % + base_on) - self.__instloop = ExtDiskMount(SparseLoopbackDisk(self._image, - self.__image_size), - self._instroot, - self.__fstype, - self.__blocksize, - self.fslabel, - self.tmpdir) try: - self.__instloop.mount() + self.liveosmnt.mount() except MountError, e: - raise CreatorError("Failed to loopback mount '%s' : %s" % - (self._image, e)) + raise CreatorError("Failed to mount '%s' : %s" % + (rootfs_img, e)) + else: + if self.squashmnt is None: + self.squashmnt = self.liveosmnt.squashmnt - subprocess.call(['rsync', '-ptgorlHASx', '--specials', '--progress', - '--include', '/*/', - '--exclude', '/etc/mtab', - '--exclude', '/etc/blkid/*', - '--exclude', '/dev/*', - '--exclude', '/proc/*', - '--exclude', '/home/*', - '--exclude', '/media/*', - '--exclude', '/mnt/live', - '--exclude', '/sys/*', - '--exclude', '/tmp/*', - '--exclude', '/.liveimg*', - '--exclude', '/.autofsck', - '/', self._instroot]) - subprocess.call(['sync']) + return self.liveosmnt - self._ImageCreator__create_minimal_dev() - self.__instloop.cleanup() + def _space_to_proceed(self, srcmntdir, base_on): + """Provide estimated size requirements and availability of staging + resources for the build. + """ - def __copy_img_root(self, base_on): - """helper function to copy root content of the base LiveIMG to - ISOdir""" + # Determine compression and minimization states + liveosdir = os.path.join(srcmntdir, 'LiveOS') + squash_img = os.path.join(liveosdir, 'squashfs.img') + img = squash_img + if not os.path.exists(squash_img): + img = os.path.join(liveosdir, 'osmin.img') + if not os.path.exists(img): + if self.skip_minimize is None: + self.skip_minimize = True + if self.skip_compression is None: + # default to compressed in all cases + self.skip_compression = False + if self.skip_compression is None: + self.skip_compression = False + # 'self.compress_type = None' will force reading it from + # base_on. + if self.compress_type is None: + self.compress_type = squashfs_compression_type(img) + if self.compress_type == 'undetermined': + # 'gzip' for compatibility with older versions. + self.compress_type = 'gzip' + + du_args = ['/usr/bin/du', '-csxb', '--files0-from=-'] ignore_list = ['ext3fs.img', 'squashfs.img', 'osmin.img', 'home.img', 'overlay-*'] if self.clone: ignore_list.remove('home.img') - includes = 'boot, /EFI, /syslinux, /LiveOS' - if self._include: - includes += ", " + self._include - - imgmnt = DiskMount(RawDisk(0, base_on), self._mkdtemp()) + if os.path.exists(os.path.join(liveosdir, 'home.img')): + du_args.extend(['--exclude=/home/*']) + for fn in ignore_list: + du_args.extend(['--exclude=%s' % fn]) + + # remove duplicate files to reduce iso size + efi_vmlinuz = os.path.join(srcmntdir, 'EFI', 'boot', 'vmlinuz0') + efi_initrd = os.path.join(srcmntdir, 'EFI', 'boot', 'initrd0.img') + if os.path.exists(efi_vmlinuz): + self._exclude.extend(['%s' % efi_vmlinuz]) + self._exclude.extend(['%s' % efi_initrd]) + + if self._src_type == 'live': + stdin = '/' + elif self._src_type in ['blk', 'iso']: + self.liveosmnt = self.liveos_mount(srcmntdir, base_on, 'ro') + stdin = self.liveosmnt.mountdir + rootfs_size = long(rcall(du_args, stdin, stderr=False).split()[-2]) + # staged copy + required = rootfs_size + + if self.skip_compression: + # Space on iso image + required += rootfs_size else: - imgmnt = DiskMount(LoopbackDisk(base_on, 0), self._mkdtemp()) + # assuming a 2:1 compression ratio, at most + required += rootfs_size / 2L + + if not self.skip_minimize: + required += 8192L + + tmp_available = long(disc_free_info(self.tmpdir, '-TB1')[4]) + e2img_size = 1024L * 1024L * 512L + if tmp_available > e2img_size: + e2img = tempfile.NamedTemporaryFile(dir=self.tmpdir) + if self._src_type == 'live': + device = '/dev/mapper/live-rw' + elif self._src_type in ['blk', 'iso']: + device = disc_free_info(self.liveosmnt.mountdir)[0] + rcall(['/sbin/e2image', '-r', device, <a href="http://e2img.name">e2img.name</a>]) + e2img_size = tmp_available - long(disc_free_info( + self.tmpdir, '-TB1')[4]) + if os.stat(self.tmpdir).st_dev == os.stat(self.output).st_dev: + available = tmp_available + required += e2img_size + else: + available = long(disc_free_info(self.output, '-TB1')[4]) - self._LiveImageCreatorBase__isodir = self._ImageCreator__builddir + "/iso" + for fn in self._exclude: + du_args.extend(['--exclude=%s' % os.path.join(srcmntdir, + fn.lstrip(os.sep))]) - try: - imgmnt.mount() - except MountError, e: - raise CreatorError("Failed to mount '%s' : %s" % (base_on, e)) + if self.clone: + includes = ['boot', 'EFI', 'isolinux', 'syslinux', 'LiveOS'] + files0from = '' + for fn in includes: + self._include.extend(['%s' % fn]) + for i, fn in enumerate(self._include): + src = os.path.join(srcmntdir, fn.lstrip(os.sep)) + files0from += '%s\0' % src + self._include[i] = src else: + files0from = srcmntdir + + tobe_copied = long( + rcall(du_args, files0from, stderr=False).split()[-2]) + # 1 staged copy and another in iso9660 file. + required += tobe_copied * 2L + # amount for uncertain overhead + required += self._extra_space + + print '\n{:20,} bytes set to be copied'.format(tobe_copied), + print 'from the source filesystem {:s}.'.format(base_on) + print '{:20,} bytes are in the LiveOS filesystem.\n'.format( + rootfs_size) + print '{:20,} bytes are estimated to be required'.format(required), + print 'for staging this build.\n' + print '{:20,} bytes are available'.format(available), + print 'on {:s} for staging the build.'.format(self.output) + if available <= required: + needed = (required - available) / (1024 * 1024) + print >> sys.stderr, "\n" + "There is insufficient space to build a remix on %s.\n" + "%s MiB of additional space is needed." % (self.output, needed) + liveosmnt.cleanup() + return False + return True + + + def __copy_src_root(self, srcmntdir, base_on): + """helper function to copy root content of the base LiveOS source to + ISOdir + + """ + ignore_list = ['ext3fs.img', 'squashfs.img', 'osmin.img', 'home.img', + 'overlay-*'] + + if self.clone: + ignore_list.remove('home.img') + liveosdir = os.path.join(srcmntdir, 'LiveOS') + + for fn in self._exclude: + ignore_list.extend(['%s' % fn]) + + isodir = self._LiveImageCreatorBase__isodir + + if self.clone: # include specified files or directories - if self.clone: - baseimg = os.path.join(imgmnt.mountdir, 'LiveOS', - 'squashfs.img') - # 'self.compress_type = None' will force reading it from - # base_on. - if self.compress_type is None: - self.compress_type = squashfs_compression_type(baseimg) - if self.compress_type == 'undetermined': - # 'gzip' for compatibility with older versions. - self.compress_type = 'gzip' - - dst = self._LiveImageCreatorBase__isodir - print includes - for fd in includes.split(', /'): - src = os.path.join(imgmnt.mountdir, fd) - if os.path.isfile(src): - shutil.copy2(src, os.path.join(dst, fd)) - elif os.path.isdir(src): - shutil.copytree(src, os.path.join(dst, fd), + tgt_list = [] + for fp in self._include: + if os.path.exists(fp): + tgt_list.extend([os.path.basename(fp)]) + if os.path.isfile(fp): + shutil.copy2(fp, os.path.join(isodir, tgt_list[-1])) + elif os.path.isdir(fp): + shutil.copytree(fp, os.path.join(isodir, tgt_list[-1]), symlinks=True, ignore=shutil.ignore_patterns( *ignore_list)) + print """ + \rDevice folders and files to be included in the new image: + \r %s""" % tgt_list + else: + #copy over everything but squashfs.img or ext3fs.img + print """ + \rDevice folders & files to be excluded from the source root copy: + \r %s""" % ignore_list + shutil.copytree(srcmntdir, isodir, + ignore=shutil.ignore_patterns(*ignore_list)) + + bootfolder = os.path.join(isodir, 'isolinux') + installedpath = os.path.join(isodir, 'syslinux') + if os.path.exists(installedpath): + os.rename(installedpath, bootfolder) + + # build symlinks to reduce iso size + efi_vmlinuz = os.path.join(isodir, 'EFI', 'boot', 'vmlinuz0') + isolinux_vmlinuz = os.path.join(bootfolder, 'vmlinuz0') + efi_initrd = os.path.join(isodir, 'EFI', 'boot', 'initrd0.img') + isolinux_initrd = os.path.join(bootfolder, 'initrd0.img') + + if os.path.exists(os.path.join(srcmntdir, 'EFI', 'boot', 'vmlinuz0')): + os.unlink(efi_vmlinuz) + os.unlink(efi_initrd) + os.symlink(isolinux_vmlinuz, efi_vmlinuz) + os.symlink(isolinux_initrd, efi_initrd) + + subprocess.call(['sync']) + + + def _base_on(self, base_on): + """Clone the running or /dev/mapper LiveOS image as the basis for the + new image. + + """ + + if self._src_type == 'blk': + base_on = self.liveosmnt.mountdir + os.sep + if self.squashmnt: + image = self.liveosmnt.imgloop else: - #copy over everything but squashfs.img or ext3fs.img - shutil.copytree(imgmnt.mountdir, - self._LiveImageCreatorBase__isodir, - ignore=shutil.ignore_patterns(*ignore_list)) - subprocess.call(['sync']) + image = self.liveosmnt.disk + try: + self.srcmnt.mount() + except MountError, e: + raise CreatorError("Failed to mount '%s' : %s" % + (self.srcmnt.device, e)) + self.__image_size = os.stat(image.lofile).st_size + device = image.device + elif self._src_type == 'live': + base_on = '/' + device = '/dev/mapper/live-rw' + liveosdir = os.path.join('/mnt', 'live', 'LiveOS') + rootfs_img = os.path.join(liveosdir, 'ext3fs.img') + squashmnt = None + if not os.path.exists(rootfs_img): + squashloop = get_loop('/sysroot/LiveOS/squashfs.img') + squashmnt = DiskMount(Disk(None, squashloop), + self._mkdtemp('squash-')) + try: + squashmnt.mount() + except MountError, e: + raise CreatorError("Failed to mount '%s' : %s" % + (squashloop, e)) + rootfs_img = os.path.join(squashmnt.mountdir, + 'LiveOS', 'ext3fs.img') + self.__image_size = os.stat(rootfs_img).st_size + if squashmnt: + squashmnt.unmount() + self.__fstype = get_fsvalue('value', 'TYPE', None, device) + self.__blocksize = os.stat(device).st_blksize + if self.squashmnt is not None: + self.srcmnt.unmount() + + self.__instloop = ExtDiskMount(SparseLoopbackDisk(self._image, + self.__image_size), + self._instroot, + self.__fstype, + self.__blocksize, + self.fslabel, + tmpdir=self.tmpdir) + try: + self.__instloop.mount() + except MountError, e: + raise CreatorError("Failed to loop mount '%s' : %s" % + (self._image, e)) + else: + livecfg = 'in' + dbusid = 'ex' + if self.refresh_only: + dbusid = 'in' + if not os.path.exists(os.path.join(base_on, + '.liveimg-configured')): + self.refresh_only = None + + home = 'in' + if os.path.exists(os.path.join(self._LiveImageCreatorBase__isodir, + 'LiveOS', 'home.img')): + home = 'ex' + print "\nCopying the source image..." + subprocess.call(['rsync', '-ptgorlHASxXW', '--specials', '--stats', + '--partial-dir=.rsync-partial', + '--exclude', '- /selinux/', + '--include', '/*/', + '--exclude', '/dev/*', + '--exclude', '/etc/blkid/*', + '--exclude', '/etc/mtab', + '--%sclude' % home, '/home/*', + '--exclude', '/proc/*', + '--exclude', '/mnt/live', + '--exclude', '/sys/*', + '--exclude', '/tmp/*', + '--%sclude' % dbusid, '/var/lib/dbus/machine-id', + '--%sclude' % livecfg, '/.liveimg*', + '--exclude', '/.autofsck', + '%s' % base_on, self._instroot]) + makedirs(os.path.join(self._instroot, 'selinux')) + subprocess.call(['/bin/sync']) + + if self._src_type == 'blk': + self.liveosmnt.cleanup() + finally: - imgmnt.cleanup() + self.__instloop.cleanup() - def _brand (self, _builder): + def _brand(self, _builder): """Adjust the image branding to show its variation from original - source by builder and build date.""" + source by builder and build date. + + """ self.fslabel = <a href="http://self.name">self.name</a> dt = time.strftime('%d-%b-%Y') - lst = ['isolinux/isolinux.cfg', 'syslinux/syslinux.cfg', - 'syslinux/extlinux.conf'] + lst = ['isolinux/isolinux.cfg', 'isolinux/syslinux.cfg', + 'isolinux/extlinux.conf'] for f in lst: fpath = os.path.join(self._LiveImageCreatorBase__isodir, f) if os.path.exists(fpath): @@ -365,13 +673,17 @@ class LiveImageEditor(LiveImageCreator): ntext = dt.translate(None, '-') + '-' + _builder + '-Remix-' + release # Update fedora-release message with Remix details. - releasefiles = '/etc/fedora-release, /etc/generic-release' + releasefiles = ['fedora-release', 'generic-release', 'issue'] + for i, rf in enumerate(releasefiles): + releasefiles[i] = os.path.join(self._instroot, 'etc', rf) if self._releasefile: - releasefiles += ', ' + self._releasefile - for fn in releasefiles.split(', '): - if os.path.exists(fn): + for fn in self._releasefile: + releasefiles.extend([os.path.join(self._instroot, + fn.lstrip(os.sep))]) + for rfpath in releasefiles: + if os.path.exists(rfpath): try: - with open(self._instroot + fn, 'r') as f: + with open(rfpath, 'r') as f: text = ntext + '\n' + f.read() open(<a href="http://f.name">f.name</a>, 'w').write(text) except IOError, e: @@ -379,16 +691,16 @@ class LiveImageEditor(LiveImageCreator): (<a href="http://f.name">f.name</a>, e)) self._releasefile = ntext - <a href="http://self.name">self.name</a> += '-' + os.uname()[4] + '-' + time.strftime('%Y%m%d.%H%M') + kernel = os.path.join(self._LiveImageCreatorBase__isodir, 'isolinux', + 'vmlinuz0') + arch = get_file_info(kernel)[7].split('.')[-1] + <a href="http://self.name">self.name</a> += '-' + arch + '-' + time.strftime('%Y%m%d.%H%M') def _configure_bootloader(self, isodir): """Restore the boot configuration files for an iso image boot.""" bootfolder = os.path.join(isodir, 'isolinux') - oldpath = os.path.join(isodir, 'syslinux') - if os.path.exists(oldpath): - os.rename(oldpath, bootfolder) cfgf = os.path.join(bootfolder, 'isolinux.cfg') for f in ['syslinux.cfg', 'extlinux.conf']: @@ -400,7 +712,9 @@ class LiveImageEditor(LiveImageCreator): '-e', 's/Welcome to .*/Welcome to ' + self._releasefile + '!/', '-e', 's/root=[^ ]*/root=live:CDLABEL=' + <a href="http://self.name">self.name</a> + '/', '-e', 's/rootfstype=[^ ]* [^ ]*/rootfstype=auto ro/', - '-e', 's/liveimg .* quiet/liveimg quiet/', cfgf] + '-e', 's/liveimg .* rd_NO_LUKS/liveimg quiet rhgb rd_NO_LUKS/', + '-e', 's/liveimg .* rd.luks=0/liveimg quiet rhgb rd.luks=0/', + cfgf] dev_null = os.open("/dev/null", os.O_WRONLY) try: @@ -408,29 +722,209 @@ class LiveImageEditor(LiveImageCreator): stdout = subprocess.PIPE, stderr = dev_null).communicate()[0] return 0 - except IOError, e: raise CreatorError("Failed to configure bootloader file: %s" % e) return 1 finally: os.close(dev_null) + + def no_new_iso(self, iso): + """Hijacked the _LiveImageCreatorBase__create_iso base method to + prevent creating a new iso when the refresh_only option is selected. + + """ + + self.refresh(iso) + + def refresh(self, iso): + """Hijacked the _LiveImageCreatorBase__implant_md5sum base method to + insert code to refresh a running or livemounted image's ext3fs.img and + overlay files (just before the staged files are deleted). + + """ + + if not self.refresh_only: + # Implant an isomd5sum. + if os.path.exists("/usr/bin/implantisomd5"): + implantisomd5 = "/usr/bin/implantisomd5" + elif os.path.exists("/usr/lib/anaconda-runtime/implantisomd5"): + implantisomd5 = "/usr/lib/anaconda-runtime/implantisomd5" + else: + logging.warn("isomd5sum not installed; not setting up mediacheck") + return + subprocess.call([implantisomd5, iso]) + + if self._src_type == 'live': + print """ + \r Notice: Refreshing from running images is not yet supported. + """ + return + elif self._src_type == 'iso': + return + + print "\nRefreshing the source device with the new image..." + isodir = self._LiveImageCreatorBase__isodir + + try: + self.srcmnt.mount() + except MountError, e: + raise CreatorError("Failed to mount '%s' : '%s'" % + (self.srcmnt.mountdir, e)) + else: + if self._space_for_refresh(self.srcmnt, isodir): + liveosdir = os.path.join(self.srcmnt.mountdir, 'LiveOS') + if self._overlay: + self._overlay = os.path.join(liveosdir, self._overlay) + dd_args = ['/bin/dd', 'if=/dev/zero', + 'of=%s' % self._overlay, 'bs=64k', 'count=1', + 'conv=notrunc'] + + staged_LiveOS = os.path.join(isodir, 'LiveOS') + + def _proclists(fname, subt=None): + if fname: + fpath = os.path.join(liveosdir, fname) + if os.path.exists(fpath): + dellist.extend([fpath]) + if subt: + filelist.extend([os.path.join(staged_LiveOS, subt)]) + target = os.path.join(liveosdir, subt) + tgtlist.extend([target]) + dellist.extend([target]) + + filelist, tgtlist, dellist = ([], [], []) + if self.skip_minimize: + _proclists('osmin.img') + else: + _proclists(None, 'osmin.img') + if self.skip_compression: + _proclists('squashfs.img', 'ext3fs.img') + elif self.squashmnt: + _proclists('ext3fs.img', 'squashfs.img') + else: + filelist.extend([os.path.join(os.path.dirname(self._image), + 'LiveOS', 'ext3fs.img')]) + target = os.path.join(liveosdir, 'ext3fs.img') + tgtlist.extend([target]) + dellist.extend([target]) + + for f in ['syslinux.cfg', 'extlinux.conf']: + cfgf = os.path.join(self.srcmnt.mountdir, 'syslinux', f) + if os.path.exists(cfgf): + cfgf_sed = ['/bin/sed', '-i', '-e', + 's/Welcome to .*/Welcome to ' + self._releasefile + '!/', + cfgf] + + environ = os.environ.copy() + cmd_cp = '/bin/cp' + cmd_rm = '/bin/rm' + + # Copy new ext3fs.img or squashfs.img and osmin.img, and reset + # the overlay. + for f in dellist: + rcall([cmd_rm, f], env=environ) + for i, f in enumerate(filelist): + rcall([cmd_cp, '--remove-destination', f, tgtlist[i]], + env=environ) + if self._overlay: + print "Resetting the overlay file..." + rcall(dd_args, env=environ) + rcall(cfgf_sed, env=environ) + + # flag refresh image as configured. (This has already been done + # for the refresh_only case.) + if self.refresh_only is not None: + self.liveosmnt.ops = 'rw' + try: + self.liveosmnt.mount() + except MountError, e: + raise CreatorError("Failed to mount '%s' : %s" % + (self.liveosmnt.mountdir, e)) + else: + shutil.copy('/.liveimg-configured', + self.liveosmnt.mountdir) + shutil.copy('/.liveimg-late-configured', + self.liveosmnt.mountdir) + finally: + self.liveosmnt.cleanup() + + finally: + self.srcmnt.unmount() + + + def _space_for_refresh(self, srcmnt, isodir): + """Check for sufficient space on the installation device for a refresh. + """ + + du_args = ['/usr/bin/du', '-csxb', '--files0-from=-'] + + include = ['squashfs.img', 'osmin.img', 'ext3fs.img', + 'livecd-iso-to-disk'] + include2 = include[:] + files0from = '' + for i, fn in enumerate(include): + include[i] = os.path.join(isodir, 'LiveOS', fn) + files0from += '%s\0' % include[i] + if not self.squashmnt: + files0from += '%s\0' % os.path.join(os.path.dirname(self._image), + 'LiveOS', 'ext3fs.img') + tobe_copied = long( + rcall(du_args, files0from, stderr=False).split()[-2]) + files0from = '' + for i, fn in enumerate(include2): + include2[i] = os.path.join(srcmnt.mountdir, 'LiveOS', fn) + files0from += '%s\0' % include2[i] + tobe_deleted = long( + rcall(du_args, files0from, stderr=False).split()[-2]) + available = long(disc_free_info(srcmnt.mountdir, + '-TB1')[4]) + tobe_deleted + files0from = os.path.join(srcmnt.mountdir, 'LiveOS', self._overlay) + overlay_size = long( + rcall(du_args, files0from, stderr=False).split()[-2]) + files0from = os.path.join(srcmnt.mountdir, 'LiveOS', 'home.img') + home_size = long(rcall(du_args, files0from, stderr=False).split()[-2]) + reduced_overlay_size = available + overlay_size + print '\n{:20,} bytes to be copied'.format(tobe_copied) + print '{:20,} bytes to be deleted'.format(tobe_deleted) + print '{:20,} bytes in home.img'.format(home_size) + print '{:20,} bytes in overlay'.format(overlay_size) + print '{:20,} bytes available'.format(available) + if available <= 0: + print '{:20,} bytes in potential overlay'.format( + reduced_overlay_size) + return False + return True + + + def _unmount_instroot(self): + """Undo anything performed in mount().""" + + if self.liveosmnt: + self.liveosmnt.cleanup() + if self.squashmnt: + self.squashmnt.cleanup() + def parse_options(args): - parser = optparse.OptionParser(usage = """ - %prog [-n=<name>] - [-o=<output>] - [-s=<script.sh>] - [-t=<tmpdir>] - [-e=<excludes>] - [-f=<exclude-file>] - [-i=<includes>] - [-r=<releasefile>] - [-b=<builder>] + parser = optparse.OptionParser(usage = ''' + %prog [-n, --name <name>] + [-o, --output <output directory>] + [-s, --script <script.sh>] + [-N, --noshell] + [-t, --tmpdir <tmpdir>] + [-e, --exclude <exclude, s>] + [-i, --include <include, s>] + [-r, --releasefile <releasefile, s>] + [-b, --builder <builder>] [--clone] - [-c=<compress_type>] + [--refresh_only] + [--no-refresh] + [-c, --compress-type <compression type>] + [--compress] [--skip-compression] [--skip-minimize] - <LIVEIMG.src>""") + [--extra-space] + <LIVEIMG.src>''') parser.add_option("-n", "--name", type="string", dest="name", help="name of new LiveOS (don't include .iso, it will " @@ -443,46 +937,74 @@ def parse_options(args): help="specify script to run chrooted in the LiveOS " "fsimage") + parser.add_option("-N", "--noshell", action="store_false", + dest="shell", default=True, + help="Specify no break to shell after edit.") + parser.add_option("-t", "--tmpdir", type="string", dest="tmpdir", default="/var/tmp", help="Temporary directory to use (default: /var/tmp)") parser.add_option("-e", "--exclude", type="string", dest="exclude", - help="Specify directory or file patterns to be excluded " - "from the rsync copy of the filesystem.") - - parser.add_option("-f", "--exclude-file", type="string", - dest="exclude_file", - help="Specify a file containing file patterns to be " - "excluded from the rsync copy of the filesystem.") + help='''A string of file or directory paths in the outer device + filesystem to exclude from __copy_src_root(). Include multiple files + as "file1, file2, ..." ''') parser.add_option("-i", "--include", type="string", dest="include", - help="Specify directory or file patterns to be included " - "in copy_img_root.") + help='''A string of file or directory paths in the outer device + filesystem to include in __copy_src_root(). Include multiple files + as "file1, file2, ..." ''') parser.add_option("-r", "--releasefile", type="string", dest="releasefile", - help="Specify release file/s for branding.") + help='''Specify release file/s for branding. Include + multiple files as "file1, file2, ..." ''') parser.add_option("-b", "--builder", type="string", dest="builder", default=os.getlogin(), help="Specify the builder of a Remix.") - parser.add_option("", "--clone", action="store_true", dest="clone", - help="Specify that source image is LiveOS block device.") + parser.add_option("", "--clone", action="store_true", + dest="clone", default=False, + help="""Specify copying of the LiveOS root filesystem. + This option will also refresh the source filesystem, + unless the --no-refresh option is also invoked.""") + + parser.add_option("", "--refresh-only", action="store_true", + dest="refresh_only", default=False, + help='''Specify replacing the squashfs.img or ext3fs.img + of the source LiveOS installation instance with such + files from the new build, and resetting any overlay. + No new iso installation file will be produced.''') + + parser.add_option("", "--no-refresh", action="store_true", + dest="no_refresh", default=False, + help='''Specify no refresh of the source filesystems.''') - parser.add_option("-c", "--compress_type", type="string", + parser.add_option("-c", "--compress-type", type="string", dest="compress_type", help="Specify the compression type for SquashFS. Will " "override the current compression or lack thereof.") + parser.add_option("", "--compress", action="store_true", + dest="compress", default=None, + help="Specify compression of the filesystem image, " + "ext3fs.img. Used when overriding the base_on " + "compression status.") + parser.add_option("", "--skip-compression", action="store_true", - dest="skip_compression", default=False, - help="Specify no compression of filesystem, ext3fs.img") + dest="skip_compression", default=None, + help="Specify no compression of filesystem image, " + "ext3fs.img") parser.add_option("", "--skip-minimize", action="store_true", - dest="skip_minimize", default=False, + dest="skip_minimize", default=None, help="Specify no osmin.img minimal snapshot.") + parser.add_option("", "--extra-space", type="string", + dest="extra_space", default='2', + help='''Specify extra space to reserve for unexpected + staging area needs.''') + setup_logging(parser) (options, args) = parser.parse_args() @@ -491,36 +1013,105 @@ def parse_options(args): parser.print_usage() sys.exit(1) - print args[0] + print "\nSource image at %s" % args[0] return (args[0], options) -def get_fsvalue(filesystem, tag): + +def get_fsvalue(format=None, tag=None, token=None, filesystem=None): + """Return filesystem information based on a blkid tag, token, or device.""" + dev_null = os.open('/dev/null', os.O_WRONLY) - args = ['/sbin/blkid', '-s', tag, '-o', 'value', filesystem] + args = ['/sbin/blkid', '-c', '/dev/null'] + if format: + args.extend(['-o', format]) + if tag: + args.extend(['-s', tag]) + if token: + args.extend(['-l', '-t', token]) + if filesystem: + args.extend([filesystem]) try: fs_type = subprocess.Popen(args, stdout=subprocess.PIPE, - stderr=dev_null).communicate()[0] + stderr=dev_null).communicate()[0].rstrip() except IOError, e: - raise CreatorError("Failed to determine fs %s: %s" % value, e ) + raise CreatorError("Failed to determine fs %s: %s" % value, e) finally: os.close(dev_null) + return fs_type - return fs_type.rstrip() -def rebuild_iso_symlinks(isodir): - # remove duplicate files and rebuild symlinks to reduce iso size - efi_vmlinuz = "%s/EFI/boot/vmlinuz0" % isodir - isolinux_vmlinuz = "%s/isolinux/vmlinuz0" % isodir - efi_initrd = "%s/EFI/boot/initrd0.img" % isodir - isolinux_initrd = "%s/isolinux/initrd0.img" % isodir +def rcall(args, stdin=None, stderr=True, env=None): + out, err, p, environ = None, None, None, None + if env: + environ = os.environ.copy() + environ.update(env) + try: + p = subprocess.Popen(args, stdin=subprocess.PIPE, env=environ, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate(stdin) + except OSError, e: + raise CreatorError(u"Failed executing:\n'%s'\n'%s'" % (args, e)) + except: + raise CreatorError(u"""Failed to execute:\n'%s' + \renviron: '%s'\nstdout: '%s'\nstderr: '%s'""" % + (args, environ, out, err)) + else: + if p.returncode != 0 and stderr: + raise CreatorError(u"""Error in call:\n'%s'\nenviron: '%s' + \rstdout: '%s'\nstderr: '%s'\nreturncode: '%s'""" % + (args, environ, out, err, p.returncode)) + return out + + +def disc_free_info(path, options=None): + """Return the disc free information for a file or device path.""" + + info = [None] * 7 + if os.path.exists(path): + st_mode = os.stat(path).st_mode + if stat.S_ISBLK( + st_mode) or os.path.ismount(path) or stat.S_ISDIR(st_mode): + df_args = ['/bin/df'] + if options: + df_args.extend([options]) + df_args.extend([path]) + devinfo = rcall(df_args).splitlines() + info = devinfo[1].split(None) + if len(info) == 1: + info.extend(devinfo[2].split(None)) + return info + + +def get_loop(path): + """Return the loop device for a given mount point path.""" + + loopdevice = None + for loop in rcall(['/sbin/losetup', '-a']).splitlines(): + info = loop.split() + if path == info[2].strip('()'): + loopdevice = info[0].rstrip(':') + break + return loopdevice - if os.path.exists(efi_vmlinuz): - os.remove(efi_vmlinuz) - os.remove(efi_initrd) - os.symlink(isolinux_vmlinuz,efi_vmlinuz) - os.symlink(isolinux_initrd,efi_initrd) + +def add_loop(n=8): + """Add a loop device.""" + + while os.path.exists('/dev/loop%s' % n): + n += 1 + os.mknod('/dev/loop%s' % n, 0660 | stat.S_IFBLK, os.makedev(7, n)) + return '/dev/loop%s' % n + + +def get_file_info(path): + """Retrieve file information.""" + + info = [None] * 16 + if os.path.exists(path): + info = rcall(['/usr/bin/file', '-b', path]).split() + return info def main(): # LiveOS set to <LIVEIMG.src> @@ -530,51 +1121,91 @@ def main(): print >> sys.stderr, "You must run edit-liveos as root" return 1 - if stat.S_ISBLK(os.stat(LiveOS).st_mode): - name = get_fsvalue(LiveOS, 'LABEL') + '.edited' - elif <a href="http://options.name">options.name</a> and <a href="http://options.name">options.name</a> != os.path.basename(LiveOS): + try: + long(options.extra_space) + except ValueError, e: + raise CreatorError('''Notice, --extra-space: %s is not a number. + \rPlease correct this.''' % options.extra_space, e) + + name = os.path.basename(LiveOS) + ".edited" + src_type = 'iso' + st_mode = os.stat(LiveOS).st_mode + if stat.S_ISBLK(st_mode): + src_type = 'blk' + name = get_fsvalue('value', 'LABEL', None, LiveOS) + '.edited' + if LiveOS == '/dev/live': + src_type = 'live' + if <a href="http://options.name">options.name</a> and <a href="http://options.name">options.name</a> != os.path.basename(LiveOS): name = <a href="http://options.name">options.name</a> - else: - name = os.path.basename(LiveOS) + ".edited" if options.output: output = options.output else: - output = os.path.dirname(LiveOS) - if output == '/dev': + output = os.path.dirname(os.path.normpath(LiveOS)) + if output in ['/dev']: output = options.tmpdir editor = LiveImageEditor(name) - editor._exclude = options.exclude - editor._exclude_file = options.exclude_file - editor._include = options.include + editor._src_type = src_type + if options.exclude: + editor._exclude = options.exclude.split(', ') + if options.include: + editor._include = options.include.split(', ') + editor._script = options.script + editor._shell = options.shell editor.clone = options.clone + if options.refresh_only: + editor.refresh_only = options.refresh_only + editor.clone = True editor.tmpdir = options.tmpdir + editor.output = output editor._builder = options.builder - editor._releasefile = options.releasefile + if options.releasefile: + editor._releasefile = options.releasefile.split(', ') editor.compress_type = options.compress_type editor.skip_compression = options.skip_compression + if options.compress: + editor.skip_compression = False editor.skip_minimize = options.skip_minimize + editor._extra_space = long(options.extra_space) * 1024L * 1024L try: editor.mount(LiveOS, cachedir = None) + editor._brand(editor._builder) editor._configure_bootloader(editor._LiveImageCreatorBase__isodir) if options.script: print "Running edit script '%s'" % options.script editor._run_script(options.script) - else: + elif editor._shell: print "Launching shell. Exit to continue." print "----------------------------------" editor.launch_shell() - rebuild_iso_symlinks(editor._LiveImageCreatorBase__isodir) editor.unmount() + if options.refresh_only: + self.refresh_only = True + editor._LiveImageCreatorBase__create_iso = editor.no_new_iso + if not editor.skip_compression: + print '''\nThe new image will now be resquashed. + Please wait...''' + elif options.clone and not options.no_refresh: + editor._LiveImageCreatorBase__implant_md5sum = editor.refresh + print '\nThe new image will now be packaged. Please wait...' editor.package(output) + call(['/bin/sync']) <a href="http://logging.info">logging.info</a>("%s.iso saved to %s" % (<a href="http://editor.name">editor.name</a>, output)) except CreatorError, e: - logging.error(u"Error editing LiveOS : %s" % e) + logging.error(u"Error editing LiveOS : '%s'" % e) + print "\nError editing LiveOS: '%s'" % e return 1 finally: + name = <a href="http://editor.name">editor.name</a> + for loop in editor.extra_loops: + if loop: + call(['/bin/rm', '-f', loop]) editor.cleanup() + print 'LiveOS edit has completed.' + if not editor.refresh_only: + print ' See %s in %s' % (name, output) return 0