def generate_bmap(self):
        """ Generate block map file for the image. The idea is that while disk
        images we generate may be large (e.g., 4GiB), they may actually contain
        only little real data, e.g., 512MiB. This data are files, directories,
        file-system meta-data, partition table, etc. In other words, when
        flashing the image to the target device, you do not have to copy all the
        4GiB of data, you can copy only 512MiB of it, which is 4 times faster.

        This function generates the block map file for an arbitrary image that
        mic has generated. The block map file is basically an XML file which
        contains a list of blocks which have to be copied to the target device.
        The other blocks are not used and there is no need to copy them. """

        if self.bmap_needed is None:
            return

        msger.info("Generating the map file(s)")

        for name in self.__disks.keys():
            image = self._full_path(self.__imgdir, name, self.__disk_format)
            bmap_file = self._full_path(self._outdir, name, "bmap")
            self.image_files.setdefault(name, {}).update({'bmap': \
                                            os.path.basename(bmap_file)})

            msger.debug("Generating block map file '%s'" % bmap_file)
            
            bmaptoolcmd = misc.find_binary_path('bmaptool')
            rc = runner.show([bmaptoolcmd, 'create', image, '-o', bmap_file])
            if rc != 0:
                raise CreatorError("Failed to create bmap file: %s" % bmap_file)

    def _mount_instroot(self, base_on = None):
        parts = self._get_parts()
        self.__instloop = PartitionedMount(self._instroot)

        for p in parts:
            self.__instloop.add_partition(int(p.size),
                                          p.disk,
                                          p.mountpoint,
                                          p.fstype,
                                          p.label,
                                          fsopts = p.fsopts,
                                          boot = p.active,
                                          align = p.align,
                                          part_type = p.part_type)

        self.__instloop.layout_partitions(self._ptable_format)

        # Create the disks
        self.__imgdir = self._mkdtemp()
        for disk_name, disk in self.__instloop.disks.items():
            full_path = self._full_path(self.__imgdir, disk_name, "raw")
            msger.debug("Adding disk %s as %s with size %s bytes" \
                        % (disk_name, full_path, disk['min_size']))

            disk_obj = fs_related.SparseLoopbackDisk(full_path,
                                                     disk['min_size'])
            self.__disks[disk_name] = disk_obj
            self.__instloop.add_disk(disk_name, disk_obj)

        self.__instloop.mount()
        self._create_mkinitrd_config()

    def _install_syslinux(self):
        for name in self.__disks.keys():
            loopdev = self.__disks[name].device

            # Set MBR
            mbrfile = "%s/usr/share/syslinux/" % self._instroot
            if self._ptable_format == 'gpt':
                mbrfile += "gptmbr.bin"
            else:
                mbrfile += "mbr.bin"

            msger.debug("Installing syslinux bootloader '%s' to %s" % \
                        (mbrfile, loopdev))

            rc = runner.show(['dd', 'if=%s' % mbrfile, 'of=' + loopdev])
            if rc != 0:
                raise MountError("Unable to set MBR to %s" % loopdev)


            # Ensure all data is flushed to disk before doing syslinux install
            runner.quiet('sync')

            fullpathsyslinux = fs_related.find_binary_path("extlinux")
            rc = runner.show([fullpathsyslinux,
                              "-i",
                              "%s/boot/extlinux" % self._instroot])
            if rc != 0:
                raise MountError("Unable to install syslinux bootloader to %s" \
                                 % loopdev)

    def __format_filesystem(self):
        if self.skipformat:
            msger.debug("Skip filesystem format.")
            return

        msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
        cmdlist = [self.mkfscmd, "-F", "-L", self.fslabel, "-m", "1", "-b",
                   str(self.blocksize)]
        if self.extopts:
            cmdlist.extend(self.extopts.split())
        cmdlist.extend([self.disk.device])

        rc, errout = runner.runtool(cmdlist, catch=2)
        if rc != 0:
            raise MountError("Error creating %s filesystem on disk %s:\n%s" %
                             (self.fstype, self.disk.device, errout))

        if not self.extopts:
            msger.debug("Tuning filesystem on %s" % self.disk.device)
            runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device])

        rc, errout = runner.runtool([self.dumpe2fs, '-h', self.disk.device],
                                    catch=2)
        if rc != 0:
            raise MountError("Error dumpe2fs %s filesystem on disk %s:\n%s" %
                             (self.fstype, self.disk.device, errout))
        # FIXME: specify uuid in mkfs parameter
        try:
            self.uuid = self.__parse_field(out, "Filesystem UUID")
        except:
            self.uuid = None

def exec_native_cmd(cmd_and_args, native_sysroot, catch = 3):
    """
    Execute native command, catching stderr, stdout

    Need to execute as_shell if the command uses wildcards

    Always need to execute native commands as_shell
    """
    native_paths = \
        "export PATH=%s/sbin:%s/usr/sbin:%s/usr/bin:$PATH" % \
        (native_sysroot, native_sysroot, native_sysroot)
    native_cmd_and_args = "%s;%s" % (native_paths, cmd_and_args)
    msger.debug("exec_native_cmd: %s" % cmd_and_args)

    args = cmd_and_args.split()
    msger.debug(args)

    rc, out = exec_cmd(native_cmd_and_args, True, catch)

    if rc == 127: # shell command-not-found
        msger.error("A native (host) program required to build the image "
                    "was not found (see details above). Please make sure "
                    "it's installed and try again.")

    return (rc, out)

    def generate_bmap(self):
        """ Generate block map file for the image. The idea is that while disk
        images we generate may be large (e.g., 4GiB), they may actually contain
        only little real data, e.g., 512MiB. This data are files, directories,
        file-system meta-data, partition table, etc. In other words, when
        flashing the image to the target device, you do not have to copy all the
        4GiB of data, you can copy only 512MiB of it, which is 4 times faster.

        This function generates the block map file for an arbitrary image that
        mic has generated. The block map file is basically an XML file which
        contains a list of blocks which have to be copied to the target device.
        The other blocks are not used and there is no need to copy them. """

        if self.bmap_needed is None:
            return

        from mic.utils import BmapCreate
        msger.info("Generating the map file(s)")

        for name in self.__disks.keys():
            image = self._full_path(self.__imgdir, name, self.__disk_format)
            bmap_file = self._full_path(self._outdir, name, "bmap")

            msger.debug("Generating block map file '%s'" % bmap_file)

            try:
                creator = BmapCreate.BmapCreate(image, bmap_file)
                creator.generate()
                del creator
            except BmapCreate.Error as err:
                raise CreatorError("Failed to create bmap file: %s" % str(err))

    def _stage_final_image(self):
        """Stage the final system image in _outdir.
           write meta data
        """
        self._resparse()

        if self.compress_image:
            for imgfile in os.listdir(self.__imgdir):
                if imgfile.endswith('.raw') or imgfile.endswith('bin'):
                    imgpath = os.path.join(self.__imgdir, imgfile)
                    msger.info("Compressing image %s" % imgfile)
                    misc.compressing(imgpath, self.compress_image)

        if self.pack_to:
            dst = os.path.join(self._outdir, self.pack_to)
            msger.info("Pack all raw images to %s" % dst)
            misc.packing(dst, self.__imgdir)
        else:
            msger.debug("moving disks to stage location")
            for imgfile in os.listdir(self.__imgdir):
                src = os.path.join(self.__imgdir, imgfile)
                dst = os.path.join(self._outdir, imgfile)
                msger.debug("moving %s to %s" % (src,dst))
                shutil.move(src,dst)
        self._write_image_xml()

    def package(self, destdir = "."):
        """Prepares the created image for final delivery.

        In its simplest form, this method merely copies the install root to the
        supplied destination directory; other subclasses may choose to package
        the image by e.g. creating a bootable ISO containing the image and
        bootloader configuration.

        destdir -- the directory into which the final image should be moved;
                   this defaults to the current directory.

        """
        self._stage_final_image()

        if not os.path.exists(destdir):
            fs.makedirs(destdir)
        if self._img_compression_method:
            if not self._img_name:
                raise CreatorError("Image name not set.")
            rc = None
            img_location = os.path.join(self._outdir,self._img_name)
            if self._img_compression_method == "bz2":
                bzip2 = fs.find_binary_path('bzip2')
                msger.info("Compressing %s with bzip2. Please wait..." \
                           % img_location)
                rc = runner.show([bzip2, "-f", img_location])
                if rc:
                    raise CreatorError("Failed to compress image %s with %s." \
                                % (img_location, self._img_compression_method))

                for bootimg in glob.glob(os.path.dirname(img_location) + \
                                         "/*-boot.bin"):
                    msger.info("Compressing %s with bzip2. Please wait..." \
                               % bootimg)
                    rc = runner.show([bzip2, "-f", bootimg])
                    if rc:
                        raise CreatorError("Failed to compress image %s with "
                                           "%s." \
                                           % (bootimg,
                                              self._img_compression_method))

        if self._recording_pkgs:
            self._save_recording_pkgs(destdir)

        # For image formats with two or multiple image files, it will be
        # better to put them under a directory
        if self.image_format in ("raw", "vmdk", "vdi", "nand", "mrstnand"):
            destdir = os.path.join(destdir, "%s-%s" \
                                            % (self.name, self.image_format))
            msger.debug("creating destination dir: %s" % destdir)
            fs.makedirs(destdir)

        # Ensure all data is flushed to _outdir
        runner.quiet('sync')

        for f in os.listdir(self._outdir):
            shutil.move(os.path.join(self._outdir, f),
                        os.path.join(destdir, f))
            self.outimage.append(os.path.join(destdir, f))
            self.do_genchecksum(os.path.join(destdir, f))

 def testDebug(self):
     excepted = "Debug: hello\n"
     msger.debug("hello")
     self.assertEqual("", sys.stdout.getvalue())
     msger.set_loglevel("debug")
     msger.debug("hello")
     self.assertEqual(excepted, sys.stderr.getvalue())

    def addUser(self, userconfig):
        args = [ "/usr/sbin/useradd" ]
        if userconfig.groups:
            args += [ "--groups", string.join(userconfig.groups, ",") ]
        if userconfig.name:
            args += [ "-m"]
            args += [ "-d", "/home/%s" % userconfig.name  ]
            args.append(userconfig.name)
            try:
                dev_null = os.open("/dev/null", os.O_WRONLY)
                msger.debug('adding user with %s' % args)
                subprocess.call(args,
                                 stdout = dev_null,
                                 stderr = dev_null,
                                 preexec_fn = self.chroot)
                os.close(dev_null)
            except:
                msger.warning('Cannot add user using "useradd"')

            if userconfig.password not in (None, ""):
                if userconfig.isCrypted:
                    self.set_encrypted_passwd(userconfig.name,
                                              userconfig.password)
                else:
                    self.set_unencrypted_passwd(userconfig.name,
                                                userconfig.password)
            else:
                self.set_empty_passwd(userconfig.name)
        else:
            raise errors.KsError("Invalid kickstart command: %s" \
                                 % userconfig.__str__())

    def __copy_kernel_and_initramfs(self, isodir, version, index):
        bootdir = self._instroot + "/boot"
        isDracut = False

        if self._alt_initrd_name:
            src_initrd_path = os.path.join(bootdir, self._alt_initrd_name)
        else:
            if os.path.exists(bootdir + "/initramfs-" + version + ".img"):
                src_initrd_path = os.path.join(bootdir, "initramfs-" +version+ ".img")
                isDracut = True
            else:
                src_initrd_path = os.path.join(bootdir, "initrd-" +version+ ".img")

        try:
            msger.debug("copy %s to %s" % (bootdir + "/vmlinuz-" + version, isodir + "/isolinux/vmlinuz" + index))
            shutil.copyfile(bootdir + "/vmlinuz-" + version,
                    isodir + "/isolinux/vmlinuz" + index)

            msger.debug("copy %s to %s" % (src_initrd_path, isodir + "/isolinux/initrd" + index + ".img"))
            shutil.copyfile(src_initrd_path,
                            isodir + "/isolinux/initrd" + index + ".img")
        except:
            raise CreatorError("Unable to copy valid kernels or initrds, "
                               "please check the repo.")

        is_xen = False
        if os.path.exists(bootdir + "/xen.gz-" + version[:-3]):
            shutil.copyfile(bootdir + "/xen.gz-" + version[:-3],
                            isodir + "/isolinux/xen" + index + ".gz")
            is_xen = True

        return (is_xen,isDracut)

def normalize_ksfile(ksconf, release, arch):
    def _clrtempks():
        try:
            os.unlink(ksconf)
        except:
            pass

    if not os.path.exists(ksconf):
        return

    if not release:
        release = "latest"
    if not arch or re.match(r'i.86', arch):
        arch = "ia32"

    with open(ksconf) as f:
        ksc = f.read()

    if "@[email protected]" in ksc or "@[email protected]" in ksc:
        msger.info("Substitute macro variable @[email protected]/@ARCH in ks: %s" % ksconf)
        ksc = ksc.replace("@[email protected]", arch)
        ksc = ksc.replace("@[email protected]", release)
        fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf), dir="/tmp/")
        os.write(fd, ksc)
        os.close(fd)

        msger.debug('new ks path %s' % ksconf)

        import atexit
        atexit.register(_clrtempks)

    return ksconf

    def expand(self, create = False, size = None):
        flags = os.O_WRONLY
        if create:
            flags |= os.O_CREAT
            if not os.path.exists(self.lofile):
                makedirs(os.path.dirname(self.lofile))

        if size is None:
            size = self.size

        msger.debug("Extending sparse file %s to %d" % (self.lofile, size))
        if create:
            fd = os.open(self.lofile, flags, 0644)
        else:
            fd = os.open(self.lofile, flags)

        if size <= 0:
            size = 1
        try:
            os.ftruncate(fd, size)
        except:
            # may be limited by 2G in 32bit env
            os.ftruncate(fd, 2**31L)

        os.close(fd)

    def __create_subvolume_metadata(self, p, pdisk):
        if len(self.subvolumes) == 0:
            return

        argv = [ self.btrfscmd, "subvolume", "list", pdisk.mountdir ]
        rc, out = runner.runtool(argv)
        msger.debug(out)

        if rc != 0:
            raise MountError("Failed to get subvolume id from %s', return code: %d." % (pdisk.mountdir, rc))

        subvolid_items = out.splitlines()
        subvolume_metadata = ""
        for subvol in self.subvolumes:
            for line in subvolid_items:
                if line.endswith(" path %s" % subvol["subvol"]):
                    subvolid = line.split()[1]
                    if not subvolid.isdigit():
                        raise MountError("Invalid subvolume id: %s" % subvolid)

                    subvolid = int(subvolid)
                    opts = subvol["fsopts"].split(",")
                    for opt in opts:
                        if opt.strip().startswith("subvol="):
                            opts.remove(opt)
                            break
                    fsopts = ",".join(opts)
                    subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], subvol['mountpoint'], fsopts)

        if subvolume_metadata:
            fd = open("%s/.subvolume_metadata" % pdisk.mountdir, "w")
            fd.write(subvolume_metadata)
            fd.close()

    def package(self, destdir="."):
        """Prepares the created image for final delivery.

        In its simplest form, this method merely copies the install root to the
        supplied destination directory; other subclasses may choose to package
        the image by e.g. creating a bootable ISO containing the image and
        bootloader configuration.

        destdir -- the directory into which the final image should be moved;
                   this defaults to the current directory.

        """
        self._stage_final_image()

        if not os.path.exists(destdir):
            fs.makedirs(destdir)

        if self._recording_pkgs:
            self._save_recording_pkgs(destdir)

        # For image formats with two or multiple image files, it will be
        # better to put them under a directory
        if self.image_format in ("raw", "vmdk", "vdi", "nand", "mrstnand"):
            destdir = os.path.join(destdir, "%s-%s" % (self.name, self.image_format))
            msger.debug("creating destination dir: %s" % destdir)
            fs.makedirs(destdir)

        # Ensure all data is flushed to _outdir
        runner.quiet("sync")

        misc.check_space_pre_cp(self._outdir, destdir)
        for f in os.listdir(self._outdir):
            shutil.move(os.path.join(self._outdir, f), os.path.join(destdir, f))
            self.outimage.append(os.path.join(destdir, f))
            self.do_genchecksum(os.path.join(destdir, f))

    def mount(self, options = None):
        if self.mounted:
            return

        if not os.path.isdir(self.mountdir):
            msger.debug("Creating mount point %s" % self.mountdir)
            os.makedirs(self.mountdir)
            self.rmdir = self.rmmountdir

        self.__create()

        msger.debug("Mounting %s at %s" % (self.disk.device, self.mountdir))
        if options:
            args = [ self.mountcmd, "-o", options, self.disk.device, self.mountdir ]
        else:
            args = [ self.mountcmd, self.disk.device, self.mountdir ]
        if self.fstype:
            args.extend(["-t", self.fstype])

        rc = runner.show(args)
        if rc != 0:
            raise MountError("Failed to mount '%s' to '%s' with command '%s'. Retval: %s" %
                             (self.disk.device, self.mountdir, " ".join(args), rc))

        self.mounted = True

    def _mount_instroot(self, base_on = None):
        self.__imgdir = self._mkdtemp()

        parts = self._get_parts()

        #create disk
        for item in self.get_diskinfo():
            msger.debug("Adding disk %s as %s/%s-%s.raw with size %s bytes" %
                        (item['name'], self.__imgdir, self.name, item['name'],
                         item['size']))

            disk = fs_related.SparseLoopbackDisk("%s/%s-%s.raw" % (
                                                                self.__imgdir,
                                                                self.name,
                                                                item['name']),
                                                 item['size'])
            self.__disks[item['name']] = disk

        self.__instloop = PartitionedMount(self.__disks, self._instroot)

        for p in parts:
            self.__instloop.add_partition(int(p.size),
                                          p.disk,
                                          p.mountpoint,
                                          p.fstype,
                                          fsopts = p.fsopts,
                                          boot = p.active,
                                          align = p.align)

        self.__instloop.mount()
        self._create_mkinitrd_config()

 def do_prepare_partition(self, part, cr, cr_workdir, oe_builddir, bootimg_dir,
                          kernel_dir, native_sysroot):
     """
     Called to do the actual content population for a partition i.e. it
     'prepares' the partition to be incorporated into the image.
     """
     msger.debug("SourcePlugin: do_prepare_partition: part: %s" % part)

    def _install_syslinux(self):
        i = 0
        for name in self.__disks.keys():
            loopdev = self.__disks[name].device
            i =i+1

        msger.debug("Installing syslinux bootloader to %s" % loopdev)

        (bootdevnum, rootdevnum, rootdev, prefix) = self._get_syslinux_boot_config()


        #Set MBR
        mbrsize = os.stat("%s/usr/share/syslinux/mbr.bin" % self._instroot)[stat.ST_SIZE]
        rc = runner.show(['dd', "if=%s/usr/share/syslinux/mbr.bin" % self._instroot, "of=" + loopdev])
        if rc != 0:
            raise MountError("Unable to set MBR to %s" % loopdev)

        #Set Bootable flag
        parted = fs_related.find_binary_path("parted")
        rc = runner.quiet([parted, "-s", loopdev, "set", "%d" % (bootdevnum + 1), "boot", "on"])
        #XXX disabled return code check because parted always fails to
        #reload part table with loop devices. Annoying because we can't
        #distinguish this failure from real partition failures :-(
        if rc != 0 and 1 == 0:
            raise MountError("Unable to set bootable flag to %sp%d" % (loopdev, (bootdevnum + 1)))

        #Ensure all data is flushed to disk before doing syslinux install
        runner.quiet('sync')

        fullpathsyslinux = fs_related.find_binary_path("extlinux")
        rc = runner.show([fullpathsyslinux, "-i", "%s/boot/extlinux" % self._instroot])
        if rc != 0:
            raise MountError("Unable to install syslinux bootloader to %sp%d" % (loopdev, (bootdevnum + 1)))

 def do_install_disk(self, disk, disk_name, cr, workdir, oe_builddir,
                     bootimg_dir, kernel_dir, native_sysroot):
     """
     Called after all partitions have been prepared and assembled into a
     disk image.  This provides a hook to allow finalization of a
     disk image e.g. to write an MBR to it.
     """
     msger.debug("SourcePlugin: do_install_disk: disk: %s" % disk_name)