def _create_subvolume(self, src, dst):
     src_volume = os.path.dirname(src)
     shell.call('mkdir -p %s' % os.path.dirname(dst))
     shell.call('btrfs subvolume snapshot %s %s' % (src_volume, dst))
     src_file_name = os.path.basename(src)
     dst_path = os.path.join(dst, src_file_name)
     return dst_path

 def apply_iptables_save_doc(content):
     f = linux.write_to_temp_file(content)
     try:
         logger.debug('apply iptables:\n %s' % content)
         shell.call('/sbin/iptables-restore < %s' % f)
     finally:
         os.remove(f)

    def _delete_eips(self, eips):
        delete_eip_cmd = '''
PUB_ODEV={{pub_odev}}
NS_NAME="{{ns_name}}"

exit_on_error() {
    if [ $? -ne 0 ]; then
        echo "error on line $1"
        exit 1
    fi
}

ip netns | grep $NS_NAME > /dev/null
if [ $? -eq 0 ]; then
   ip netns delete $NS_NAME
   exit_on_error $LINENO
fi

ip link | grep $PUB_ODEV > /dev/null
if [ $? -eq 0 ]; then
    ip link del $PUB_ODEV
    exit_on_error $LINENO
fi

exit 0
'''
        for eip in eips:
            ctx = {
                "pub_odev": "%s_o" % eip.vip.replace(".", ""),
                "ns_name": "%s_%s" % (eip.publicBridgeName, eip.vip.replace(".", "_"))
            }
            ctx.update(eip.__dict__)
            tmpt = Template(delete_eip_cmd)
            cmd = tmpt.render(ctx)
            shell.call(cmd)

 def purge_snapshots(self, req):
     cmd = jsonobject.loads(req[http.REQUEST_BODY])
     vpath = self._normalize_install_path(cmd.volumePath)
     shell.call('rbd snap purge %s' % vpath)
     rsp = AgentResponse()
     self._set_capacity_to_response(rsp)
     return jsonobject.dumps(rsp)

    def commit_to_imagestore(self, cmd, req):
        fpath = cmd.primaryStorageInstallPath

        # Synchronize cached writes for 'fpath'
        shell.call('/bin/sync ' + fpath)

        # Add the image to registry
        cmdstr = '%s -json  -callbackurl %s -taskid %s -imageUuid %s add -file %s' % (self.ZSTORE_CLI_PATH, req[http.REQUEST_HEADER].get(http.CALLBACK_URI),
                req[http.REQUEST_HEADER].get(http.TASK_UUID), cmd.imageUuid, fpath)

        logger.debug('adding %s to local image store' % fpath)
        shell.call(cmdstr)
        logger.debug('%s added to local image store' % fpath)

        name, imageid = self._get_image_reference(fpath)

        rsp = kvmagent.AgentResponse()
        rsp.backupStorageInstallPath = self._build_install_path(name, imageid)
        rsp.size = linux.qcow2_size_and_actual_size(cmd.primaryStorageInstallPath)[0]

        # we need to sum all the disk size within the chain ...
        chain = linux.qcow2_get_file_chain(cmd.primaryStorageInstallPath)
        rsp.actualSize = sum([ linux.qcow2_size_and_actual_size(f)[1] for f in chain ])

        return jsonobject.dumps(rsp)

    def create(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])

        path = self._normalize_install_path(cmd.installPath)
        rsp = CreateEmptyVolumeRsp()

        call_string = None
        if isXsky():
            # do NOT round to MB
            call_string = 'rbd create --size %dB --image-format 2 %s' % (cmd.size, path)
            rsp.size = cmd.size
        else:
            size_M = sizeunit.Byte.toMegaByte(cmd.size) + 1
            call_string = 'rbd create --size %s --image-format 2 %s' % (size_M, path)
            rsp.size = cmd.size + sizeunit.MegaByte.toByte(1)

        if cmd.shareable:
            call_string = call_string + " --image-shared"

        skip_cmd = "rbd info %s ||" % path if cmd.skipIfExisting else ""
        shell.call(skip_cmd + call_string)


        self._set_capacity_to_response(rsp)
        return jsonobject.dumps(rsp)

    def _make_conf_path(self, bridge_name):
        folder = os.path.join(self.DNSMASQ_CONF_FOLDER, bridge_name)
        if not os.path.exists(folder):
            shell.call('mkdir -p %s' % folder)

        # the conf is created at the initializing time
        conf = os.path.join(folder, 'dnsmasq.conf')

        dhcp = os.path.join(folder, 'hosts.dhcp')
        if not os.path.exists(dhcp):
            shell.call('touch %s' % dhcp)

        dns = os.path.join(folder, 'hosts.dns')
        if not os.path.exists(dns):
            shell.call('touch %s' % dns)

        option = os.path.join(folder, 'hosts.option')
        if not os.path.exists(option):
            shell.call('touch %s' % option)

        log = os.path.join(folder, 'dnsmasq.log')
        if not os.path.exists(log):
            shell.call('touch %s' % log)

        return conf, dhcp, dns, option, log

def kill_vm(maxAttempts, mountPaths=None, isFileSystem=None):
    zstack_uuid_pattern = "'[0-9a-f]{8}[0-9a-f]{4}[1-5][0-9a-f]{3}[89ab][0-9a-f]{3}[0-9a-f]{12}'"

    virsh_list = shell.call("virsh list --all")
    logger.debug("virsh_list:\n" + virsh_list)

    vm_in_process_uuid_list = shell.call("virsh list | egrep -o " + zstack_uuid_pattern + " | sort | uniq")
    logger.debug('vm_in_process_uuid_list:\n' + vm_in_process_uuid_list)

    # kill vm's qemu process
    vm_pids_dict = {}
    for vm_uuid in vm_in_process_uuid_list.split('\n'):
        vm_uuid = vm_uuid.strip(' \t\n\r')
        if not vm_uuid:
            continue

        if mountPaths and isFileSystem is not None \
                and not is_need_kill(vm_uuid, mountPaths, isFileSystem):
            continue

        vm_pid = shell.call("ps aux | grep qemu-kvm | grep -v grep | awk '/%s/{print $2}'" % vm_uuid)
        vm_pid = vm_pid.strip(' \t\n\r')
        vm_pids_dict[vm_uuid] = vm_pid

    for vm_uuid, vm_pid in vm_pids_dict.items():
        kill = shell.ShellCmd('kill -9 %s' % vm_pid)
        kill(False)
        if kill.return_code == 0:
            logger.warn('kill the vm[uuid:%s, pid:%s] because we lost connection to the storage.'
                        'failed to read the heartbeat file %s times' % (vm_uuid, vm_pid, maxAttempts))
        else:
            logger.warn('failed to kill the vm[uuid:%s, pid:%s] %s' % (vm_uuid, vm_pid, kill.stderr))

    return vm_pids_dict

    def download_image(self, req):
        #TODO: report percentage to mgmt server
        def percentage_callback(percent, url):
            logger.debug('Downloading %s ... %s%%' % (url, percent))

        def use_wget(url, name, workdir, timeout):
            return linux.wget(url, workdir=workdir, rename=name, timeout=timeout, interval=2, callback=percentage_callback, callback_data=url)

        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = DownloadResponse()
        supported_schemes = [self.URL_HTTP, self.URL_HTTPS, self.URL_FILE]
        if cmd.urlScheme not in supported_schemes:
            rsp.success = False
            rsp.error = 'unsupported url scheme[%s], SimpleSftpBackupStorage only supports %s' % (cmd.urlScheme, supported_schemes)
            return jsonobject.dumps(rsp)

        path = os.path.dirname(cmd.installPath)
        if not os.path.exists(path):
            os.makedirs(path, 0777)
        image_name = os.path.basename(cmd.installPath)
        install_path = cmd.installPath

        timeout = cmd.timeout if cmd.timeout else 7200
        if cmd.urlScheme in [self.URL_HTTP, self.URL_HTTPS]:
            try:
                ret = use_wget(cmd.url, image_name, path, timeout)
                if ret != 0:
                    rsp.success = False
                    rsp.error = 'http/https download failed, [wget -O %s %s] returns value %s' % (image_name, cmd.url, ret)
                    return jsonobject.dumps(rsp)
            except linux.LinuxError as e:
                traceback.format_exc()
                rsp.success = False
                rsp.error = str(e)
                return jsonobject.dumps(rsp)
        elif cmd.urlScheme == self.URL_FILE:
            src_path = cmd.url.lstrip('file:')
            src_path = os.path.normpath(src_path)
            if not os.path.isfile(src_path):
                raise Exception('cannot find the file[%s]' % src_path)
            logger.debug("src_path is: %s" % src_path)
            shell.call('yes | cp %s %s' % (src_path, install_path))



        os.chmod(cmd.installPath, stat.S_IRUSR + stat.S_IRGRP + stat.S_IROTH)


        image_format =  bash_o("qemu-img info %s | grep -w '^file format' | awk '{print $3}'" % install_path).strip('\n')
        size = os.path.getsize(install_path)
        md5sum = 'not calculated'
        logger.debug('successfully downloaded %s to %s' % (cmd.url, install_path))
        (total, avail) = self.get_capacity()
        rsp.md5Sum = md5sum
        rsp.actualSize = size
        rsp.size = linux.qcow2_virtualsize(install_path)
        rsp.totalCapacity = total
        rsp.availableCapacity = avail
        rsp.format = image_format
        return jsonobject.dumps(rsp)

    def unprotect_snapshot(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        spath = self._normalize_install_path(cmd.snapshotPath)

        shell.call('rbd snap unprotect %s' % spath)

        return jsonobject.dumps(AgentResponse())

 def download_from_imagestore(self, cachedir, host, backupStorageInstallPath, primaryStorageInstallPath):
     name, imageid = self._parse_image_reference(backupStorageInstallPath)
     cmdstr = '%s -url %s:%s -cachedir %s pull -installpath %s %s:%s' % (self.ZSTORE_CLI_PATH, host, self.ZSTORE_DEF_PORT, cachedir, primaryStorageInstallPath, name, imageid)
     logger.debug('pulling %s:%s from image store' % (name, imageid))
     shell.call(cmdstr)
     logger.debug('%s:%s pulled to local cache' % (name, imageid))
     return

    def remove_dhcp_entry(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = RemoveDhcpEntryRsp()
        try:
            for e in cmd.dhcpEntries:
                net_dev = shell.call("ifconfig|grep -i %s|awk '{print $1}'" % e.vrNicMac)
                net_dev = net_dev.strip('\t\r\n ')
                mac2 = e.mac.replace(':', '')
                shell.call("sed -i '/%s/d' %s; \
                        sed -i '/^$/d' %s; \
                        sed -i '/%s/d' %s; \
                        sed -i '/^$/d' %s; \
                        sed -i '/%s/d' %s; \
                        sed -i '/^$/d' %s; \
                        dhcp_release %s %s %s"\
                        % (e.mac, self.HOST_DHCP_FILE, \
                        self.HOST_DHCP_FILE, \
                        mac2, self.HOST_OPTION_FILE, \
                        self.HOST_OPTION_FILE, \
                        e.ip, self.HOST_DNS_FILE, \
                        self.HOST_DNS_FILE, \
                        net_dev, e.ip, e.mac))

        except virtualrouter.VirtualRouterError as e:
            logger.warn(linux.get_exception_stacktrace())
            rsp.error = str(e)
            rsp.success = False

        return jsonobject.dumps(rsp)

    def check_origin_data_exist(self, root_vol=True):
        if root_vol:
            vol_installPath = self.former_root_vol_install_path
            vol_uuid = self.former_root_vol_uuid
            vol_size = self.former_root_vol_size
        else:
            vol_installPath = self.former_data_vol_installPath
            vol_uuid = self.former_data_volume_uuid
            vol_size = self.former_data_volume_size
        if self.origin_ps.type == 'Ceph':
            ceph_mon_ip = self.origin_ps.mons[0].monAddr
            self.chk_cmd = 'sshpass -p password ssh -o LogLevel=quiet -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no [email protected]%s "rbd info %s --format=json"' \
                        % (ceph_mon_ip, vol_installPath.split('ceph://')[-1])
            data_info = shell.call(self.chk_cmd)
            origin_meta = jsonobject.loads(data_info)
#             assert origin_meta.name == vol_uuid
            if root_vol:
                assert origin_meta.size == vol_size
            else:
                assert origin_meta.size >= vol_size
            assert 'rbd_data' in origin_meta.block_name_prefix
            ps_trash = ps_ops.get_trash_on_primary_storage(self.origin_ps.uuid).storageTrashSpecs
            trash_install_path_list = [trsh.installPath for trsh in ps_trash]
            assert vol_installPath in trash_install_path_list
        else:
            nfs_ip, mount_path = self.origin_ps.url.split(':')
            self.chk_cmd = 'sshpass -p password ssh -o LogLevel=quiet -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
                            [email protected]%s "qemu-img info %s"' % (nfs_ip, mount_path + vol_installPath.split(self.origin_ps.uuid)[-1])
            data_info = shell.call(self.chk_cmd)
            assert str(vol_size) in data_info
            ps_trash = ps_ops.get_trash_on_primary_storage(self.origin_ps.uuid).storageTrashSpecs
            trash_install_path_list = [trsh.installPath for trsh in ps_trash]
            assert '/'.join(vol_installPath.split('/')[:8]) in trash_install_path_list
        return self

def shell_cmd_thread(shell_cmd, ignore_exception = False):
    try:
        shell.call(shell_cmd)
    except Exception as e:
        if not ignore_exception:
            node_exception.append(sys.exc_info())
            raise e

    def _deploy_db(self, keep_db = False):
        if not keep_db:
            extra_opts = '--drop'
        else:
            extra_opts = '--keep-db'

        if not self.need_deploy_db:
            return
        ssh.make_ssh_no_password(self.db_server, 'root', \
                self.db_server_root_password)

        if not self.db_admin_password:
            cmd = 'zstack-ctl install_db --debug --host=%s --login-password=zstack.mysql.password' % self.db_server
        else:
            cmd = 'zstack-ctl install_db --debug --host=%s \
                    --login-password=%s' \
                    % (self.db_server, \
                    self.db_admin_password)

        print('installing db ...')
        shell.call(cmd)

        cmd = 'zstack-ctl deploydb %s --host=%s' % (extra_opts, self.db_server)
        if self.db_admin_password:
            cmd = '%s --root-password=%s' % (cmd, self.db_admin_password )
        else:
            cmd = '%s --root-password=zstack.mysql.password' % cmd

        if self.db_password:
            cmd = '%s --zstack-password=%s' % (cmd, self.db_password)

        print('deploying db ...')
        shell.call(cmd)

    def _upgrade_local_zstack(self):
        cmd = 'WEBSITE=localhost bash %s -f %s -u -r %s' % \
                (self.zstack_install_script, self.zstack_pkg, \
                self.install_path)

        shell.call(cmd)
        self._extra_deployment()

    def execute_plan_without_deploy_test_agent(self):
        if os.environ.get('ZSTACK_ALREADY_INSTALLED') != "yes":
            try:
                self._stop_nodes()
                shell.call('zstack-ctl kairosdb --stop')
                shell.call('zstack-ctl cassandra --stop')
            except:
                pass
    
            self._install_local_zstack()
            self._deploy_db()
            self._deploy_rabbitmq()
            self._install_management_nodes()
            self._set_extra_node_config()
        else:
            self._change_node_ip()
            self._install_management_nodes()
            self._set_extra_node_config()
        try:
            with open('/root/.bashrc', 'a+') as bashrc:
                bashrc.write('export ZSTACK_ALREADY_INSTALLED=yes\n')
        except:
            pass

        self._start_multi_nodes(restart=True)
        #NOTE: Only one key pair will take effect
        self._copy_sshkey_from_node()
        self._enable_jacoco_dump()

    def download(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        pool, image_name = self._parse_install_path(cmd.installPath)
        tmp_image_name = 'tmp-%s' % image_name

        lichbd_file = os.path.join(pool, image_name)
        tmp_lichbd_file = os.path.join(pool, tmp_image_name)

        lichbd.lichbd_mkpool(os.path.dirname(lichbd_file))
        shell.call('set -o pipefail; wget --no-check-certificate -q -O - %s | lichbd import - %s -p lichbd' % (cmd.url, tmp_lichbd_file))

        @rollbackable
        def _1():
            if lichbd.lichbd_file_exist(tmp_lichbd_file):
                lichbd.lichbd_rm(tmp_lichbd_file)
            lichbd.lichbd_rm(lichbd_file)
        _1()

        qemu_img = lichbd.lichbd_get_qemu_img_path()
        file_format = shell.call("set -o pipefail;%s info rbd:%s/%s 2>/dev/null | grep 'file format' | cut -d ':' -f 2" % (qemu_img, pool, tmp_image_name))
        file_format = file_format.strip()
        if file_format not in ['qcow2', 'raw']:
            raise Exception('unknown image format: %s' % file_format)

        lichbd.lichbd_mv(lichbd_file, tmp_lichbd_file)
        size = lichbd.lichbd_file_size(lichbd_file)
        rsp = DownloadRsp()
        rsp.size = size
        self._set_capacity_to_response(rsp)
        return jsonobject.dumps(rsp)

    def init(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])

        o = shell.call('ceph mon_status')
        mon_status = jsonobject.loads(o)
        fsid = mon_status.monmap.fsid_

        existing_pools = shell.call('ceph osd lspools')
        for pool in cmd.pools:
            if pool.predefined and pool.name not in existing_pools:
                raise Exception('cannot find pool[%s] in the ceph cluster, you must create it manually' % pool.name)
            elif pool.name not in existing_pools:
                shell.call('ceph osd pool create %s 128' % pool.name)

        rsp = InitRsp()

        if cmd.nocephx is False:
            o = shell.call("ceph -f json auth get-or-create client.zstack mon 'allow r' osd 'allow *' 2>/dev/null").strip(
                ' \n\r\t')
            o = jsonobject.loads(o)
            rsp.userKey = o[0].key_

        rsp.fsid = fsid
        self._set_capacity_to_response(rsp)

        return jsonobject.dumps(rsp)

    def _deploy_rabbitmq(self):
        ssh.make_ssh_no_password(self.rabbitmq_server, 'root', \
                self.rabbitmq_server_root_passwd)

        cmd = "zstack-ctl install_rabbitmq --host=%s" % self.rabbitmq_server

        print('deploying rabbitmq ...')
        shell.call(cmd)