def get_block_devices():
    # 1. get multi path devices
    # 2. get multi path device information from raw device
    # 3. get information of other devices
    mpath_devices = []
    block_devices = []  # type: List[SharedBlockCandidateStruct]
    slave_devices = []
    cmd = shell.ShellCmd("multipath -l -v1")
    cmd(is_exception=False)
    if cmd.return_code == 0 and cmd.stdout.strip() != "":
        mpath_devices = cmd.stdout.strip().split("\n")

    for mpath_device in mpath_devices:  # type: str
        try:
            cmd = shell.ShellCmd("realpath /dev/mapper/%s | grep -E -o 'dm-.*'" % mpath_device)
            cmd(is_exception=False)
            if cmd.return_code != 0 or cmd.stdout.strip() == "":
                continue

            dm = cmd.stdout.strip()
            slaves = shell.call("ls /sys/class/block/%s/slaves/" % dm).strip().split("\n")
            if slaves is None or len(slaves) == 0:
                struct = SharedBlockCandidateStruct()
                cmd = shell.ShellCmd("udevadm info -n %s | grep dm-uuid-mpath | grep -o 'dm-uuid-mpath-\S*' | head -n 1 | awk -F '-' '{print $NF}'" % dm)
                cmd(is_exception=True)
                struct.wwids = [cmd.stdout.strip().strip("()")]
                struct.type = "mpath"
                block_devices.append(struct)
                continue

            slave_devices.extend(slaves)
            struct = get_device_info(slaves[0])
            cmd = shell.ShellCmd("udevadm info -n %s | grep dm-uuid-mpath | grep -o 'dm-uuid-mpath-\S*' | head -n 1 | awk -F '-' '{print $NF}'" % dm)
            cmd(is_exception=True)
            struct.wwids = [cmd.stdout.strip().strip("()")]
            struct.type = "mpath"
            block_devices.append(struct)
        except Exception as e:
            logger.warn(linux.get_exception_stacktrace())
            continue

    disks = shell.call("lsblk -p -o NAME,TYPE | grep disk | awk '{print $1}'").strip().split()
    for disk in disks:
        try:
            if disk.split("/")[-1] in slave_devices or is_slave_of_multipath(disk):
                continue
            d = get_device_info(disk.strip().split("/")[-1])
            if len(d.wwids) is 0:
                continue
            if get_pv_uuid_by_path("/dev/disk/by-id/%s" % d.wwids[0]) not in ("", None):
                d.type = "lvm-pv"
            block_devices.append(d)
        except Exception as e:
            logger.warn(linux.get_exception_stacktrace())
            continue

    return block_devices

 def reboot(self, timeout=60):
     self.stop(timeout=20, undefine=False)
     try:
         self.domain.createWithFlags(0)
     except libvirt.libvirtError as e:
         logger.warn(linux.get_exception_stacktrace())
         raise kvmagent.KvmError('unable to start vm[uuid:%s], %s' % (self.uuid, str(e)))

    def _wait_for_block_job(self, disk_path, abort_on_error=False,
                            wait_for_job_clean=False):
        """Wait for libvirt block job to complete.

        Libvirt may return either cur==end or an empty dict when
        the job is complete, depending on whether the job has been
        cleaned up by libvirt yet, or not.

        :returns: True if still in progress
                  False if completed
        """

        status = self.domain.blockJobInfo(disk_path, 0)
        if status == -1 and abort_on_error:
            raise kvmagent.KvmError('libvirt error while requesting blockjob info.')

        try:
            cur = status.get('cur', 0)
            end = status.get('end', 0)
        except Exception as e:
            logger.warn(linux.get_exception_stacktrace())
            return False

        if wait_for_job_clean:
            job_ended = not status
        else:
            job_ended = cur == end

        return not job_ended

    def rebase_and_merge_snapshot(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        snapshots = cmd.snapshotInstallPaths
        count = len(snapshots)
        for i in range(count):
            if i+1 < count:
                target = snapshots[i]
                backing_file = snapshots[i+1]
                linux.qcow2_rebase_no_check(backing_file, target)

        latest = snapshots[0]
        rsp = RebaseAndMergeSnapshotsResponse()
        workspace_dir = os.path.dirname(cmd.workspaceInstallPath)
        if not os.path.exists(workspace_dir):
            os.makedirs(workspace_dir)

        try:
            linux.qcow2_create_template(latest, cmd.workspaceInstallPath)
            rsp.size, rsp.actualSize = cmd.workspaceInstallPath
            self._set_capacity_to_response(cmd.uuid, rsp)
        except linux.LinuxError as e:
            logger.warn(linux.get_exception_stacktrace())
            rsp.error = str(e)
            rsp.success = False

        return jsonobject.dumps(rsp)

def main():
    usage = 'usage: python -c "from appliancevm import daemon; daemon.main()" start|stop|restart'
    if len(sys.argv) != 2 or not sys.argv[1] in ['start', 'stop', 'restart']:
        print usage
        sys.exit(1)
        
    pidfile = '/var/run/zstack/appliancevm.pid'
    dirname = os.path.dirname(pidfile)
    if not os.path.exists(dirname):
        os.makedirs(dirname, 0755)
    
    try:
        iptables.insert_single_rule_to_filter_table('-A INPUT -i eth0 -p tcp -m tcp --dport 7759 -j ACCEPT')
        cmd = sys.argv[1]
        py_process_name = 'from appliancevm import daemon'
        agentdaemon = appliancevm.ApplianceVmDaemon(pidfile, py_process_name)
        if cmd == 'start':
            agentdaemon.start()
        elif cmd == 'stop':
            agentdaemon.stop()
        elif cmd == 'restart':
            agentdaemon.restart()
        
        sys.exit(0)
    except Exception:
        logger.warning(linux.get_exception_stacktrace())
        sys.exit(1)

def main():
    usage = 'usage: python -c "from kvmagent import kdaemon; kdaemon.main()" start|stop|restart'
    if len(sys.argv) != 2 or not sys.argv[1] in ['start', 'stop', 'restart']:
        print usage
        sys.exit(1)
    
    global pidfile
    prepare_pid_dir(pidfile)

    try:
        iptc = iptables.from_iptables_save()
        iptc.add_rule('-A INPUT -p tcp -m tcp --dport 7070 -j ACCEPT')
        # open vnc ports
        iptc.add_rule('-A INPUT -p tcp -m tcp --dport 5900:6200 -j ACCEPT')
        iptc.iptable_restore()

        cmd = sys.argv[1]
        agentdaemon = kvmagent.KvmDaemon(pidfile)
        if cmd == 'start':
            logger.debug('zstack-kvmagent starts')
            agentdaemon.start()
        elif cmd == 'stop':
            logger.debug('zstack-kvmagent stops')
            agentdaemon.stop()
        elif cmd == 'restart':
            logger.debug('zstack-kvmagent restarts')
            agentdaemon.restart()
        sys.exit(0)    
    except Exception:
        logger.warning(linux.get_exception_stacktrace())
        sys.exit(1)

    def sync_eip(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = SyncEipRsp()

        def remove_eip_chain(table):
            for c in table.children:
                if c.name.startswith('eip-'):
                    c.delete()

        ipt = iptables.from_iptables_save()
        nat = ipt.get_table(ipt.NAT_TABLE_NAME)
        if nat:
            remove_eip_chain(nat)
        filter_table = ipt.get_table(ipt.FILTER_TABLE_NAME)
        if filter_table:
            remove_eip_chain(filter_table)
        ipt.iptable_restore()

        try:
            for eip in cmd.eips:
                self._create_eip(eip)
        except virtualrouter.VirtualRouterError as e:
            logger.warning(linux.get_exception_stacktrace())
            rsp.error = str(e)
            rsp.success = False

        return jsonobject.dumps(rsp)

    def create_empty_volume(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = CreateEmptyVolumeResponse()
        try:
            dirname = os.path.dirname(cmd.installUrl)
            if not os.path.exists(dirname):
                os.makedirs(dirname)
                
            linux.qcow2_create(cmd.installUrl, cmd.size)
        except Exception as e:
            logger.warn(linux.get_exception_stacktrace())
            rsp.error = 'unable to create empty volume[uuid:%s, name:%s], %s' % (cmd.uuid, cmd.name, str(e))
            rsp.success = False
            return jsonobject.dumps(rsp)
        
        meta = VolumeMeta()
        meta.account_uuid = cmd.accountUuid
        meta.hypervisor_type = cmd.hypervisorType
        meta.name = cmd.name
        meta.uuid = cmd.volumeUuid
        meta.size = cmd.size
        meta_path = self._json_meta_file_name(cmd.installUrl)
        with open(meta_path, 'w') as fd:
            fd.write(jsonobject.dumps(meta, pretty=True))

        self._set_capacity_to_response(cmd.uuid, rsp)
        logger.debug('successfully create empty volume[uuid:%s, name:%s, size:%s] at %s' % (cmd.uuid, cmd.name, cmd.size, cmd.installUrl))
        return jsonobject.dumps(rsp)

def main():
    usage = 'usage: python -c "from baremetalpxeserver import cdaemon; cdaemon.main()" start|stop|restart'
    if len(sys.argv) != 2 or not sys.argv[1] in ['start', 'stop', 'restart']:
        print usage
        sys.exit(1)

    global pidfile
    prepare_pid_dir(pidfile)

    try:
        iptc = iptables.from_iptables_save()
        iptc.add_rule('-A INPUT -p tcp -m tcp --dport 7770 -j ACCEPT')
        iptc.iptable_restore()

        cmd = sys.argv[1]
        py_process_name = 'from baremetalpxeserver import cdaemon'
        agentdaemon = pxeserveragent.PxeServerDaemon(pidfile, py_process_name)
        if cmd == 'start':
            logger.debug('zstack-baremetalpxeserver starts')
            agentdaemon.start()
        elif cmd == 'stop':
            logger.debug('zstack-baremetalpxeserver stops')
            agentdaemon.stop()
        elif cmd == 'restart':
            logger.debug('zstack-baremetalpxeserver restarts')
            agentdaemon.restart()
        sys.exit(0)
    except Exception:
        logger.warning(linux.get_exception_stacktrace())
        sys.exit(1)

def main():
    usage = 'usage: python -c "from sftpbackupstorage import sftpbackupstoragedaemon; sftpbackupstoragedaemon.main()" start|stop|restart'
    if len(sys.argv) != 2 or not sys.argv[1] in ['start', 'stop', 'restart']:
        print usage
        sys.exit(1)
    
    global pidfile
    prepare_pid_dir(pidfile)
    
    try:
        iptables.insert_single_rule_to_filter_table('-A INPUT -p tcp -m tcp --dport 7171 -j ACCEPT')
        cmd = sys.argv[1]
        py_process_name = 'from sftpbackupstorage import sftpbackupstoragedaemon'
        agentdaemon = sftpbackupstorage.SftpBackupStorageDaemon(pidfile, py_process_name)
        if cmd == 'start':
            agentdaemon.start()
        elif cmd == 'stop':
            agentdaemon.stop()
        elif cmd == 'restart':
            agentdaemon.restart()
        
        sys.exit(0)    
    except Exception:
        logger.warning(linux.get_exception_stacktrace())
        sys.exit(1)

    def remove_dhcp_entry(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = RemoveDhcpEntryRsp()
        try:
            for e in cmd.dhcpEntries:
                net_dev = shell.call("ifconfig|grep -i %s|awk '{print $1}'" % e.vrNicMac)
                net_dev = net_dev.strip('\t\r\n ')
                mac2 = e.mac.replace(':', '')
                shell.call("sed -i '/%s/d' %s; \
                        sed -i '/^$/d' %s; \
                        sed -i '/%s/d' %s; \
                        sed -i '/^$/d' %s; \
                        sed -i '/%s/d' %s; \
                        sed -i '/^$/d' %s; \
                        dhcp_release %s %s %s"\
                        % (e.mac, self.HOST_DHCP_FILE, \
                        self.HOST_DHCP_FILE, \
                        mac2, self.HOST_OPTION_FILE, \
                        self.HOST_OPTION_FILE, \
                        e.ip, self.HOST_DNS_FILE, \
                        self.HOST_DNS_FILE, \
                        net_dev, e.ip, e.mac))

        except virtualrouter.VirtualRouterError as e:
            logger.warn(linux.get_exception_stacktrace())
            rsp.error = str(e)
            rsp.success = False

        return jsonobject.dumps(rsp)

def main():
    usage = 'usage: python -c "from virtualrouter import virtualrouterdaemon; virtualrouterdaemon.main()" start|stop|restart'
    if len(sys.argv) != 2 or not sys.argv[1] in ['start', 'stop', 'restart']:
        print usage
        sys.exit(1)
        
    pidfile = '/var/run/zstack/virtualrouter.pid'
    dirname = os.path.dirname(pidfile)
    if not os.path.exists(dirname):
        os.makedirs(dirname, 0755)
    
    try:
        iptables.insert_single_rule_to_filter_table('-A INPUT -i eth0 -p tcp -m tcp --dport 7272 -j ACCEPT')
        cmd = sys.argv[1]
        agentdaemon = virtualrouter.VirutalRouterDaemon(pidfile)
        if cmd == 'start':
            agentdaemon.start()
        elif cmd == 'stop':
            agentdaemon.stop()
        elif cmd == 'restart':
            agentdaemon.restart()
        
        sys.exit(0)
    except Exception:
        logger.warning(linux.get_exception_stacktrace())
        sys.exit(1)

 def add_dhcp_entry(self, req):
     cmd = jsonobject.loads(req[http.REQUEST_BODY])
     entries = []
     gateways = []
     for e in cmd.dhcpEntries:
         entry = DhcpEntry.from_dhcp_info(e)
         entries.append(entry)
         gateways.append(entry.gateway)
         
     if cmd.rebuild:
         self._rebuild_all(entries)
     else:
         self._merge(entries)
     
     rsp = AddDhcpEntryRsp()
     try:
         if self._add_dhcp_range_if_need(gateways):
             self._restart_dnsmasq()
         else:
             self._refresh_dnsmasq()
     except virtualrouter.VirtualRouterError as e:
         logger.warn(linux.get_exception_stacktrace())
         rsp.error = str(e)
         rsp.success = False
     
     return jsonobject.dumps(rsp)

def main():
    usage = 'usage: python -c "from cephbackupstorage import cdaemon; cdaemon.main()" start|stop|restart'
    if len(sys.argv) != 2 or not sys.argv[1] in ['start', 'stop', 'restart']:
        print usage
        sys.exit(1)
    
    global pidfile
    prepare_pid_dir(pidfile)

    try:
        iptc = iptables.from_iptables_save()
        iptc.add_rule('-A INPUT -p tcp -m tcp --dport 7761 -j ACCEPT')
        iptc.iptable_restore()

        cmd = sys.argv[1]
        agentdaemon = cephagent.CephDaemon(pidfile)
        if cmd == 'start':
            logger.debug('zstack-ceph-backupstorage starts')
            agentdaemon.start()
        elif cmd == 'stop':
            logger.debug('zstack-ceph-backupstorage stops')
            agentdaemon.stop()
        elif cmd == 'restart':
            logger.debug('zstack-ceph-backupstorage restarts')
            agentdaemon.restart()
        sys.exit(0)    
    except Exception:
        logger.warning(linux.get_exception_stacktrace())
        sys.exit(1)

    def post(_):
        try:
            pool = urllib3.PoolManager(timeout=120.0, retries=urllib3.util.retry.Retry(15))
            header = {'Content-Type': 'application/json', 'Connection': 'close'}
            for k in headers.keys():
                header[k] = headers[k]

            if body is not None:
                assert isinstance(body, types.StringType)
                header['Content-Length'] = str(len(body))
                content = pool.urlopen(method, uri, headers=header, body=str(body)).data

                #(resp, content) = http_obj.request(uri, 'POST', body='%s' % body, headers=header)
            else:
                header['Content-Length'] = '0'
                #(resp, content) = http_obj.request(uri, 'POST', headers=header)
                content = pool.urlopen(method, uri, headers=header).data

            #logger.debug('post to %s, with content: %s, with header: %s' % (uri, body, header))
            pool.clear()
            ret.append(content)
            return True
        except Exception as e:
            if fail_soon:
                raise e

            logger.warn(linux.get_exception_stacktrace())
            return False

def bash_progress_1(cmd, func):
    ctx = __collect_locals_on_stack()
    cmd = bash_eval(cmd, ctx)
    logger.debug(cmd)
    p = subprocess.Popen('/bin/bash', stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
    watch_thread = WatchThread_1(func)
    try:
        watch_thread.start()
        o, e = p.communicate(cmd)
        r = p.returncode

        __BASH_DEBUG_INFO__ = ctx.get('__BASH_DEBUG_INFO__')
        if __BASH_DEBUG_INFO__ is not None:
            __BASH_DEBUG_INFO__.append({
                'cmd': cmd,
                'return_code': p.returncode,
                'stderr': e
            })

        if r != 0:
            watch_thread.stop()
            raise BashError('failed to execute bash[%s], return code: %s, stderr: %s' % (cmd, r, e))
        return r, o, None
    except Exception as ex:
        logger.debug(linux.get_exception_stacktrace())
        return None, None, ex
    finally:
        watch_thread.stop()

 def run(self):
     logger.debug("watch_thread_1: %s start" % self.__class__)
     try:
         synced = 0
         while self.keepRunning:
             time.sleep(1)
             synced = self.func(synced)
     except:
         logger.warning(linux.get_exception_stacktrace())

    def take_volume_snapshot(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = TakeSnapshotResponse()

        def makedir_if_need(new_path):
            dirname = os.path.dirname(new_path)
            if not os.path.exists(dirname):
                os.makedirs(dirname, 0755)

        def take_full_snapshot_by_qemu_img_convert(previous_install_path, install_path):
            makedir_if_need(install_path)
            linux.qcow2_create_template(previous_install_path, install_path)
            new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
            makedir_if_need(new_volume_path)
            linux.qcow2_clone(install_path, new_volume_path)
            return install_path, new_volume_path

        def take_delta_snapshot_by_qemu_img_convert(previous_install_path, install_path):
            new_volume_path = os.path.join(os.path.dirname(install_path), '{0}.qcow2'.format(uuidhelper.uuid()))
            makedir_if_need(new_volume_path)
            linux.qcow2_clone(previous_install_path, new_volume_path)
            return previous_install_path, new_volume_path

        try:
            if not cmd.vmUuid:
                if cmd.fullSnapshot:
                    rsp.snapshotInstallPath, rsp.newVolumeInstallPath = take_full_snapshot_by_qemu_img_convert(cmd.volumeInstallPath, cmd.installPath)
                else:
                    rsp.snapshotInstallPath, rsp.newVolumeInstallPath = take_delta_snapshot_by_qemu_img_convert(cmd.volumeInstallPath, cmd.installPath)

            else:
                vm = get_vm_by_uuid(cmd.vmUuid, exception_if_not_existing=False)

                if vm and vm.state != vm.VM_STATE_RUNNING and vm.state != vm.VM_STATE_SHUTDOWN:
                    raise kvmagent.KvmError('unable to take snapshot on vm[uuid:{0}] volume[id:{1}], because vm is not Running or Stopped, current state is {2}'.format(vm.uuid, cmd.deviceId, vm.state))

                if vm and vm.state == vm.VM_STATE_RUNNING:
                    rsp.snapshotInstallPath, rsp.newVolumeInstallPath = vm.take_volume_snapshot(cmd.deviceId, cmd.installPath, cmd.fullSnapshot)
                else:
                    if cmd.fullSnapshot:
                        rsp.snapshotInstallPath, rsp.newVolumeInstallPath = take_full_snapshot_by_qemu_img_convert(cmd.volumeInstallPath, cmd.installPath)
                    else:
                        rsp.snapshotInstallPath, rsp.newVolumeInstallPath = take_delta_snapshot_by_qemu_img_convert(cmd.volumeInstallPath, cmd.installPath)


                if cmd.fullSnapshot:
                    logger.debug('took full snapshot on vm[uuid:{0}] volume[id:{1}], snapshot path:{2}, new volulme path:{3}'.format(cmd.vmUuid, cmd.deviceId, rsp.snapshotInstallPath, rsp.newVolumeInstallPath))
                else:
                    logger.debug('took delta snapshot on vm[uuid:{0}] volume[id:{1}], snapshot path:{2}, new volulme path:{3}'.format(cmd.vmUuid, cmd.deviceId, rsp.snapshotInstallPath, rsp.newVolumeInstallPath))

            rsp.size = os.path.getsize(rsp.snapshotInstallPath)
        except kvmagent.KvmError as e:
            logger.warn(linux.get_exception_stacktrace())
            rsp.error = str(e)
            rsp.success = False

        return jsonobject.dumps(rsp)

 def _stop_vm(self, cmd):
     try:
         vm = get_vm_by_uuid(cmd.uuid)
     except kvmagent.KvmError as e:
         logger.debug(linux.get_exception_stacktrace())
         logger.debug('however, the stop operation is still considered as success')
         return
     
     vm.stop(timeout=cmd.timeout / 2)

 def vm_sync(self, req):
     rsp = VmSyncResponse()
     try:
         rsp.states = get_all_vm_states()
     except kvmagent.KvmError as e:
         logger.warn(linux.get_exception_stacktrace())
         rsp.error = str(e)
         rsp.success = False
         
     return jsonobject.dumps(rsp)