def get_ganeti_nodes(backend=None, bulk=False):
    nodes = []
    for backend in get_backends(backend):
        with pooled_rapi_client(backend) as client:
            nodes.append(client.GetNodes(bulk=bulk))

    return reduce(list.__add__, nodes, [])

def set_firewall_profile(vm, profile, nic):
    uuid = nic.backend_uuid
    try:
        tag = _firewall_tags[profile] % uuid
    except KeyError:
        raise ValueError("Unsopported Firewall Profile: %s" % profile)

    log.debug("Setting tag of VM %s, NIC %s, to %s", vm, nic, profile)

    with pooled_rapi_client(vm) as client:
        # Delete previous firewall tags
        old_tags = client.GetInstanceTags(vm.backend_vm_id)
        delete_tags = [(t % uuid) for t in _firewall_tags.values()
                       if (t % uuid) in old_tags]
        if delete_tags:
            client.DeleteInstanceTags(vm.backend_vm_id, delete_tags,
                                      dry_run=settings.TEST)

        if profile != "DISABLED":
            client.AddInstanceTags(vm.backend_vm_id, [tag],
                                   dry_run=settings.TEST)

        # XXX NOP ModifyInstance call to force process_net_status to run
        # on the dispatcher
        os_name = settings.GANETI_CREATEINSTANCE_KWARGS['os']
        client.ModifyInstance(vm.backend_vm_id,
                              os_name=os_name)
    return None

def connect_to_network(vm, nic):
    network = nic.network
    backend = vm.backend
    bnet, depend_jobs = ensure_network_is_active(backend, network.id)

    depends = create_job_dependencies(depend_jobs)

    nic = {'name': nic.backend_uuid,
           'network': network.backend_id,
           'ip': nic.ipv4_address}

    log.debug("Adding NIC %s to VM %s", nic, vm)

    kwargs = {
        "instance": vm.backend_vm_id,
        "nics": [("add", "-1", nic)],
        "depends": depends,
    }
    if vm.backend.use_hotplug():
        kwargs["hotplug_if_possible"] = True
    if settings.TEST:
        kwargs["dry_run"] = True

    with pooled_rapi_client(vm) as client:
        return client.ModifyInstance(**kwargs)

def get_instance_console(vm):
    # RAPI GetInstanceConsole() returns endpoints to the vnc_bind_address,
    # which is a cluster-wide setting, either 0.0.0.0 or 127.0.0.1, and pretty
    # useless (see #783).
    #
    # Until this is fixed on the Ganeti side, construct a console info reply
    # directly.
    #
    # WARNING: This assumes that VNC runs on port network_port on
    #          the instance's primary node, and is probably
    #          hypervisor-specific.
    #
    log.debug("Getting console for vm %s", vm)

    console = {}
    console['kind'] = 'vnc'

    with pooled_rapi_client(vm) as client:
        i = client.GetInstance(vm.backend_vm_id)

    if vm.backend.hypervisor == "kvm" and i['hvparams']['serial_console']:
        raise Exception("hv parameter serial_console cannot be true")
    console['host'] = i['pnode']
    console['port'] = i['network_port']

    return console

def stale_servers_in_db(D, G):
    idD = set(D.keys())
    idG = set(G.keys())

    stale = set()
    for i in idD - idG:
        if D[i] == 'BUILD':
            vm = VirtualMachine.objects.get(id=i)
            if needs_reconciliation(vm):
                with pooled_rapi_client(vm) as c:
                    try:
                        job_status = c.GetJobStatus(vm.backendjobid)['status']
                        if job_status in ('queued', 'waiting', 'running'):
                            # Server is still building in Ganeti
                            continue
                        else:
                            c.GetInstance(utils.id_to_instance_name(i))
                            # Server has just been created in Ganeti
                            continue
                    except GanetiApiError:
                        stale.add(i)
        else:
            stale.add(i)

    return stale

def _create_network(network, backend):
    """Create a network."""

    network_type = network.public and 'public' or 'private'

    tags = network.backend_tag
    if network.dhcp:
        tags.append('nfdhcpd')

    if network.public:
        conflicts_check = True
    else:
        conflicts_check = False

    try:
        bn = BackendNetwork.objects.get(network=network, backend=backend)
        mac_prefix = bn.mac_prefix
    except BackendNetwork.DoesNotExist:
        raise Exception("BackendNetwork for network '%s' in backend '%s'"
                        " does not exist" % (network.id, backend.id))

    with pooled_rapi_client(backend) as client:
        return client.CreateNetwork(network_name=network.backend_id,
                                    network=network.subnet,
                                    network6=network.subnet6,
                                    gateway=network.gateway,
                                    gateway6=network.gateway6,
                                    network_type=network_type,
                                    mac_prefix=mac_prefix,
                                    conflicts_check=conflicts_check,
                                    tags=tags)

def set_firewall_profile(vm, profile):
    try:
        tag = _firewall_tags[profile]
    except KeyError:
        raise ValueError("Unsopported Firewall Profile: %s" % profile)

    try:
        public_nic = vm.nics.filter(network__public=True)[0]
    except IndexError:
        public_nic = None

    log.debug("Setting tag of VM %s to %s", vm, profile)

    with pooled_rapi_client(vm) as client:
        # Delete all firewall tags
        for t in _firewall_tags.values():
            client.DeleteInstanceTags(vm.backend_vm_id, [t],
                                      dry_run=settings.TEST)
            if public_nic is not None:
                tag_with_name = t.replace("0", public_nic.backend_uuid)
                client.DeleteInstanceTags(vm.backend_vm_id, [tag_with_name],
                                          dry_run=settings.TEST)

        client.AddInstanceTags(vm.backend_vm_id, [tag], dry_run=settings.TEST)
        if public_nic is not None:
            tag_with_name = tag.replace("0", public_nic.backend_uuid)
            client.AddInstanceTags(vm.backend_vm_id, [tag_with_name],
                                   dry_run=settings.TEST)

        # XXX NOP ModifyInstance call to force process_net_status to run
        # on the dispatcher
        os_name = settings.GANETI_CREATEINSTANCE_KWARGS['os']
        client.ModifyInstance(vm.backend_vm_id,
                              os_name=os_name)

def update_db(vm, msg, event_time):
    """Process a notification of type 'ganeti-op-status'"""
    log.debug("Processing ganeti-op-status msg: %s", msg)

    if msg['type'] != "ganeti-op-status":
        log.error("Message is of unknown type %s.", msg['type'])
        return

    operation = msg["operation"]
    status = msg["status"]
    jobID = msg["jobId"]
    logmsg = msg["logmsg"]
    nics = msg.get("instance_nics", None)
    disks = msg.get("instance_disks", None)
    job_fields = msg.get("job_fields", {})
    result = msg.get("result", [])

    # Special case: OP_INSTANCE_CREATE with opportunistic locking may fail
    # if all Ganeti nodes are already locked. Retry the job without
    # opportunistic locking..
    if (operation == "OP_INSTANCE_CREATE" and status == "error" and
       job_fields.get("opportunistic_locking", False)):
        try:
            error_code = result[1][1]
        except IndexError:
            error_code = None
        if error_code == rapi.ECODE_TEMP_NORES:
            if vm.backendjobid != jobID:  # The job has already been retried!
                return
            # Remove extra fields
            [job_fields.pop(f, None) for f in ("OP_ID", "reason")]
            # Remove 'pnode' and 'snode' if they were set by Ganeti iallocator.
            # Ganeti will fail if both allocator and nodes are specified.
            allocator = job_fields.pop("iallocator", None)
            if allocator is not None:
                [job_fields.pop(f, None) for f in ("pnode", "snode")]
            name = job_fields.pop("name",
                                  job_fields.pop("instance_name", None))
            # Turn off opportunistic locking before retrying the job
            job_fields["opportunistic_locking"] = False
            with pooled_rapi_client(vm) as c:
                jobID = c.CreateInstance(name=name, **job_fields)
            # Update the VM fields
            vm.backendjobid = jobID
            # Update the task_job_id for commissions
            vm.task_job_id = jobID
            vm.backendjobstatus = None
            vm.save()
            log.info("Retrying failed creation of instance '%s' without"
                     " opportunistic locking. New job ID: '%s'", name, jobID)
            return

    backend_mod.process_op_status(vm, event_time, jobID,
                                  operation, status,
                                  logmsg, nics=nics,
                                  disks=disks,
                                  job_fields=job_fields)

    log.debug("Done processing ganeti-op-status msg for vm %s.",
              msg['instance'])

def connect_to_network(vm, nic):
    backend = vm.backend
    network = nic.network
    address = nic.ipv4
    network = Network.objects.select_for_update().get(id=network.id)
    bnet, created = BackendNetwork.objects.get_or_create(backend=backend,
                                                         network=network)
    depend_jobs = []
    if bnet.operstate != "ACTIVE":
        depend_jobs = create_network(network, backend, connect=True)

    depends = [[job, ["success", "error", "canceled"]] for job in depend_jobs]

    nic = {'ip': address,
           'network': network.backend_id,
           'name': nic.backend_uuid}

    log.debug("Connecting vm %s to network %s(%s)", vm, network, address)

    kwargs = {
        "nics": [('add',  "-1", nic)],
        "depends": depends,
        "dry_run": settings.TEST

    }
    if vm.backend.use_hotplug():
        kwargs["hotplug_if_possible"] = True

    with pooled_rapi_client(vm) as client:
        return client.ModifyInstance(vm.backend_vm_id, **kwargs)

def job_is_still_running(vm):
    with pooled_rapi_client(vm) as c:
        try:
            job_info = c.GetJobStatus(vm.backendjobid)
            return not (job_info["status"] in JOB_STATUS_FINALIZED)
        except GanetiApiError:
            return False

def create_instance(vm, public_nic, flavor, image):
    """`image` is a dictionary which should contain the keys:
            'backend_id', 'format' and 'metadata'

        metadata value should be a dictionary.
    """

    # Handle arguments to CreateInstance() as a dictionary,
    # initialize it based on a deployment-specific value.
    # This enables the administrator to override deployment-specific
    # arguments, such as the disk template to use, name of os provider
    # and hypervisor-specific parameters at will (see Synnefo #785, #835).
    #
    kw = vm.backend.get_create_params()
    kw['mode'] = 'create'
    kw['name'] = vm.backend_vm_id
    # Defined in settings.GANETI_CREATEINSTANCE_KWARGS

    kw['disk_template'] = flavor.disk_template
    kw['disks'] = [{"size": flavor.disk * 1024}]
    provider = flavor.disk_provider
    if provider:
        kw['disks'][0]['provider'] = provider
        kw['disks'][0]['origin'] = flavor.disk_origin

    kw['nics'] = [{"name": public_nic.backend_uuid,
                  "network": public_nic.network.backend_id,
                  "ip": public_nic.ipv4}]
    # Defined in settings.GANETI_CREATEINSTANCE_KWARGS
    # kw['os'] = settings.GANETI_OS_PROVIDER
    kw['ip_check'] = False
    kw['name_check'] = False

    # Do not specific a node explicitly, have
    # Ganeti use an iallocator instead
    #kw['pnode'] = rapi.GetNodes()[0]

    kw['dry_run'] = settings.TEST

    kw['beparams'] = {
        'auto_balance': True,
        'vcpus': flavor.cpu,
        'memory': flavor.ram}

    kw['osparams'] = {
        'config_url': vm.config_url,
        # Store image id and format to Ganeti
        'img_id': image['backend_id'],
        'img_format': image['format']}

    # Use opportunistic locking
    kw['opportunistic_locking'] = settings.GANETI_USE_OPPORTUNISTIC_LOCKING

    # Defined in settings.GANETI_CREATEINSTANCE_KWARGS
    # kw['hvparams'] = dict(serial_console=False)

    log.debug("Creating instance %s", utils.hide_pass(kw))
    with pooled_rapi_client(vm) as client:
        return client.CreateInstance(**kw)

def disconnect_from_network(vm, nic):
    op = [('remove', nic.index, {})]

    log.debug("Removing nic of VM %s, with index %s", vm, str(nic.index))

    with pooled_rapi_client(vm) as client:
        return client.ModifyInstance(vm.backend_vm_id, nics=op,
                                     hotplug=vm.backend.use_hotplug(),
                                     dry_run=settings.TEST)

def job_is_still_running(vm, job_id=None):
    with pooled_rapi_client(vm) as c:
        try:
            if job_id is None:
                job_id = vm.backendjobid
            job_info = c.GetJobStatus(job_id)
            return not (job_info["status"] in rapi.JOB_STATUS_FINALIZED)
        except rapi.GanetiApiError:
            return False

def connect_network_synced(network, backend):
    with pooled_rapi_client(backend) as client:
        for group in client.GetGroups():
            job = client.ConnectNetwork(network.backend_id, group,
                                        network.mode, network.link)
            result = wait_for_job(client, job)
            if result[0] != rapi.JOB_STATUS_SUCCESS:
                return result

    return result

def disconnect_network(network, backend, group=None):
    log.debug("Disconnecting network %s to backend %s", network, backend)

    with pooled_rapi_client(backend) as client:
        groups = [group] if group is not None else client.GetGroups()
        job_ids = []
        for group in groups:
            job_id = client.DisconnectNetwork(network.backend_id, group)
            job_ids.append(job_id)
    return job_ids

def get_networks_from_ganeti(backend):
    prefix = settings.BACKEND_PREFIX_ID + 'net-'

    networks = {}
    with pooled_rapi_client(backend) as c:
        for net in c.GetNetworks(bulk=True):
            if net['name'].startswith(prefix):
                id = utils.id_from_network_name(net['name'])
                networks[id] = net

    return networks

def get_memory_from_instances(backend):
    """ Get the memory that is used from instances.

    Get the used memory of a backend. Note: This is different for
    the real memory used, due to kvm's memory de-duplication.

    """
    with pooled_rapi_client(backend) as client:
        instances = client.GetInstances(bulk=True)
    mem = 0
    for i in instances:
        mem += i['oper_ram']
    return mem

def _create_network(network, backend):
    """Create a network."""

    tags = network.backend_tag
    subnet = None
    subnet6 = None
    gateway = None
    gateway6 = None
    for _subnet in network.subnets.all():
        if _subnet.dhcp and not "nfdhcpd" in tags:
            tags.append("nfdhcpd")
        if _subnet.ipversion == 4:
            subnet = _subnet.cidr
            gateway = _subnet.gateway
        elif _subnet.ipversion == 6:
            subnet6 = _subnet.cidr
            gateway6 = _subnet.gateway

    conflicts_check = False
    if network.public:
        tags.append('public')
        if subnet is not None:
            conflicts_check = True
    else:
        tags.append('private')

    # Use a dummy network subnet for IPv6 only networks. Currently Ganeti does
    # not support IPv6 only networks. To bypass this limitation, we create the
    # network with a dummy network subnet, and make Cyclades connect instances
    # to such networks, with address=None.
    if subnet is None:
        subnet = "10.0.0.0/29"

    try:
        bn = BackendNetwork.objects.get(network=network, backend=backend)
        mac_prefix = bn.mac_prefix
    except BackendNetwork.DoesNotExist:
        raise Exception("BackendNetwork for network '%s' in backend '%s'"
                        " does not exist" % (network.id, backend.id))

    with pooled_rapi_client(backend) as client:
        return client.CreateNetwork(network_name=network.backend_id,
                                    network=subnet,
                                    network6=subnet6,
                                    gateway=gateway,
                                    gateway6=gateway6,
                                    mac_prefix=mac_prefix,
                                    conflicts_check=conflicts_check,
                                    tags=tags)

def disconnect_from_network(vm, nic):
    op = [('remove', str(nic.index), {})]

    log.debug("Removing nic of VM %s, with index %s", vm, str(nic.index))

    kwargs = {
        "nics": op,
        "dry_run": settings.TEST

    }
    if vm.backend.use_hotplug():
        kwargs["hotplug_if_possible"] = True

    with pooled_rapi_client(vm) as client:
        return client.ModifyInstance(vm.backend_vm_id, **kwargs)

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError("Please provide a network ID")

        network = get_network(args[0])

        # Validate subnet
        if options.get('subnet'):
            validate_network_info(options)

        # Validate state
        state = options.get('state')
        if state:
            allowed = [x[0] for x in Network.OPER_STATES]
            if state not in allowed:
                msg = "Invalid state, must be one of %s" % ', '.join(allowed)
                raise CommandError(msg)

        dhcp = options.get("dhcp")
        if dhcp:
            options["dhcp"] = parse_bool(dhcp)
        drained = options.get("drained")
        if drained:
            options["drained"] = parse_bool(drained)
        fields = ('name', 'userid', 'subnet', 'gateway', 'subnet6', 'gateway6',
                  'dhcp', 'state', 'link', 'mac_prefix', 'drained')
        for field in fields:
            value = options.get(field, None)
            if value is not None:
                network.__setattr__(field, value)

        add_reserved_ips = options.get('add_reserved_ips')
        remove_reserved_ips = options.get('remove_reserved_ips')
        if add_reserved_ips or remove_reserved_ips:
            if add_reserved_ips:
                add_reserved_ips = add_reserved_ips.split(",")
            if remove_reserved_ips:
                remove_reserved_ips = remove_reserved_ips.split(",")

        for bnetwork in network.backend_networks.all():
            with pooled_rapi_client(bnetwork.backend) as c:
                c.ModifyNetwork(network=network.backend_id,
                                add_reserved_ips=add_reserved_ips,
                                remove_reserved_ips=remove_reserved_ips)

        network.save()