def test_subnet_update_set_alloc_pool_cache_concurrency(self):
        subnet = {"cidr": "192.168.10.0/24"}
        subnet_db = db_api.subnet_create(self.context, **subnet)
        self.context.session.flush()

        # establish second session
        old_session = self.context.session
        self.context._session = None

        subnet_to_delete = db_api.subnet_find(
            self.context, id=subnet_db.id, scope=db_api.ONE)
        db_api.subnet_delete(self.context, subnet_to_delete)
        self.context.session.flush()

        # restore first session
        self.context._session = old_session

        try:
            db_api.subnet_update_set_alloc_pool_cache(
                self.context, subnet_db, {"foo": "bar"})
            self.context.session.flush()
        except exc.StaleDataError as e:
            self.fail("Did not expect StaleDataError exception: {0}".format(e))
        self.assertEqual(subnet_db["_allocation_pool_cache"],
                         "{\"foo\": \"bar\"}")

    def test_subnets_get_marked_as_full_retroactively(self):
        models = []
        models.append(self._create_models(
            "0.0.0.0/31",
            4,
            netaddr.IPNetwork("0.0.0.0/31").ipv6().last))
        models.append(self._create_models(
            "1.1.1.0/31",
            4,
            netaddr.IPNetwork("1.1.1.0/31").ipv6().last))
        models.append(self._create_models(
            "2.2.2.0/30",
            4,
            netaddr.IPNetwork("2.2.2.0/30").ipv6().first))

        with self._fixtures(models) as net:
            ipaddress = []
            self.ipam.allocate_ip_address(self.context, ipaddress,
                                          net['id'], 0, 0)
            self.assertEqual(ipaddress[0].version, 4)
            self.assertEqual(ipaddress[0].address_readable, "2.2.2.1")

            with self.context.session.begin():
                subnets = db_api.subnet_find(self.context, None, None, None,
                                             False).all()
                self.assertEqual(len(subnets), 3)

                full_subnets = [s for s in subnets
                                if s.next_auto_assign_ip == -1]
                self.assertEqual(len(full_subnets), 2)
                available_subnets = list(set(full_subnets) ^ set(subnets))
                self.assertEqual(len(available_subnets), 1)
                self.assertEqual(available_subnets[0].cidr, "2.2.2.0/30")
                self.assertEqual(available_subnets[0].next_auto_assign_ip,
                                 netaddr.IPAddress("2.2.2.2").ipv6().value)

def get_subnet(context, id, fields=None):
    """Retrieve a subnet.

    : param context: neutron api request context
    : param id: UUID representing the subnet to fetch.
    : param fields: a list of strings that are valid keys in a
        subnet dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.
    """
    LOG.info("get_subnet %s for tenant %s with fields %s" %
             (id, context.tenant_id, fields))
    subnet = db_api.subnet_find(context=context, limit=None,
                                page_reverse=False, sorts=['id'],
                                marker_obj=None, fields=None, id=id,
                                join_dns=True, join_routes=True,
                                scope=db_api.ONE)
    if not subnet:
        raise n_exc.SubnetNotFound(subnet_id=id)

    cache = subnet.get("_allocation_pool_cache")
    if not cache:
        new_cache = subnet.allocation_pools
        db_api.subnet_update_set_alloc_pool_cache(context, subnet, new_cache)
    return v._make_subnet_dict(subnet)

def split_and_validate_requested_subnets(context, net_id, segment_id,
                                         fixed_ips):
    subnets = []
    ip_addresses = {}
    for fixed_ip in fixed_ips:
        subnet_id = fixed_ip.get("subnet_id")
        ip_address = fixed_ip.get("ip_address")
        if not subnet_id:
            raise exceptions.BadRequest(resource="fixed_ips",
                                        msg="subnet_id required")
        if ip_address:
            ip_addresses[ip_address] = subnet_id
        else:
            subnets.append(subnet_id)

    subnets = ip_addresses.values() + subnets

    sub_models = db_api.subnet_find(context, id=subnets, scope=db_api.ALL)
    if len(sub_models) == 0:
        raise exceptions.NotFound(msg="Requested subnet(s) not found")

    for s in sub_models:
        if s["network_id"] != net_id:
            raise exceptions.InvalidInput(
                error_message="Requested subnet doesn't belong to requested "
                              "network")

        if segment_id and segment_id != s["segment_id"]:
            raise q_exc.AmbiguousNetworkId(net_id=net_id)

    return ip_addresses, subnets

def get_subnets(context, limit=None, page_reverse=False, sorts=None,
                marker=None, filters=None, fields=None):
    """Retrieve a list of subnets.

    The contents of the list depends on the identity of the user
    making the request (as indicated by the context) as well as any
    filters.
    : param context: neutron api request context
    : param filters: a dictionary with keys that are valid keys for
        a subnet as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictiontary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.
    : param fields: a list of strings that are valid keys in a
        subnet dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.
    """
    LOG.info("get_subnets for tenant %s with filters %s fields %s" %
             (context.tenant_id, filters, fields))
    filters = filters or {}
    subnets = db_api.subnet_find(context, limit=limit,
                                 page_reverse=page_reverse, sorts=sorts,
                                 marker_obj=marker,
                                 join_dns=True, join_routes=True, **filters)
    for subnet in subnets:
        cache = subnet.get("_allocation_pool_cache")
        if not cache:
            db_api.subnet_update_set_alloc_pool_cache(
                context, subnet, subnet.allocation_pools)
    return v._make_subnets_list(subnets, fields=fields)

def create_ip_policy(context, ip_policy):
    LOG.info("create_ip_policy for tenant %s" % context.tenant_id)

    ipp = ip_policy["ip_policy"]

    if not ipp.get("exclude"):
        raise exceptions.BadRequest(resource="ip_policy",
                                    msg="Empty ip_policy.exclude regions")

    ipp["exclude"] = netaddr.IPSet(ipp["exclude"])
    network_id = ipp.get("network_id")
    subnet_id = ipp.get("subnet_id")

    model = None
    if subnet_id:
        model = db_api.subnet_find(context, id=subnet_id, scope=db_api.ONE)
        if not model:
            raise exceptions.SubnetNotFound(id=subnet_id)
    elif network_id:
        model = db_api.network_find(context, id=network_id,
                                    scope=db_api.ONE)
        if not model:
            raise exceptions.NetworkNotFound(id=network_id)
    else:
        raise exceptions.BadRequest(
            resource="ip_policy",
            msg="network_id or subnet_id unspecified")

    if model["ip_policy"]:
        raise quark_exceptions.IPPolicyAlreadyExists(
            id=model["ip_policy"]["id"], n_id=model["id"])
    model["ip_policy"] = db_api.ip_policy_create(context, **ipp)
    return v._make_ip_policy_dict(model["ip_policy"])

def get_subnet(context, id, fields=None):
    """Retrieve a subnet.

    : param context: neutron api request context
    : param id: UUID representing the subnet to fetch.
    : param fields: a list of strings that are valid keys in a
        subnet dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.
    """
    LOG.info("get_subnet %s for tenant %s with fields %s" %
             (id, context.tenant_id, fields))
    subnet = db_api.subnet_find(context, None, None, None, False, id=id,
                                join_dns=True, join_routes=True,
                                scope=db_api.ONE)
    if not subnet:
        raise exceptions.SubnetNotFound(subnet_id=id)

    # Check the network_id against the strategies
    net_id = subnet["network_id"]
    net_id = STRATEGY.get_parent_network(net_id)
    subnet["network_id"] = net_id

    cache = subnet.get("_allocation_pool_cache")
    if not cache:
        new_cache = subnet.allocation_pools
        db_api.subnet_update_set_alloc_pool_cache(context, subnet, new_cache)
    return v._make_subnet_dict(subnet)

def create_route(context, route):
    LOG.info("create_route for tenant %s" % context.tenant_id)
    route = route["route"]
    for key in ["gateway", "cidr", "subnet_id"]:
        if key not in route:
            raise exceptions.BadRequest(resource="routes", msg="%s is required" % key)

    subnet_id = route["subnet_id"]
    with context.session.begin():
        subnet = db_api.subnet_find(context, id=subnet_id, scope=db_api.ONE)
        if not subnet:
            raise exceptions.SubnetNotFound(subnet_id=subnet_id)
        policies = db_models.IPPolicy.get_ip_policy_cidrs(subnet)
        alloc_pools = allocation_pool.AllocationPools(subnet["cidr"], policies=policies)
        alloc_pools.validate_gateway_excluded(route["gateway"])

        # TODO(anyone): May want to denormalize the cidr values into columns
        #               to achieve single db lookup on conflict check
        route_cidr = netaddr.IPNetwork(route["cidr"])
        subnet_routes = db_api.route_find(context, subnet_id=subnet_id, scope=db_api.ALL)

        quota.QUOTAS.limit_check(context, context.tenant_id, routes_per_subnet=len(subnet_routes) + 1)

        for sub_route in subnet_routes:
            sub_route_cidr = netaddr.IPNetwork(sub_route["cidr"])
            if sub_route_cidr.value == DEFAULT_ROUTE.value:
                continue
            if route_cidr in sub_route_cidr or sub_route_cidr in route_cidr:
                raise quark_exceptions.RouteConflict(route_id=sub_route["id"], cidr=str(route_cidr))
        new_route = db_api.route_create(context, **route)
    return v._make_route_dict(new_route)

def _validate_subnet_cidr(context, network_id, new_subnet_cidr):
    """Validate the CIDR for a subnet.

    Verifies the specified CIDR does not overlap with the ones defined
    for the other subnets specified for this network, or with any other
    CIDR if overlapping IPs are disabled.

    """
    if neutron_cfg.cfg.CONF.allow_overlapping_ips:
        return

    new_subnet_ipset = netaddr.IPSet([new_subnet_cidr])

    # Using admin context here, in case we actually share networks later
    subnet_list = db_api.subnet_find(context.elevated(),
                                     network_id=network_id)
    for subnet in subnet_list:
        if (netaddr.IPSet([subnet.cidr]) & new_subnet_ipset):
            # don't give out details of the overlapping subnet
            err_msg = (_("Requested subnet with cidr: %(cidr)s for "
                         "network: %(network_id)s overlaps with another "
                         "subnet") %
                       {'cidr': new_subnet_cidr,
                        'network_id': network_id})
            LOG.error(_("Validation for CIDR: %(new_cidr)s failed - "
                        "overlaps with subnet %(subnet_id)s "
                        "(CIDR: %(cidr)s)"),
                      {'new_cidr': new_subnet_cidr,
                       'subnet_id': subnet.id,
                       'cidr': subnet.cidr})
            raise exceptions.InvalidInput(error_message=err_msg)

def update_ip_policy(context, id, ip_policy):
    LOG.info("update_ip_policy for tenant %s" % context.tenant_id)

    ipp = ip_policy["ip_policy"]

    with context.session.begin():
        ipp_db = db_api.ip_policy_find(context, id=id, scope=db_api.ONE)
        if not ipp_db:
            raise quark_exceptions.IPPolicyNotFound(id=id)

        ip_policy_cidrs = ipp.get("exclude")
        network_ids = ipp.get("network_ids")
        subnet_ids = ipp.get("subnet_ids")

        if subnet_ids and network_ids:
            raise exceptions.BadRequest(
                resource="ip_policy", msg="network_ids and subnet_ids specified. only one allowed"
            )

        models = []
        all_subnets = []
        if subnet_ids:
            for subnet in ipp_db["subnets"]:
                subnet["ip_policy"] = None
            subnets = db_api.subnet_find(context, id=subnet_ids, scope=db_api.ALL)
            if len(subnets) != len(subnet_ids):
                raise exceptions.SubnetNotFound(id=subnet_ids)
            if ip_policy_cidrs is not None:
                ensure_default_policy(ip_policy_cidrs, subnets)
                _validate_cidrs_fit_into_subnets(ip_policy_cidrs, subnets)
            all_subnets.extend(subnets)
            models.extend(subnets)

        if network_ids:
            for network in ipp_db["networks"]:
                network["ip_policy"] = None
            nets = db_api.network_find(context, id=network_ids, scope=db_api.ALL)
            if len(nets) != len(network_ids):
                raise exceptions.NetworkNotFound(net_id=network_ids)
            subnets = [subnet for net in nets for subnet in net.get("subnets", [])]
            if ip_policy_cidrs is not None:
                ensure_default_policy(ip_policy_cidrs, subnets)
                _validate_cidrs_fit_into_subnets(ip_policy_cidrs, subnets)
            all_subnets.extend(subnets)
            models.extend(nets)

        if not subnet_ids and not network_ids and ip_policy_cidrs is not None:
            ensure_default_policy(ip_policy_cidrs, ipp_db["subnets"])
            _validate_cidrs_fit_into_subnets(ip_policy_cidrs, ipp_db["subnets"])

        for model in models:
            if model["ip_policy"]:
                raise quark_exceptions.IPPolicyAlreadyExists(id=model["ip_policy"]["id"], n_id=model["id"])
            model["ip_policy"] = ipp_db

        if ip_policy_cidrs:
            _validate_policy_with_routes(context, ip_policy_cidrs, all_subnets)
        ipp_db = db_api.ip_policy_update(context, ipp_db, **ipp)
    return v._make_ip_policy_dict(ipp_db)

def get_subnets_cidr_set(context, network_ids):
    ipset = netaddr.IPSet()
    subnets = db_api.subnet_find(context, network_id=network_ids,
                                 shared=[False])
    for subnet in subnets:
        net = netaddr.IPNetwork(subnet["cidr"])
        ipset.add(net)
    return ipset

def update_subnet(context, id, subnet):
    """Update values of a subnet.

    : param context: neutron api request context
    : param id: UUID representing the subnet to update.
    : param subnet: dictionary with keys indicating fields to update.
        valid keys are those that have a value of True for 'allow_put'
        as listed in the RESOURCE_ATTRIBUTE_MAP object in
        neutron/api/v2/attributes.py.
    """
    LOG.info("update_subnet %s for tenant %s" %
             (id, context.tenant_id))

    with context.session.begin():
        subnet_db = db_api.subnet_find(context, id=id, scope=db_api.ONE)
        if not subnet_db:
            raise exceptions.SubnetNotFound(id=id)

        s = subnet["subnet"]

        dns_ips = s.pop("dns_nameservers", [])
        host_routes = s.pop("host_routes", [])
        gateway_ip = s.pop("gateway_ip", None)

        if gateway_ip:
            default_route = None
            for route in host_routes:
                netaddr_route = netaddr.IPNetwork(route["destination"])
                if netaddr_route.value == routes.DEFAULT_ROUTE.value:
                    default_route = route
                    break
            if default_route is None:
                route_model = db_api.route_find(
                    context, cidr=str(routes.DEFAULT_ROUTE), subnet_id=id,
                    scope=db_api.ONE)
                if route_model:
                    db_api.route_update(context, route_model,
                                        gateway=gateway_ip)
                else:
                    db_api.route_create(context,
                                        cidr=str(routes.DEFAULT_ROUTE),
                                        gateway=gateway_ip, subnet_id=id)

        if dns_ips:
            subnet_db["dns_nameservers"] = []
        for dns_ip in dns_ips:
            subnet_db["dns_nameservers"].append(db_api.dns_create(
                context,
                ip=netaddr.IPAddress(dns_ip)))

        if host_routes:
            subnet_db["routes"] = []
        for route in host_routes:
            subnet_db["routes"].append(db_api.route_create(
                context, cidr=route["destination"], gateway=route["nexthop"]))

        subnet = db_api.subnet_update(context, subnet_db, **s)
    return v._make_subnet_dict(subnet)

    def attempt_to_reallocate_ip(self, context, net_id, port_id, reuse_after,
                                 version=None, ip_address=None,
                                 segment_id=None, subnets=None):
        version = version or [4, 6]
        elevated = context.elevated()

        # We never want to take the chance of an infinite loop here. Instead,
        # we'll clean up multiple bad IPs if we find them (assuming something
        # is really wrong)

        #TODO(mdietz & mpath): Perhaps remove, select for update might quash
        for times in xrange(3):
            with context.session.begin(subtransactions=True):

                sub_ids = []
                if subnets:
                    sub_ids = subnets
                else:
                    if segment_id:
                        subnets = db_api.subnet_find(elevated,
                                                     network_id=net_id,
                                                     segment_id=segment_id)
                        sub_ids = [s["id"] for s in subnets]
                        if not sub_ids:
                            raise exceptions.IpAddressGenerationFailure(
                                net_id=net_id)

                ip_kwargs = {
                    "network_id": net_id, "reuse_after": reuse_after,
                    "deallocated": True, "scope": db_api.ONE,
                    "ip_address": ip_address, "lock_mode": True,
                    "version": version, "order_by": "address"}

                if sub_ids:
                    ip_kwargs["subnet_id"] = sub_ids

                address = db_api.ip_address_find(elevated, **ip_kwargs)

                if address:
                    #NOTE(mdietz): We should always be in the CIDR but we've
                    #              also said that before :-/
                    if address.get("subnet"):
                        cidr = netaddr.IPNetwork(address["subnet"]["cidr"])
                        addr = netaddr.IPAddress(int(address["address"]),
                                                 version=int(cidr.version))
                        if addr in cidr:
                            updated_address = db_api.ip_address_update(
                                elevated, address, deallocated=False,
                                deallocated_at=None,
                                allocated_at=timeutils.utcnow())
                            return [updated_address]
                        else:
                            # Make sure we never find it again
                            context.session.delete(address)
                            continue
                break
        return []

 def _create_ip_address(self, ip_address, ip_version, subnet_cidr, net_id):
     with self.context.session.begin():
         subnet = db_api.subnet_find(self.context, None, False, None, None,
                                     cidr=subnet_cidr).all()
         ip = dict(subnet_id=subnet[0].id,
                   network_id=net_id,
                   version=ip_version,
                   address=netaddr.IPAddress(ip_address))
         db_api.ip_address_create(self.context, **ip)

 def _create_ip_address(self, ip_address, ip_version, subnet_cidr, net_id):
     with self.context.session.begin():
         subnet = db_api.subnet_find(context=self.context, limit=None,
                                     page_reverse=False, sorts=['id'],
                                     marker_obj=None,
                                     cidr=subnet_cidr).all()
         ip = dict(subnet_id=subnet[0].id,
                   network_id=net_id,
                   version=ip_version,
                   address=netaddr.IPAddress(ip_address))
         db_api.ip_address_create(self.context, **ip)

def delete_subnet(context, id):
    """Delete a subnet.

    : param context: neutron api request context
    : param id: UUID representing the subnet to delete.
    """
    LOG.info("delete_subnet %s for tenant %s" % (id, context.tenant_id))
    subnet = db_api.subnet_find(context, id=id, scope=db_api.ONE)
    if not subnet:
        raise exceptions.SubnetNotFound(subnet_id=id)
    _delete_subnet(context, subnet)

def create_ip_policy(context, ip_policy):
    LOG.info("create_ip_policy for tenant %s" % context.tenant_id)

    ipp = ip_policy['ip_policy']

    if not ipp.get("exclude"):
        raise exceptions.BadRequest(resource="ip_policy",
                                    msg="Empty ip_policy.exclude")

    ip_policy_cidrs = ipp.get("exclude", [])
    network_ids = ipp.get("network_ids")
    subnet_ids = ipp.get("subnet_ids")

    if subnet_ids and network_ids:
        raise exceptions.BadRequest(
            resource="ip_policy",
            msg="network_ids and subnet_ids specified. only one allowed")

    if not subnet_ids and not network_ids:
        raise exceptions.BadRequest(
            resource="ip_policy",
            msg="network_ids or subnet_ids not specified")

    with context.session.begin():
        models = []
        if subnet_ids:
            subnets = db_api.subnet_find(
                context, id=subnet_ids, scope=db_api.ALL)
            if not subnets:
                raise exceptions.SubnetNotFound(id=subnet_ids)
            if ip_policy_cidrs:
                _validate_cidrs_fit_into_subnets(ip_policy_cidrs, subnets)
            models.extend(subnets)

        if network_ids:
            nets = db_api.network_find(
                context, id=network_ids, scope=db_api.ALL)
            if not nets:
                raise exceptions.NetworkNotFound(net_id=network_ids)
            subnets = [subnet for net in nets
                       for subnet in net.get("subnets", [])]
            if ip_policy_cidrs and subnets:
                _validate_cidrs_fit_into_subnets(ip_policy_cidrs, subnets)
            models.extend(nets)

        for model in models:
            if model["ip_policy"]:
                raise quark_exceptions.IPPolicyAlreadyExists(
                    id=model["ip_policy"]["id"], n_id=model["id"])
            model["ip_policy"] = db_api.ip_policy_create(context, **ipp)

    return v._make_ip_policy_dict(model["ip_policy"])

 def test_subnet_set_full(self):
     cidr4 = "0.0.0.0/30"  # 2 bits
     net4 = netaddr.IPNetwork(cidr4)
     with self._fixtures([
         self._create_models(cidr4, 4, net4[0])
     ]) as net:
         subnet = db_api.subnet_find(self.context, network_id=net['id'],
                                     scope=db_api.ALL)[0]
         with self.context.session.begin():
             updated = db_api.subnet_update_set_full(self.context, subnet)
             self.context.session.refresh(subnet)
             self.assertTrue(updated)
             self.assertEqual(subnet["next_auto_assign_ip"], -1)

def create_ip_policy(context, ip_policy):
    LOG.info("create_ip_policy for tenant %s" % context.tenant_id)

    ipp = ip_policy['ip_policy']

    if not ipp.get("exclude"):
        raise n_exc.BadRequest(resource="ip_policy",
                               msg="Empty ip_policy.exclude")

    network_ids = ipp.get("network_ids")
    subnet_ids = ipp.get("subnet_ids")

    if subnet_ids and network_ids:
        raise n_exc.BadRequest(
            resource="ip_policy",
            msg="network_ids and subnet_ids specified. only one allowed")

    if not subnet_ids and not network_ids:
        raise n_exc.BadRequest(
            resource="ip_policy",
            msg="network_ids or subnet_ids not specified")

    with context.session.begin():
        if subnet_ids:
            subnets = db_api.subnet_find(
                context, id=subnet_ids, scope=db_api.ALL)
            if not subnets:
                raise n_exc.SubnetNotFound(subnet_id=subnet_ids)
            _check_for_pre_existing_policies_in(subnets)
            ensure_default_policy(ipp["exclude"], subnets)
            _validate_cidrs_fit_into_subnets(ipp["exclude"], subnets)
            ipp.pop("subnet_ids")
            ipp["subnets"] = subnets

        if network_ids:
            nets = db_api.network_find(
                context, id=network_ids, scope=db_api.ALL)
            if not nets:
                raise n_exc.NetworkNotFound(net_id=network_ids)
            _check_for_pre_existing_policies_in(nets)
            subnets = [subnet for net in nets
                       for subnet in net.get("subnets", [])]
            ensure_default_policy(ipp["exclude"], subnets)
            _validate_cidrs_fit_into_subnets(ipp["exclude"], subnets)
            ipp.pop("network_ids")
            ipp["networks"] = nets

        ip_policy = db_api.ip_policy_create(context, **ipp)
    return v._make_ip_policy_dict(ip_policy)

def delete_subnet(context, id):
    """Delete a subnet.

    : param context: neutron api request context
    : param id: UUID representing the subnet to delete.
    """
    LOG.info("delete_subnet %s for tenant %s" % (id, context.tenant_id))
    with context.session.begin():
        subnet = db_api.subnet_find(context, id=id, scope=db_api.ONE)
        if not subnet:
            raise exceptions.SubnetNotFound(subnet_id=id)

        payload = dict(tenant_id=subnet["tenant_id"],
                       ip_block_id=subnet["id"],
                       created_at=subnet["created_at"],
                       deleted_at=timeutils.utcnow())

        _delete_subnet(context, subnet)

        n_rpc.get_notifier("network").info(context, "ip_block.delete", payload)