/*
 * Concatenate two pkthdr mbuf chains.
 */
void
m_catpkt(struct mbuf *m, struct mbuf *n)
{

	M_ASSERTPKTHDR(m);
	M_ASSERTPKTHDR(n);

	m->m_pkthdr.len += n->m_pkthdr.len;
	m_demote(n, 1, 0);

	m_cat(m, n);
}

/*
 * "Move" mbuf pkthdr from "from" to "to".
 * "from" must have M_PKTHDR set, and "to" must be empty.
 */
void
m_move_pkthdr(struct mbuf *to, struct mbuf *from)
{

#if 0
	/* see below for why these are not enabled */
	M_ASSERTPKTHDR(to);
	/* Note: with MAC, this may not be a good assertion. */
	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
	    ("m_move_pkthdr: to has tags"));
#endif
#ifdef MAC
	/*
	 * XXXMAC: It could be this should also occur for non-MAC?
	 */
	if (to->m_flags & M_PKTHDR)
		m_tag_delete_chain(to, NULL);
#endif
	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
	if ((to->m_flags & M_EXT) == 0)
		to->m_data = to->m_pktdat;
	to->m_pkthdr = from->m_pkthdr;		/* especially tags */
	SLIST_INIT(&from->m_pkthdr.tags);	/* purge tags from src */
	from->m_flags &= ~M_PKTHDR;
}

static int
do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
	struct adapter *sc = iq->adapter;
	struct cpl_iscsi_data *cpl =  mtod(m, struct cpl_iscsi_data *);
	u_int tid = GET_TID(cpl);
	struct toepcb *toep = lookup_tid(sc, tid);
	struct icl_cxgbei_pdu *icp = toep->ulpcb2;

	M_ASSERTPKTHDR(m);

	/* Must already have received the header (but not the data). */
	MPASS(icp != NULL);
	MPASS(icp->pdu_flags == SBUF_ULP_FLAG_HDR_RCVD);
	MPASS(icp->ip.ip_data_mbuf == NULL);
	MPASS(icp->ip.ip_data_len == 0);

	m_adj(m, sizeof(*cpl));

	icp->pdu_flags |= SBUF_ULP_FLAG_DATA_RCVD;
	icp->ip.ip_data_mbuf = m;
	icp->ip.ip_data_len = m->m_pkthdr.len;

#if 0
	CTR4(KTR_CXGBE, "%s: tid %u, cpl->len dlen %u, m->m_len dlen %u",
	    __func__, tid, ntohs(cpl->len), m->m_len);
#endif

	return (0);
}

static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
	struct adapter *sc = iq->adapter;
	struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
	u_int tid = GET_TID(cpl);
	struct toepcb *toep = lookup_tid(sc, tid);
	struct icl_pdu *ip;
	struct icl_cxgbei_pdu *icp;

	M_ASSERTPKTHDR(m);

	ip = icl_cxgbei_new_pdu(M_NOWAIT);
	if (ip == NULL)
		CXGBE_UNIMPLEMENTED("PDU allocation failure");
	icp = ip_to_icp(ip);
	bcopy(mtod(m, caddr_t) + sizeof(*cpl), icp->ip.ip_bhs, sizeof(struct
	    iscsi_bhs));
	icp->pdu_seq = ntohl(cpl->seq);
	icp->pdu_flags = SBUF_ULP_FLAG_HDR_RCVD;

	/* This is the start of a new PDU.  There should be no old state. */
	MPASS(toep->ulpcb2 == NULL);
	toep->ulpcb2 = icp;

#if 0
	CTR4(KTR_CXGBE, "%s: tid %u, cpl->len hlen %u, m->m_len hlen %u",
	    __func__, tid, ntohs(cpl->len), m->m_len);
#endif

	m_freem(m);
	return (0);
}

/*
 * Duplicate "from"'s mbuf pkthdr in "to".
 * "from" must have M_PKTHDR set, and "to" must be empty.
 * In particular, this does a deep copy of the packet tags.
 */
int
m_dup_pkthdr(struct mbuf *to, const struct mbuf *from, int how)
{

#if 0
	/*
	 * The mbuf allocator only initializes the pkthdr
	 * when the mbuf is allocated with m_gethdr(). Many users
	 * (e.g. m_copy*, m_prepend) use m_get() and then
	 * smash the pkthdr as needed causing these
	 * assertions to trip.  For now just disable them.
	 */
	M_ASSERTPKTHDR(to);
	/* Note: with MAC, this may not be a good assertion. */
	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
#endif
	MBUF_CHECKSLEEP(how);
#ifdef MAC
	if (to->m_flags & M_PKTHDR)
		m_tag_delete_chain(to, NULL);
#endif
	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
	if ((to->m_flags & M_EXT) == 0)
		to->m_data = to->m_pktdat;
	to->m_pkthdr = from->m_pkthdr;
	SLIST_INIT(&to->m_pkthdr.tags);
	return (m_tag_copy_chain(to, from, how));
}

static int
discoutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
    struct route *ro)
{
	u_int32_t af;

	M_ASSERTPKTHDR(m);

	/* BPF writes need to be handled specially. */
	if (dst->sa_family == AF_UNSPEC)
		bcopy(dst->sa_data, &af, sizeof(af));
	else
		af = dst->sa_family;

	if (bpf_peers_present(ifp->if_bpf))
		bpf_mtap2(ifp->if_bpf, &af, sizeof(af), m);

	m->m_pkthdr.rcvif = ifp;

	ifp->if_opackets++;
	ifp->if_obytes += m->m_pkthdr.len;

	m_freem(m);
	return (0);
}

/*
 * IP output.  The packet in mbuf chain m contains a skeletal IP
 * header (with len, off, ttl, proto, tos, src, dst).
 * The mbuf chain containing the packet will be freed.
 * The mbuf opt, if present, will not be freed.
 * If route ro is present and has ro_rt initialized, route lookup would be
 * skipped and ro->ro_rt would be used. If ro is present but ro->ro_rt is NULL,
 * then result of route lookup is stored in ro->ro_rt.
 *
 * In the IP forwarding case, the packet will arrive with options already
 * inserted, so must have a NULL opt pointer.
 */
int
ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
    struct ip_moptions *imo, struct inpcb *inp)
{
	struct rm_priotracker in_ifa_tracker;
	struct ip *ip;
	struct ifnet *ifp = NULL;	/* keep compiler happy */
	struct mbuf *m0;
	int hlen = sizeof (struct ip);
	int mtu;
	int error = 0;
	struct sockaddr_in *dst;
	const struct sockaddr_in *gw;
	struct in_ifaddr *ia;
	int isbroadcast;
	uint16_t ip_len, ip_off;
	struct route iproute;
	struct rtentry *rte;	/* cache for ro->ro_rt */
	uint32_t fibnum;
	int have_ia_ref;
#ifdef IPSEC
	int no_route_but_check_spd = 0;
#endif
	M_ASSERTPKTHDR(m);

	if (inp != NULL) {
		INP_LOCK_ASSERT(inp);
		M_SETFIB(m, inp->inp_inc.inc_fibnum);
		if ((flags & IP_NODEFAULTFLOWID) == 0) {
			m->m_pkthdr.flowid = inp->inp_flowid;
			M_HASHTYPE_SET(m, inp->inp_flowtype);
		}
	}

	if (ro == NULL) {
		ro = &iproute;
		bzero(ro, sizeof (*ro));
	}

#ifdef FLOWTABLE
	if (ro->ro_rt == NULL)
		(void )flowtable_lookup(AF_INET, m, ro);
#endif

	if (opt) {
		int len = 0;
		m = ip_insertoptions(m, opt, &len);
		if (len != 0)
			hlen = len; /* ip->ip_hl is updated above */
	}
	ip = mtod(m, struct ip *);
	ip_len = ntohs(ip->ip_len);
	ip_off = ntohs(ip->ip_off);

	if ((flags & (IP_FORWARDING|IP_RAWOUTPUT)) == 0) {
		ip->ip_v = IPVERSION;
		ip->ip_hl = hlen >> 2;
		ip_fillid(ip);
		IPSTAT_INC(ips_localout);
	} else {

int
bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
    bus_dma_segment_t *segs, int *nsegs, int flags)
{
	int error = 0;

	M_ASSERTPKTHDR(m0);

	*nsegs = 0;

	if (m0->m_pkthdr.len <= dmat->maxsize) {
		int first = 1;
		vm_offset_t lastaddr = 0;
		struct mbuf *m;

		for (m = m0; m != NULL && error == 0; m = m->m_next) {
			if (m->m_len > 0) {
				error = bus_dmamap_load_buffer(dmat,
				    segs, m->m_data, m->m_len, NULL,
				    flags, &lastaddr, nsegs, first);
				first = 0;
			}
		}
		++*nsegs;
	} else {
		error = EINVAL;
	}

	return (error);
}

static int
do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
	struct adapter *sc = iq->adapter;
	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
	struct cpl_iscsi_data *cpl =  mtod(m, struct cpl_iscsi_data *);
	u_int tid = GET_TID(cpl);
	struct toepcb *toep = lookup_tid(sc, tid);
	struct icl_cxgbei_pdu *icp = toep->ulpcb2;

	M_ASSERTPKTHDR(m);
	MPASS(m->m_pkthdr.len == be16toh(cpl->len) + sizeof(*cpl));

	/* Must already have received the header (but not the data). */
	MPASS(icp != NULL);
	MPASS(icp->icp_flags == ICPF_RX_HDR);
	MPASS(icp->ip.ip_data_mbuf == NULL);


	m_adj(m, sizeof(*cpl));
	MPASS(icp->ip.ip_data_len == m->m_pkthdr.len);

	icp->icp_flags |= ICPF_RX_FLBUF;
	icp->ip.ip_data_mbuf = m;
	counter_u64_add(ci->fl_pdus, 1);
	counter_u64_add(ci->fl_bytes, m->m_pkthdr.len);

#if 0
	CTR3(KTR_CXGBE, "%s: tid %u, cpl->len %u", __func__, tid,
	    be16toh(cpl->len));
#endif

	return (0);
}

static void
ether_nh_input(struct rte_mbuf *m)
{

	M_ASSERTPKTHDR(m);
	ether_input_internal(NULL, m);
}

/*
 * netisr CPU affinity lookup routine for use by protocols.
 */
struct mbuf *
rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
{

	M_ASSERTPKTHDR(m);
	*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
	return (m);
}

int
rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
{

	M_ASSERTPKTHDR(m);

	return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
	    bucket_id));
}

void
m_demote_pkthdr(struct mbuf *m)
{

	M_ASSERTPKTHDR(m);

	m_tag_delete_chain(m, NULL);
	m->m_flags &= ~M_PKTHDR;
	bzero(&m->m_pkthdr, sizeof(struct pkthdr));
}

/*
 * natmintr: interrupt
 *
 * Note: we expect a socket pointer in rcvif rather than an interface
 * pointer.  We can get the interface pointer from the so's PCB if we really
 * need it.
 */
void
natmintr(struct mbuf *m)
{
	struct socket *so;
	struct natmpcb *npcb;

#ifdef DIAGNOSTIC
	M_ASSERTPKTHDR(m);
#endif

	NATM_LOCK();
	npcb = (struct natmpcb *)m->m_pkthdr.rcvif;	/* XXX: overloaded */
	so = npcb->npcb_socket;

	npcb->npcb_inq--;

	if (npcb->npcb_flags & NPCB_DRAIN) {
		if (npcb->npcb_inq == 0)
			free(npcb, M_PCB);			/* done! */
		NATM_UNLOCK();
		m_freem(m);
		return;
	}

	if (npcb->npcb_flags & NPCB_FREE) {
		NATM_UNLOCK();
		m_freem(m);					/* drop */
		return;
	}

#ifdef NEED_TO_RESTORE_IFP
	m->m_pkthdr.rcvif = npcb->npcb_ifp;
#else
#ifdef DIAGNOSTIC
	m->m_pkthdr.rcvif = NULL;	/* null it out to be safe */
#endif
#endif

	if (sbspace(&so->so_rcv) > m->m_pkthdr.len) {
#ifdef NATM_STAT
		natm_sookcnt++;
		natm_sookbytes += m->m_pkthdr.len;
#endif
		sbappendrecord(&so->so_rcv, m);
		sorwakeup(so);
		NATM_UNLOCK();
	} else {
#ifdef NATM_STAT
		natm_sodropcnt++;
		natm_sodropbytes += m->m_pkthdr.len;
#endif
		NATM_UNLOCK();
		m_freem(m);
	}
}

static int
ifoff_socket_check_deliver(struct socket *so, struct label *solabel,
    struct mbuf *m, struct label *mlabel)
{

	M_ASSERTPKTHDR(m);
	if (m->m_pkthdr.rcvif != NULL)
		return (ifnet_check_incoming(m->m_pkthdr.rcvif, 0));

	return (0);
}

static int
discoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
    struct rtentry *rt)
{
	M_ASSERTPKTHDR(m);
	/* BPF write needs to be handled specially */
	if (dst->sa_family == AF_UNSPEC) {
		dst->sa_family = *(mtod(m, int *));
		m->m_len -= sizeof(int);
		m->m_pkthdr.len -= sizeof(int);
		m->m_data += sizeof(int);
	}

void
ip_dn_packet_free(struct dn_pkt *pkt)
{
	struct netmsg_packet *nmp;
	struct mbuf *m = pkt->dn_m;

	M_ASSERTPKTHDR(m);
	KASSERT(m->m_pkthdr.fw_flags & DUMMYNET_MBUF_TAGGED,
		("mbuf is not tagged for dummynet!"));

	nmp = &m->m_hdr.mh_netmsg;
	netmsg_init(&nmp->base, NULL, &netisr_apanic_rport,
			0, ip_dn_freepkt_dispatch);
	nmp->nm_packet = m;

	lwkt_sendmsg(pkt->msgport, &nmp->base.lmsg);
}

void
ip_dn_queue(struct mbuf *m)
{
	struct netmsg_packet *nmp;
	lwkt_port_t port;

	M_ASSERTPKTHDR(m);
	KASSERT(m->m_pkthdr.fw_flags & DUMMYNET_MBUF_TAGGED,
		("mbuf is not tagged for dummynet!"));

	nmp = &m->m_hdr.mh_netmsg;
	netmsg_init(&nmp->base, NULL, &netisr_apanic_rport,
			0, ip_dn_dispatch);
	nmp->nm_packet = m;

	port = netisr_cpuport(ip_dn_cpu);
	lwkt_sendmsg(port, &nmp->base.lmsg);
}

static inline void
rqdrop_locked(struct mbufq *q, int plen)
{
	struct mbuf *m;

	while (plen > 0) {
		m = mbufq_dequeue(q);

		/* Too many credits. */
		MPASS(m != NULL);
		M_ASSERTPKTHDR(m);

		/* Partial credits. */
		MPASS(plen >= m->m_pkthdr.len);

		plen -= m->m_pkthdr.len;
		m_freem(m);
	}
}

static int
looutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
	 struct rtentry *rt)
{
	M_ASSERTPKTHDR(m);

	if (rt && rt->rt_flags & (RTF_REJECT|RTF_BLACKHOLE)) {
		m_freem(m);
		return (rt->rt_flags & RTF_BLACKHOLE ? 0 :
		        rt->rt_flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH);
	}

	ifp->if_opackets++;
	ifp->if_obytes += m->m_pkthdr.len;
#if 1	/* XXX */
	switch (dst->sa_family) {
	case AF_INET:
	case AF_INET6:
	case AF_IPX:
	case AF_NS:
		break;
	default:
		kprintf("looutput: af=%d unexpected\n", dst->sa_family);
		m_freem(m);
		return (EAFNOSUPPORT);
	}
#endif

	if (ifp->if_capenable & IFCAP_RXCSUM) {
		int csum_flags = 0;

		if (m->m_pkthdr.csum_flags & CSUM_IP)
			csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID);
		if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA)
			csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);

		m->m_pkthdr.csum_flags |= csum_flags;
		if (csum_flags & CSUM_DATA_VALID)
			m->m_pkthdr.csum_data = 0xffff;
	}
	return (if_simloop(ifp, m, dst->sa_family, 0));
}