static int
tmpfs_inactive(struct vop_inactive_args *v)
{
	struct vnode *vp = v->a_vp;

	struct tmpfs_node *node;

	MPASS(VOP_ISLOCKED(vp));

	node = VP_TO_TMPFS_NODE(vp);

	if (node->tn_links == 0)
		vrecycle(vp);

	return 0;
}

void
_sx_xunlock(struct sx *sx, const char *file, int line)
{

	MPASS(curthread != NULL);
	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
	    ("sx_xunlock() of destroyed sx @ %s:%d", file, line));
	_sx_assert(sx, SA_XLOCKED, file, line);
	curthread->td_locks--;
	WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
	LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
	    line);
	if (!sx_recursed(sx))
		LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_XUNLOCK_RELEASE, sx);
	__sx_xunlock(sx, curthread, file, line);
}

static int
tmpfs_create(struct vop_create_args *v)
{
	struct vnode *dvp = v->a_dvp;
	struct vnode **vpp = v->a_vpp;
	struct componentname *cnp = v->a_cnp;
	struct vattr *vap = v->a_vap;
	int error;

	MPASS(vap->va_type == VREG || vap->va_type == VSOCK);

	error = tmpfs_alloc_file(dvp, vpp, vap, cnp, NULL);
	if (error == 0 && (cnp->cn_flags & MAKEENTRY) != 0)
		cache_enter(dvp, *vpp, cnp);
	return (error);
}

static void
dircache_entry_free(struct pefs_dircache_entry *pde)
{
	MPASS(pde != NULL);

	PEFSDEBUG("dircache_entry_free: %s -> %s\n",
	    pde->pde_name, pde->pde_encname);
	pefs_key_release(pde->pde_tkey.ptk_key);
	LIST_REMOVE(pde, pde_dir_entry);
	mtx_lock(&dircache_mtx);
	LIST_REMOVE(pde, pde_hash_entry);
	LIST_REMOVE(pde, pde_enchash_entry);
	dircache_entries--;
	mtx_unlock(&dircache_mtx);
	uma_zfree(dircache_entry_zone, pde);
}

int
nandfs_start_cleaner(struct nandfs_device *fsdev)
{
	int error;

	MPASS(fsdev->nd_cleaner == NULL);

	fsdev->nd_cleaner_exit = 0;

	error = kthread_add((void(*)(void *))nandfs_cleaner, fsdev, NULL,
	    &fsdev->nd_cleaner, 0, 0, "nandfs_cleaner");
	if (error)
		printf("nandfs: could not start cleaner: %d\n", error);

	return (error);
}

void
_rw_wlock(struct rwlock *rw, const char *file, int line)
{

	if (SCHEDULER_STOPPED())
		return;
	MPASS(curthread != NULL);
	KASSERT(rw->rw_lock != RW_DESTROYED,
	    ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
	WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
	    line, NULL);
	__rw_wlock(rw, curthread, file, line);
	LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
	WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
	curthread->td_locks++;
}

int
bmap_dirty_meta(struct nandfs_node *node, nandfs_lbn_t lblk, int force)
{
	struct nandfs_indir a[NIADDR+1], *ap;
#ifdef DEBUG
	nandfs_daddr_t daddr;
#endif
	struct buf *bp;
	int error;
	int num, *nump;

	DPRINTF(BMAP, ("%s: node %p lblk=%jx\n", __func__, node, lblk));

	ap = a;
	nump = #

	error = bmap_getlbns(node, lblk, ap, nump);
	if (error)
		return (error);

	/*
	 * Direct block, nothing to do
	 */
	if (num == 0)
		return (0);

	DPRINTF(BMAP, ("%s: node %p reading blocks\n", __func__, node));

	for (bp = NULL, ++ap; --num; ap++) {
		error = nandfs_bread_meta(node, ap->in_lbn, NOCRED, 0, &bp);
		if (error) {
			brelse(bp);
			return (error);
		}

#ifdef DEBUG
		daddr = ((nandfs_daddr_t *)bp->b_data)[ap->in_off];
		MPASS(daddr != 0 || node->nn_ino == 3);
#endif

		error = nandfs_dirty_buf_meta(bp, force);
		if (error)
			return (error);
	}

	return (0);
}

static void
update_tx_sched(void *context, int pending)
{
	int i, j, mode, rateunit, ratemode, maxrate, pktsize, rc;
	struct port_info *pi;
	struct tx_cl_rl_params *tc;
	struct adapter *sc = context;
	const int n = sc->chip_params->nsched_cls;

	mtx_lock(&sc->tc_lock);
	for_each_port(sc, i) {
		pi = sc->port[i];
		tc = &pi->sched_params->cl_rl[0];
		for (j = 0; j < n; j++, tc++) {
			MPASS(mtx_owned(&sc->tc_lock));
			if ((tc->flags & TX_CLRL_REFRESH) == 0)
				continue;

			mode = tc->mode;
			rateunit = tc->rateunit;
			ratemode = tc->ratemode;
			maxrate = tc->maxrate;
			pktsize = tc->pktsize;
			mtx_unlock(&sc->tc_lock);

			if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
			    "t4utxs") != 0) {
				mtx_lock(&sc->tc_lock);
				continue;
			}
			rc = t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED,
			    FW_SCHED_PARAMS_LEVEL_CL_RL, mode, rateunit,
			    ratemode, pi->tx_chan, j, 0, maxrate, 0, pktsize,
			    1);
			end_synchronized_op(sc, 0);

			mtx_lock(&sc->tc_lock);
			if (rc != 0) {
				tc->flags |= TX_CLRL_ERROR;
			} else if (tc->mode == mode &&
			    tc->rateunit == rateunit &&
			    tc->maxrate == maxrate &&
			    tc->pktsize == tc->pktsize) {
				tc->flags &= ~(TX_CLRL_REFRESH | TX_CLRL_ERROR);
			}
		}
	}

static void
dircache_expire(struct pefs_dircache *pd)
{
	struct pefs_dircache_entry *pde;

	pd->pd_gen = 0;
	if (LIST_EMPTY(DIRCACHE_STALEHEAD(pd))) {
		pd->pd_flags ^= PD_SWAPEDHEADS;
	} else while (!LIST_EMPTY(DIRCACHE_ACTIVEHEAD(pd))) {
		pde = LIST_FIRST(DIRCACHE_ACTIVEHEAD(pd));
		pde->pde_gen = 0;
		LIST_REMOVE(pde, pde_dir_entry);
		LIST_INSERT_HEAD(DIRCACHE_STALEHEAD(pd), pde, pde_dir_entry);
		PEFSDEBUG("dircache_expire: active entry: %p\n", pde);
	}
	MPASS(LIST_EMPTY(DIRCACHE_ACTIVEHEAD(pd)));
}

void
_rw_wunlock(struct rwlock *rw, const char *file, int line)
{

	if (SCHEDULER_STOPPED())
		return;
	MPASS(curthread != NULL);
	KASSERT(rw->rw_lock != RW_DESTROYED,
	    ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
	_rw_assert(rw, RA_WLOCKED, file, line);
	curthread->td_locks--;
	WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
	LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
	    line);
	if (!rw_recursed(rw))
		LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_WUNLOCK_RELEASE, rw);
	__rw_wunlock(rw, curthread, file, line);
}

int
_sx_slock(struct sx *sx, int opts, const char *file, int line)
{
	int error = 0;

	MPASS(curthread != NULL);
	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
	    ("sx_slock() of destroyed sx @ %s:%d", file, line));
	WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL);
	error = __sx_slock(sx, opts, file, line);
	if (!error) {
		LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
		WITNESS_LOCK(&sx->lock_object, 0, file, line);
		curthread->td_locks++;
	}

	return (error);
}

void
_sx_downgrade(struct sx *sx, const char *file, int line)
{

	_sx_assert(sx, SX_XLOCKED, file, line);
	mtx_lock(sx->sx_lock);
	MPASS(sx->sx_cnt == -1);

	WITNESS_DOWNGRADE(&sx->sx_object, 0, file, line);

	sx->sx_cnt = 1;
	sx->sx_xholder = NULL;
        if (sx->sx_shrd_wcnt > 0)
                cv_broadcast(&sx->sx_shrd_cv);

	LOCK_LOG_LOCK("XDOWNGRADE", &sx->sx_object, 0, 0, file, line);

	mtx_unlock(sx->sx_lock);
}

static unsigned int
cxgbei_task_reserve_ttt(struct icl_conn *ic, void **prv, union ctl_io *io,
				unsigned int *ttt)
{
	struct icl_cxgbei_conn *icc = ic_to_icc(ic);
	struct toepcb *toep = icc->toep;
	struct adapter *sc = td_adapter(toep->td);
	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
	struct cxgbei_task_data *tdata = NULL;
	int xferlen, err = -1;
	struct cxgbei_sgl *sge = NULL;

	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);

	xferlen = (io->scsiio.kern_data_len - io->scsiio.ext_data_filled);
	tdata = (struct cxgbei_task_data *)*prv;
	if ((xferlen == 0) || (tdata == NULL))
		goto out;
	if (xferlen < DDP_THRESHOLD)
		goto out;
	tdata->nsge = cxgbei_map_sg_tgt(tdata->sgl, io);
	if (tdata->nsge == 0) {
		CTR1(KTR_CXGBE, "%s: map_sg failed", __func__);
		return 0;
	}
	sge = tdata->sgl;

	tdata->sc_ddp_tag = *ttt;
	if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, tdata->sc_ddp_tag)) {
		err = t4_sk_ddp_tag_reserve(ci, icc, xferlen, sge,
		    tdata->nsge, &tdata->sc_ddp_tag);
	} else {
		CTR2(KTR_CXGBE, "%s: sc_ddp_tag:0x%x not usable",
				__func__, tdata->sc_ddp_tag);
	}
out:
	if (err < 0)
		tdata->sc_ddp_tag =
			cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *ttt);
	return tdata->sc_ddp_tag;
}

int
mp_ring_alloc(struct mp_ring **pr, int size, void *cookie, ring_drain_t drain,
    ring_can_drain_t can_drain, struct malloc_type *mt, int flags)
{
	struct mp_ring *r;

	/* All idx are 16b so size can be 65536 at most */
	if (pr == NULL || size < 2 || size > 65536 || drain == NULL ||
	    can_drain == NULL)
		return (EINVAL);
	*pr = NULL;
	flags &= M_NOWAIT | M_WAITOK;
	MPASS(flags != 0);

	r = malloc(__offsetof(struct mp_ring, items[size]), mt, flags | M_ZERO);
	if (r == NULL)
		return (ENOMEM);
	r->size = size;
	r->cookie = cookie;
	r->mt = mt;
	r->drain = drain;
	r->can_drain = can_drain;
	r->enqueues = counter_u64_alloc(flags);
	r->drops = counter_u64_alloc(flags);
	r->starts = counter_u64_alloc(flags);
	r->stalls = counter_u64_alloc(flags);
	r->restarts = counter_u64_alloc(flags);
	r->abdications = counter_u64_alloc(flags);
	if (r->enqueues == NULL || r->drops == NULL || r->starts == NULL ||
	    r->stalls == NULL || r->restarts == NULL ||
	    r->abdications == NULL) {
		mp_ring_free(r);
		return (ENOMEM);
	}

	*pr = r;
	return (0);
}

int
nandfs_get_entry_block(struct nandfs_mdt *mdt, struct nandfs_node *node,
    struct nandfs_alloc_request *req, uint32_t *entry, int create)
{
	struct buf *bp;
	nandfs_lbn_t blocknr;
	int	error;

	/* Find buffer number for given entry */
	nandfs_mdt_trans(mdt, req->entrynum, &blocknr, entry);
	DPRINTF(ALLOC, ("%s: ino %#jx entrynum:%#jx block:%#jx entry:%x\n",
	    __func__, (uintmax_t)node->nn_ino, (uintmax_t)req->entrynum,
	    (uintmax_t)blocknr, *entry));

	/* Read entry block or create if 'create' parameter is not zero */
	bp = NULL;

	if (blocknr < node->nn_inode.i_blocks)
		error = nandfs_bread(node, blocknr, NOCRED, 0, &bp);
	else if (create)
		error = nandfs_bcreate(node, blocknr, NOCRED, 0, &bp);
	else
		error = E2BIG;

	if (error) {
		DPRINTF(ALLOC, ("%s: ino %#jx block %#jx entry %x error %d\n",
		    __func__, (uintmax_t)node->nn_ino, (uintmax_t)blocknr,
		    *entry, error));
		if (bp)
			brelse(bp);
		return (error);
	}

	MPASS(nandfs_vblk_get(bp) != 0 || node->nn_ino == NANDFS_DAT_INO);

	req->bp_entry = bp;
	return (0);
}

void
mp_ring_check_drainage(struct mp_ring *r, int budget)
{
	union ring_state os, ns;

	os.state = r->state;
	if (os.flags != STALLED || os.pidx_head != os.pidx_tail ||
	    r->can_drain(r) == 0)
		return;

	MPASS(os.cidx != os.pidx_tail);	/* implied by STALLED */
	ns.state = os.state;
	ns.flags = BUSY;

	/*
	 * The acquire style atomic guarantees visibility of items associated
	 * with the pidx that we read here.
	 */
	if (!atomic_cmpset_acq_64(&r->state, os.state, ns.state))
		return;

	drain_ring(r, ns, os.flags, budget);
}

int
do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
	struct adapter *sc = iq->adapter;
	const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
	unsigned int tid = GET_TID(cpl);
	struct toepcb *toep;
#ifdef INVARIANTS
	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif

	KASSERT(opcode == CPL_SET_TCB_RPL,
	    ("%s: unexpected opcode 0x%x", __func__, opcode));
	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
	MPASS(iq != &sc->sge.fwq);

	toep = lookup_tid(sc, tid);
	if (toep->ulp_mode == ULP_MODE_TCPDDP) {
		handle_ddp_tcb_rpl(toep, cpl);
		return (0);
	}

	/*
	 * TOM and/or other ULPs don't request replies for CPL_SET_TCB or
	 * CPL_SET_TCB_FIELD requests.  This can easily change and when it does
	 * the dispatch code will go here.
	 */
#ifdef INVARIANTS
	panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__,
	    tid, iq);
#else
	log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
	    __func__, tid, iq);
#endif

	return (0);
}

static int
t4iov_attach_child(device_t dev)
{
	struct t4iov_softc *sc;
#ifdef PCI_IOV
	nvlist_t *pf_schema, *vf_schema;
#endif
	device_t pdev;
	int error;

	sc = device_get_softc(dev);
	MPASS(!sc->sc_attached);

	/*
	 * PF0-3 are associated with a specific port on the NIC (PF0
	 * with port 0, etc.).  Ask the PF4 driver for the device for
	 * this function's associated port to determine if the port is
	 * present.
	 */
	error = T4_READ_PORT_DEVICE(sc->sc_main, pci_get_function(dev), &pdev);
	if (error)
		return (0);

#ifdef PCI_IOV
	pf_schema = pci_iov_schema_alloc_node();
	vf_schema = pci_iov_schema_alloc_node();
	error = pci_iov_attach_name(dev, pf_schema, vf_schema, "%s",
	    device_get_nameunit(pdev));
	if (error) {
		device_printf(dev, "Failed to initialize SR-IOV: %d\n", error);
		return (0);
	}
#endif

	sc->sc_attached = true;
	return (0);
}