def test_wgan_cost(backend_default):
    """
    Set up a Wasserstein GANCost transform and make sure cost and errors are getting
    computed correctly.
    """
    be = backend_default
    cost = GANCost(func="wasserstein")
    y_data = be.iobuf(5).fill(1.)
    y_noise = be.iobuf(5).fill(2.)
    output = be.iobuf(1)
    expected = be.iobuf(1)
    delta = be.iobuf(5)

    # fprop for discriminator cost
    output[:] = cost(y_data, y_noise)
    expected[:] = be.sum(y_data - y_noise, axis=0)
    tensors_allclose(output, expected)

    # bprop for wasserstein cost
    delta[:] = cost.bprop_data(y_data)
    assert allclose_with_out(delta.get(), 1.)

    delta[:] = cost.bprop_noise(y_noise)
    assert allclose_with_out(delta.get(), -1.)

    delta[:] = cost.bprop_generator(y_noise)
    assert allclose_with_out(delta.get(), 1.)

def test_modified_gan_cost(backend_default):
    """
    Set up a modified GANCost transform and make sure cost and errors are getting
    computed correctly.
    """
    be = backend_default
    cost = GANCost(cost_type="dis", func="modified")

    y_data = be.iobuf(5).fill(1.)
    y_noise = be.iobuf(5).fill(2.)
    output = be.iobuf(1)
    expected = be.iobuf(1)
    delta = be.iobuf(5)

    # fprop for discriminator cost
    output[:] = cost(y_data, y_noise)
    expected[:] = -be.sum(be.safelog(y_data) + be.safelog(1-y_noise), axis=0)
    tensors_allclose(output, expected)

    # bprop for modified cost
    delta[:] = cost.bprop_data(y_data)
    assert allclose_with_out(delta.get(), -1. / 1)

    delta[:] = cost.bprop_noise(y_noise)
    assert allclose_with_out(delta.get(), 1. - 2.)

    delta[:] = cost.bprop_generator(y_noise)
    assert allclose_with_out(delta.get(), -1. / 2.)

def test_biSum(backend_default, fargs, deltas_buffer):

    seq_len, input_size, hidden_size, batch_size = fargs
    input_size *= 2

    in_shape = (input_size, seq_len)
    NervanaObject.be.bsz = batch_size

    bisum = BiSum()
    bisum.configure(in_shape)
    bisum.prev_layer = True

    bisum.allocate()
    bisum.allocate_deltas(deltas_buffer)
    deltas_buffer.allocate_buffers()
    bisum.set_deltas(deltas_buffer)

    # inputs
    inp_np = np.random.random((input_size, seq_len * batch_size))
    inp_be = bisum.be.array(inp_np)

    # outputs
    out_be = bisum.fprop(inp_be)
    del_be = bisum.bprop(out_be)

    out_ref = bisum.be.empty_like(out_be)
    out_ref[:] = inp_be[:input_size // 2] + inp_be[input_size // 2:]
    assert out_be.shape[0] * 2 == inp_be.shape[0]
    assert allclose_with_out(out_be.get(), out_ref.get(), rtol=0.0, atol=1.0e-5)

    assert allclose_with_out(del_be[:input_size // 2].get(), out_be.get(), rtol=0.0, atol=1.0e-5)
    assert allclose_with_out(del_be[input_size // 2:].get(), out_be.get(), rtol=0.0, atol=1.0e-5)

def test_model_get_outputs_rnn(backend_default, data):

    dataset = PTB(50, path=data)
    dataiter = dataset.train_iter

    # weight initialization
    init = Constant(0.08)

    # model initialization
    layers = [
        Recurrent(150, init, activation=Logistic()),
        Affine(len(dataiter.vocab), init, bias=init, activation=Rectlin())
    ]

    model = Model(layers=layers)
    output = model.get_outputs(dataiter)

    assert output.shape == (dataiter.ndata, dataiter.seq_length, dataiter.nclass)

    # since the init are all constant and model is un-trained:
    # along the feature dim, the values should be all the same
    assert allclose_with_out(output[0, 0], output[0, 0, 0], rtol=0, atol=1e-4)
    assert allclose_with_out(output[0, 1], output[0, 1, 0], rtol=0, atol=1e-4)

    # along the time dim, the values should be increasing:
    assert np.alltrue(output[0, 2] > output[0, 1])
    assert np.alltrue(output[0, 1] > output[0, 0])

def compare_helper(op, inA, inB, ng, nc, dtype):
    numpy_result = math_helper(np, op, inA, inB, dtype=np.float32).astype(dtype)

    nervanaGPU_result = math_helper(ng, op, inA, inB, dtype=dtype).get()
    allclose_with_out(numpy_result, nervanaGPU_result, rtol=0, atol=1e-5)

    nervanaCPU_result = math_helper(nc, op, inA, inB, dtype=dtype).get()
    allclose_with_out(numpy_result, nervanaCPU_result, rtol=0, atol=1e-5)

def test_bibn(backend_default, fargs):

    seq_len, input_size, hidden_size, batch_size = fargs
    in_shape = (input_size, seq_len)
    NervanaObject.be.bsz = batch_size

    hidden_size = min(10, hidden_size)

    # setup the bi-directional rnn
    init_glorot = GlorotUniform()
    birnn = BiBNRNN(hidden_size, activation=Rectlinclip(slope=0), init=init_glorot)
    birnn.configure(in_shape)
    birnn.prev_layer = True
    birnn.allocate()
    birnn.set_deltas([birnn.be.iobuf(birnn.in_shape)])

    # test fprop

    # set the ff buffer
    inp_np = np.random.random(birnn.h_ff_buffer.shape)
    inp_be = birnn.be.array(inp_np)
    birnn.h_ff_buffer[:] = inp_np

    # compare the bn output with calling the backend bn
    xsum = birnn.be.zeros_like(birnn.xmean)
    xvar = birnn.be.zeros_like(birnn.xvar)
    gmean = birnn.be.zeros_like(birnn.gmean)
    gvar = birnn.be.zeros_like(birnn.gvar)
    gamma = birnn.be.ones(birnn.gamma.shape)
    beta = birnn.be.zeros_like(birnn.beta)
    grad_gamma = birnn.be.zeros_like(gamma)
    grad_beta = birnn.be.zeros_like(beta)
    out_ref = birnn.be.zeros_like(birnn.h_ff_buffer)

    xsum[:] = birnn.be.sum(birnn.h_ff_buffer, axis=1)
    birnn.be.compound_fprop_bn(
        birnn.h_ff_buffer, xsum, xvar, gmean, gvar,
        gamma, beta, out_ref, birnn.eps, birnn.rho,
        accumbeta=0, relu=False)

    # call the bibnrnn layer fprop_bn
    out_bn = birnn._fprop_bn(birnn.h_ff_buffer, inference=False)

    assert allclose_with_out(out_bn.get(), out_ref.get(), rtol=0.0, atol=1.0e-5)

    # test bprop
    err_np = np.random.random(birnn.h_ff_buffer.shape)
    err_be = birnn.be.array(err_np)

    err_out_ref = birnn.be.empty_like(err_be)
    birnn.be.compound_bprop_bn(err_out_ref, grad_gamma, grad_beta,
                               err_be,
                               inp_be, xsum, xvar, gamma,
                               birnn.eps)

    err_out_bn = birnn._bprop_bn(err_be, out_bn)

    assert allclose_with_out(err_out_bn.get(), err_out_ref.get(), rtol=0.0, atol=2.5e-5)

def test_all_rand(backend_default, allrand_args, deltas_buffer):
    # test with random weights and random inputs
    dtypeu = np.float32
    w_rng, rngmax = allrand_args
    inp_rng = [0.0, rngmax]
    nin = 1024
    nout = 2048
    batch_size = 16
    NervanaObject.be.bsz = batch_size

    init_unif = Uniform(low=w_rng[0], high=w_rng[1])
    layer = Linear(nout=nout, init=init_unif)
    inp = np.random.random((nin, batch_size))
    inp *= inp_rng[1] - inp_rng[0]
    inp += inp_rng[0]
    inp = inp.astype(dtypeu)
    layer.configure(nin)
    layer.prev_layer = True  # Hack to force delta buffer allocation
    layer.allocate()

    layer.allocate_deltas(deltas_buffer)
    deltas_buffer.allocate_buffers()
    layer.set_deltas(deltas_buffer)

    out = layer.fprop(layer.be.array(inp)).get()
    w = layer.W.get()

    # the expected output using numpy
    out_exp = np.dot(w, inp)

    # for larger layers need to estimate numerical precision
    atol = 2 * est_mm_prec(w, inp, ntrials=1)
    assert allclose_with_out(out_exp, out, atol=atol, rtol=0.0), \
        '%e %e' % (np.max(np.abs(out - out_exp)), atol)

    err = np.random.random((nout, batch_size))
    err = err * (inp_rng[1] - inp_rng[0]) + inp_rng[0]
    err = err.astype(dtypeu)
    deltas = layer.bprop(layer.be.array(err)).get()
    dw = layer.dW.get()

    deltas_exp = np.dot(w.T, err)
    atol = 2 * est_mm_prec(w.T, err, ntrials=1)
    assert allclose_with_out(deltas_exp, deltas, atol=atol, rtol=0.0), \
        '%e %e' % (np.max(np.abs(deltas_exp - deltas)), atol)

    dw_exp = np.dot(err, inp.T)
    atol = 2 * est_mm_prec(err, inp.T, ntrials=1)
    assert allclose_with_out(dw_exp, dw, atol=atol, rtol=0.0), \
        '%e %e' % (np.max(np.abs(dw_exp - dw)), atol)

    return

def test_recurrent_mean(backend_default, refgruargs, deltas_buffer):
    seq_len, nin, batch_size = refgruargs
    NervanaObject.be.bsz = batch_size

    in_shape = (nin, seq_len)
    layer = RecurrentMean()
    layer.configure(in_shape)
    layer.prev_layer = True
    layer.allocate()

    layer.allocate_deltas(deltas_buffer)
    deltas_buffer.allocate_buffers()
    layer.set_deltas(deltas_buffer)

    # zeros
    inp = layer.be.zeros((nin, seq_len * batch_size))
    out = layer.fprop(inp)
    err = layer.bprop(out).get()
    assert np.all(out.get() == np.zeros((nin, batch_size)))
    assert np.all(err == inp.get())

    # ones
    inp = layer.be.ones((nin, seq_len * batch_size))
    out = layer.fprop(inp)
    err = layer.bprop(out).get()
    assert np.all(out.get() == np.ones((nin, batch_size)))
    assert np.all(err == 1. / seq_len * inp.get())

    # random
    rinp = np.random.random((nin, batch_size))
    inp = np.repeat(rinp, repeats=seq_len, axis=1)
    inp_g = layer.be.array(inp)
    out = layer.fprop(inp_g)
    err = layer.bprop(out)
    assert allclose_with_out(out.get(), rinp)
    assert allclose_with_out(err.get(), 1. / seq_len * inp)

    # full random
    inp = np.random.random((nin, seq_len * batch_size))
    inp_g = layer.be.array(inp)
    out = layer.fprop(inp_g)
    err = layer.bprop(out)
    out_comp = np.zeros(out.shape)
    err_comp = np.zeros(inp.shape)
    for i in range(seq_len):
        out_comp[:] = out_comp + inp[:, i * batch_size:(i + 1) * batch_size]
        err_comp[:, i * batch_size:(i + 1) * batch_size] = out.get() / float(seq_len)
    out_comp[:] /= float(seq_len)

    assert allclose_with_out(out_comp, out.get())
    assert allclose_with_out(err_comp, err.get())

def test_schedule(backend_default):
    """
    Test constant rate, fixed step and various modes of programmable steps.
    """
    lr_init = 0.1

    # default scheduler has a constant learning rate
    sch = Schedule()
    for epoch in range(10):
        lr = sch.get_learning_rate(learning_rate=lr_init, epoch=epoch)
        assert lr == lr_init

    # test a uniform step schedule
    step_config = 2
    change = 0.5
    sch = Schedule(step_config=step_config, change=change)
    for epoch in range(10):
        lr = sch.get_learning_rate(learning_rate=lr_init, epoch=epoch)
        # test a repeated call for the same epoch
        lr2 = sch.get_learning_rate(learning_rate=lr_init, epoch=epoch)
        # print epoch, lr, lr2
        assert allclose_with_out(lr, lr_init * change**(np.floor(epoch // step_config)))
        assert allclose_with_out(lr2, lr_init * change**(np.floor(epoch // step_config)))

    # test a list step schedule
    sch = Schedule(step_config=[2, 3], change=.1)
    assert allclose_with_out(.1, sch.get_learning_rate(learning_rate=.1, epoch=0))
    assert allclose_with_out(.1, sch.get_learning_rate(learning_rate=.1, epoch=1))
    assert allclose_with_out(.01, sch.get_learning_rate(learning_rate=.1, epoch=2))
    # test a repeated call for the same epoch
    assert allclose_with_out(.01, sch.get_learning_rate(learning_rate=.1, epoch=2))
    assert allclose_with_out(.001, sch.get_learning_rate(learning_rate=.1, epoch=3))
    assert allclose_with_out(.001, sch.get_learning_rate(learning_rate=.1, epoch=4))

def test_biLSTM_fprop(backend_default, fargs):

    # basic sanity check with 0 weights random inputs
    seq_len, input_size, hidden_size, batch_size = fargs
    in_shape = (input_size, seq_len)
    out_shape = (hidden_size, seq_len)
    NervanaObject.be.bsz = batch_size

    # setup the bi-directional rnn
    init_glorot = GlorotUniform()
    bilstm = BiLSTM(hidden_size, gate_activation=Logistic(), init=init_glorot,
                    activation=Tanh(), reset_cells=True)
    bilstm.configure(in_shape)
    bilstm.prev_layer = True
    bilstm.allocate()

    # same weight
    nout = hidden_size
    bilstm.W_input_b[:] = bilstm.W_input_f
    bilstm.W_recur_b[:] = bilstm.W_recur_f
    bilstm.b_b[:] = bilstm.b_f
    bilstm.dW[:] = 0

    # inputs - random and flipped left-to-right inputs
    lr = np.random.random((input_size, seq_len * batch_size))
    lr_rev = list(reversed(get_steps(lr.copy(), in_shape)))

    rl = con(lr_rev, axis=1)
    inp_lr = bilstm.be.array(lr)
    inp_rl = bilstm.be.array(rl)

    # outputs
    out_lr = bilstm.fprop(inp_lr).get().copy()

    bilstm.h_buffer[:] = 0
    out_rl = bilstm.fprop(inp_rl).get().copy()

    # views
    out_lr_f_s = get_steps(out_lr[:nout], out_shape)
    out_lr_b_s = get_steps(out_lr[nout:], out_shape)
    out_rl_f_s = get_steps(out_rl[:nout], out_shape)
    out_rl_b_s = get_steps(out_rl[nout:], out_shape)

    # asserts
    for x_f, x_b, y_f, y_b in zip(out_lr_f_s, out_lr_b_s,
                                  reversed(out_rl_f_s), reversed(out_rl_b_s)):
        assert allclose_with_out(x_f, y_b, rtol=0.0, atol=1.0e-5)
        assert allclose_with_out(x_b, y_f, rtol=0.0, atol=1.0e-5)

def test_recurrent_last(backend_default, refgruargs, deltas_buffer):
    seq_len, nin, batch_size = refgruargs
    NervanaObject.be.bsz = batch_size

    in_shape = (nin, seq_len)
    layer = RecurrentLast()
    layer.configure(in_shape)
    layer.prev_layer = True
    layer.allocate()

    layer.allocate_deltas(deltas_buffer)
    deltas_buffer.allocate_buffers()
    layer.set_deltas(deltas_buffer)

    # zeros
    inp = layer.be.zeros((nin, seq_len * batch_size))
    out = layer.fprop(inp)
    err = layer.bprop(out).get()
    assert np.all(out.get() == np.zeros((nin, batch_size)))
    assert np.all(err == inp.get())

    # ones
    inp = layer.be.ones((nin, seq_len * batch_size))
    out = layer.fprop(inp)
    err = layer.bprop(out).get()
    assert np.all(out.get() == np.ones((nin, batch_size)))
    assert np.all(err[:, -batch_size:] == inp.get()[:, -batch_size:])
    assert np.all(
        err[:, :-batch_size] == np.zeros((nin, (seq_len - 1) * batch_size)))

    # random
    rinp = np.random.random((nin, batch_size))
    inp = np.repeat(rinp, repeats=seq_len, axis=1)
    inp_g = layer.be.array(inp)
    out = layer.fprop(inp_g)
    err = layer.bprop(out)
    assert allclose_with_out(out.get(), rinp)
    assert allclose_with_out(err[:, -batch_size:].get(), rinp)

    # full random
    inp = np.random.random((nin, seq_len * batch_size))
    inp_g = layer.be.array(inp)
    out = layer.fprop(inp_g)
    err = layer.bprop(out)
    out_comp = np.zeros(out.shape)
    err_comp = np.zeros(inp.shape)
    out_comp[:] = inp[:, -batch_size:]
    err_comp[:, -batch_size:] = out.get()

def test_hdf5meansubtract(backend_default, meansubhdf):
    NervanaObject.be.bsz = 128
    bsz = 128

    datit = HDF5Iterator(meansubhdf[0])
    datit.allocate()
    typ = meansubhdf[1]
    mn = datit.mean.get()
    assert typ in ['chan_mean', 'full_mean']

    cnt_image = 0
    max_len = datit.ndata
    MAX_CNT = max_len*datit.inp.shape[1]
    for x, t in datit:
        x_ = x.get().flatten()
        x_exp = (np.arange(len(x_)) + cnt_image) % MAX_CNT
        x_exp = x_exp.reshape((-1, np.prod(datit.lshape))).T
        if typ == 'chan_mean':
            x_exp = x_exp.reshape((datit.lshape[0], -1)) - mn
        elif typ == 'full_mean':
            x_exp = x_exp.reshape((-1, bsz)) - mn
        x_exp = x_exp.flatten()
        assert allclose_with_out(x_, x_exp, atol=0.0, rtol=1.0e-7)
        cnt_image += len(x_)

    datit.cleanup()

def test_linear_ones(backend_default, basic_linargs, deltas_buffer):
    # basic sanity check with all ones on the inputs
    # and weights, check that each row in output
    # is the sum of the weights for that output
    # this check will confirm that the correct number
    # of operations is being run
    nin, nout, batch_size = basic_linargs
    NervanaObject.be.bsz = batch_size

    dtypeu = np.float32

    init_unif = Uniform(low=1.0, high=1.0)
    layer = Linear(nout=nout, init=init_unif)
    inp = layer.be.array(dtypeu(np.ones((nin, batch_size))))
    layer.configure(nin)
    layer.prev_layer = True  # Hack to force delta buffer allocation
    layer.allocate()

    layer.allocate_deltas(deltas_buffer)
    deltas_buffer.allocate_buffers()
    layer.set_deltas(deltas_buffer)

    out = layer.fprop(inp).get()
    w = layer.W.get()
    sums = np.sum(w, 1).reshape((nout, 1)) * np.ones((1, batch_size))

    # for larger layers need to estimate numerical precision
    # atol = est_mm_prec(w, inp.get())
    assert allclose_with_out(sums, out, atol=0.0, rtol=0.0), \
        '%e' % np.max(np.abs(out - sums))
    return

def test_padding(backend_default, poolargs):
    fshape, nifm, padding, stride, in_sz, batch_size = poolargs

    NervanaObject.be.bsz = batch_size

    # basic sanity check with random inputs
    inshape = (nifm, in_sz, in_sz)
    insize = np.prod(inshape)
    neon_layer = Pooling(fshape=fshape, strides=stride, padding=padding)

    inp = neon_layer.be.array(np.random.random((insize, batch_size)))
    inp.lshape = inshape
    neon_layer.configure(inshape)
    neon_layer.prev_layer = True
    neon_layer.allocate()
    neon_layer.set_deltas([neon_layer.be.iobuf(inshape)])

    out = neon_layer.fprop(inp).get()

    ncheck = [0, batch_size // 2, batch_size - 1]

    (out_exp, check_inds) = ref_pooling(inp, inp.lshape,
                                        (fshape, fshape),
                                        padding,
                                        (stride, stride),
                                        neon_layer.be,
                                        ncheck=ncheck)

    out_shape = list(out_exp.shape[0:3])
    out_shape.append(batch_size)
    outa = out.reshape(out_shape)

    assert allclose_with_out(out_exp, outa[:, :, :, check_inds], atol=0.0, rtol=0.0)

def test_softmax_big_inputs(backend_default):
    np.random.seed(1)

    be = backend_default
    assert be.bsz >= 128, 'This tests needs large batch size'

    act = Softmax()
    Nout = 1000  # 1000 input and output units to softmax

    # random inputs
    x_ = np.random.random((Nout, be.bsz))

    x = be.iobuf(Nout)
    # init input to softmax
    x[:] = x_

    # numpy softmax
    mx = np.max(x_, axis=0)
    ex = np.exp(x_ - mx)
    y_ = ex/np.sum(ex, axis=0)

    # in-place softmax on device
    x[:] = act(x)

    assert allclose_with_out(y_, x.get(), atol=0.0, rtol=1.0e-5)

def checkSequentialMatchesBatch():
    """ check LSTM I/O forward/backward interactions """

    n, b, d = (5, 3, 4)  # sequence length, batch size, hidden size
    input_size = 10
    WLSTM = LSTM.init(input_size, d)  # input size, hidden size
    X = np.random.randn(n, b, input_size)
    h0 = np.random.randn(b, d)
    c0 = np.random.randn(b, d)

    # sequential forward
    cprev = c0
    hprev = h0
    caches = [{} for t in range(n)]
    Hcat = np.zeros((n, b, d))
    for t in range(n):
        xt = X[t:t + 1]
        _, cprev, hprev, cache = LSTM.forward(xt, WLSTM, cprev, hprev)
        caches[t] = cache
        Hcat[t] = hprev

    # sanity check: perform batch forward to check that we get the same thing
    H, _, _, batch_cache = LSTM.forward(X, WLSTM, c0, h0)
    assert allclose_with_out(H, Hcat), 'Sequential and Batch forward don''t match!'

    # eval loss
    wrand = np.random.randn(*Hcat.shape)
    # loss = np.sum(Hcat * wrand)
    dH = wrand

    # get the batched version gradients
    BdX, BdWLSTM, Bdc0, Bdh0 = LSTM.backward(dH, batch_cache)

    # now perform sequential backward
    dX = np.zeros_like(X)
    dWLSTM = np.zeros_like(WLSTM)
    dc0 = np.zeros_like(c0)
    dh0 = np.zeros_like(h0)
    dcnext = None
    dhnext = None
    for t in reversed(range(n)):
        dht = dH[t].reshape(1, b, d)
        dx, dWLSTMt, dcprev, dhprev = LSTM.backward(
            dht, caches[t], dcnext, dhnext)
        dhnext = dhprev
        dcnext = dcprev

        dWLSTM += dWLSTMt  # accumulate LSTM gradient
        dX[t] = dx[0]
        if t == 0:
            dc0 = dcprev
            dh0 = dhprev

    # and make sure the gradients match
    neon_logger.display('Making sure batched version agrees with sequential version: '
                        '(should all be True)')
    neon_logger.display(np.allclose(BdX, dX))
    neon_logger.display(np.allclose(BdWLSTM, dWLSTM))
    neon_logger.display(np.allclose(Bdc0, dc0))
    neon_logger.display(np.allclose(Bdh0, dh0))

def gradient_check(seq_len, input_size, hidden_size, batch_size,
                   threshold=1.0e-3):
    # 'threshold' is the max fractional difference
    #             between gradient estimate and
    #             bprop deltas (def is 5%)
    # for a given set of layer parameters calculate
    # the gradients and compare to the derivatives
    # obtained with the bprop function.  repeat this
    # for a range of perturbations and use the
    # perturbation size with the best results.
    # This is necessary for 32 bit computations

    min_max_err = -1.0  # minimum max error
    neon_logger.display('Perturb mag, max grad diff')
    for pert_exp in range(-5, 0):
        # need to generate the scaling and input outside
        # having an issue with the random number generator
        # when these are generated inside the gradient_calc
        # function
        input_shape = (input_size, seq_len * batch_size)
        output_shape = (hidden_size, seq_len * batch_size)

        rand_scale = np.random.random(output_shape) * 2.0 - 1.0
        inp = np.random.randn(*input_shape)

        pert_mag = 10.0**pert_exp
        (grad_est, deltas) = gradient_calc(seq_len,
                                           input_size,
                                           hidden_size,
                                           batch_size,
                                           epsilon=pert_mag,
                                           rand_scale=rand_scale,
                                           inp_bl=inp)
        dd = np.max(np.abs(grad_est - deltas))
        neon_logger.display('%e, %e' % (pert_mag, dd))
        if min_max_err < 0.0 or dd < min_max_err:
            min_max_err = dd
        # reset the seed so models are same in each run
        allclose_with_out(grad_est, deltas, rtol=0.0, atol=0.0)
        NervanaObject.be.rng_reset()

    # check that best value of worst case error is less than threshold
    neon_logger.display('Worst case error %e with perturbation %e' % (min_max_err, pert_mag))
    neon_logger.display('Threshold %e' % (threshold))
    assert min_max_err < threshold

def test_dconv_rand(backend_default, rand_convargs, deltas_buffer):
    indim, nifm, fshape, nofm, batch_size, rngmax, w_rng = rand_convargs
    if isinstance(NervanaObject.be, NervanaGPU) and NervanaObject.be.compute_capability < (5, 0):
        if nofm % 4 != 0:
            pytest.skip(msg="C dim must be a multiple of 4 for Kepler bprop kernel")
    NervanaObject.be.bsz = batch_size
    dtypeu = np.float32
    inp_rng = [0.0, rngmax]

    init_unif = Uniform(low=w_rng[0], high=w_rng[1])
    inshape = (indim, indim, nifm)
    insize = np.prod(inshape)

    # generate neon deconv layer
    # need to switch to nofm here...
    neon_layer = Deconvolution(fshape=(fshape, fshape, nofm), strides=1,
                               padding=0, init=init_unif)
    insize = np.prod(inshape)

    # generate reference deconv layer
    ref_layer = DeconvRefLayer(1, batch_size, identity, inshape[0], inshape[1:3],
                               (fshape, fshape), nofm, 1, dtypeu)

    # setup input in range inp_rng
    inpa = np.random.random((insize, batch_size))
    inpa *= (inp_rng[1] - inp_rng[0])
    inpa += inp_rng[0]
    inpa = inpa.astype(dtypeu)
    inp = neon_layer.be.array(inpa)
    inp.lshape = inshape

    # run fprop on neon
    neon_layer.configure(inshape)
    neon_layer.prev_layer = True
    neon_layer.allocate()
    neon_out = neon_layer.fprop(inp).get()

    neon_layer.allocate_deltas(deltas_buffer)
    deltas_buffer.allocate_buffers()
    neon_layer.set_deltas(deltas_buffer)

    # pull neon weights into ref layer weights
    ref_layer.weights = neon_layer.W.get().T
    ref_out = np.copy(ref_layer.berror)

    # estimate the numerical precision
    ref_layer.fprop(inpa.T, permute=True)
    ref_out2 = ref_layer.berror
    atol = 10 * np.max(np.abs(ref_out - ref_out2))
    assert allclose_with_out(ref_out.T, neon_out, atol=atol, rtol=0.0), \
        '%e %e' % (np.max(np.abs(ref_out.T - neon_out)), atol)

    # generate err array
    erra = np.random.random(neon_out.shape)
    erra *= (inp_rng[1] - inp_rng[0])
    erra += inp_rng[0]
    erra = erra.astype(dtypeu)

def test_dilated_conv(backend_default, fargs_tests):

    fsz = fargs_tests[0]
    dil = fargs_tests[1]
    stride = fargs_tests[2]
    be = backend_default

    o1, w1 = run(be, False, fsz, stride, 1, dil)
    o2, w2 = run(be, True, fsz, stride, 1, dil)
    # Verify that the results of faked dilation match those of actual dilation.
    assert allclose_with_out(o1, o2, atol=1e-1, rtol=4e-3)
    try:
        assert allclose_with_out(w1, w2, atol=0, rtol=1e-3)
    except Exception:
        if not isinstance(NervanaObject.be, NervanaGPU):
            assert allclose_with_out(w1, w2, atol=1e-1, rtol=1e-3)
        else:
            assert allclose_with_out(w1, w2, atol=0, rtol=1e-3)

def test_shift_schedule(backend_default):
    """
    Test binary shift learning rate schedule
    """
    lr_init = 0.1
    interval = 1
    sch = ShiftSchedule(interval)
    for epoch in range(10):
        lr = sch.get_learning_rate(learning_rate=lr_init, epoch=epoch)
        assert allclose_with_out(lr, lr_init / (2 ** epoch))