def test_cpu_randomstate():
    # run 1
    be = gen_backend(backend='cpu', rng_seed=100)

    a = be.empty((3, 3))
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    x0 = a.get()
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    x1 = a.get()

    # run 2, using reset
    be.rng_reset()
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    y0 = a.get()
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    y1 = a.get()

    del(be)

    # run 3, using a new backend
    be = gen_backend(backend='cpu', rng_seed=100)

    a = be.empty((3, 3))
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    z0 = a.get()
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    z1 = a.get()

    # check equality
    assert tensors_allclose([x0, x1], [y0, y1], rtol=0., atol=0.)
    assert tensors_allclose([x0, x1], [z0, z1], rtol=0., atol=0.)
    del(be)

    def __init__(self, model, ngpu, options,
                 data_options=None, time_options=None):
        self.model = model

        #self.model.set_batch_size(data_options['batch_size'])
        
        self.ngpu = ngpu
        self.gpu_mode = True if ngpu >= 1 else False
        self.time_options = time_options
        self.data_options = data_options
        if self.gpu_mode:
            try:
                self.be = gen_backend(backend='nervanagpu',
                                      batch_size=data_options['batch_size'])
                print("Backgrand: nervanagpu")
            except:
                self.be = gen_backend(backend='gpu',
                                      batch_size=data_options['batch_size'])
                print("Backgrand: gpu")                
        else:
            self.be = gen_backend(backend='mkl',
                                  batch_size=data_options['batch_size'])

        self.loss = L.GeneralizedCost(costfunc=TF.CrossEntropyMulti())
        B = self.data_options['batch_size']
        self.model.bsz(B)
        
        C, W, H = self.data_options['image_shape']
            
        self.model.initialize(((C, H, W), B), self.loss)

def test_gpu_randomstate():
    # run 1
    be = gen_backend(backend='gpu', rng_seed=100)
    a = be.empty((3, 3))

    a[:] = be.rand()  # gpu rand
    x0 = a.get()
    x1 = be.rng.rand(3, 3)  # host rand
    a[:] = be.rand()  # gpu rand
    x2 = a.get()
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    x3 = a.get()

    assert len(be.context_rand_state_map) == 1 and len(be.context_rand_state_alive) == 1
    for ctx in be.context_rand_state_alive:
        assert be.context_rand_state_alive[ctx] is True

    # run 2, using reset
    be.rng_reset()

    for ctx in be.context_rand_state_alive:
        assert be.context_rand_state_alive[ctx] is False

    a[:] = be.rand()
    y0 = a.get()
    y1 = be.rng.rand(3, 3)
    a[:] = be.rand()
    y2 = a.get()
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    y3 = a.get()

    assert len(be.context_rand_state_map) == 1 and len(be.context_rand_state_alive) == 1
    for ctx in be.context_rand_state_alive:
        assert be.context_rand_state_alive[ctx] is True

    del(be)

    # run 3, using a new backend
    be = gen_backend(backend='gpu', rng_seed=100)
    a = be.empty((3, 3))

    a[:] = be.rand()  # gpu rand
    z0 = a.get()
    z1 = be.rng.rand(3, 3)  # host rand
    a[:] = be.rand()  # gpu rand
    z2 = a.get()
    be.make_binary_mask(a, keepthresh=be.rng.rand())
    z3 = a.get()

    # check equality
    assert_tensors_allclose([x0, x1, x2, x3], [y0, y1, y2, y3], rtol=0., atol=0.)
    assert_tensors_allclose([x0, x1, x2, x3], [z0, z1, z2, z3], rtol=0., atol=0.)

    del(be)

  def __init__(self, num_actions, args):
    # remember parameters
    self.num_actions = num_actions
    self.batch_size = args.batch_size
    self.discount_rate = args.discount_rate
    self.history_length = args.history_length
    self.screen_dim = (args.screen_height, args.screen_width)
    self.clip_error = args.clip_error
    self.min_reward = args.min_reward
    self.max_reward = args.max_reward
    self.batch_norm = args.batch_norm

    # create Neon backend
    self.be = gen_backend(backend = args.backend,
                 batch_size = args.batch_size,
                 rng_seed = args.random_seed,
                 device_id = args.device_id,
                 datatype = np.dtype(args.datatype).type,
                 stochastic_round = args.stochastic_round)

    # prepare tensors once and reuse them
    self.input_shape = (self.history_length,) + self.screen_dim + (self.batch_size,)
    self.input = self.be.empty(self.input_shape)
    self.input.lshape = self.input_shape # HACK: needed for convolutional networks
    self.targets = self.be.empty((self.num_actions, self.batch_size))

    # create model
    layers = self._createLayers(num_actions)
    self.model = Model(layers = layers)
    self.cost = GeneralizedCost(costfunc = SumSquared())
    # Bug fix
    for l in self.model.layers.layers:
      l.parallelism = 'Disabled'
    self.model.initialize(self.input_shape[:-1], self.cost)
    if args.optimizer == 'rmsprop':
      self.optimizer = RMSProp(learning_rate = args.learning_rate, 
          decay_rate = args.decay_rate, 
          stochastic_round = args.stochastic_round)
    elif args.optimizer == 'adam':
      self.optimizer = Adam(learning_rate = args.learning_rate, 
          stochastic_round = args.stochastic_round)
    elif args.optimizer == 'adadelta':
      self.optimizer = Adadelta(decay = args.decay_rate, 
          stochastic_round = args.stochastic_round)
    else:
      assert false, "Unknown optimizer"

    # create target model
    self.train_iterations = 0
    if args.target_steps:
      self.target_model = Model(layers = self._createLayers(num_actions))
      # Bug fix
      for l in self.target_model.layers.layers:
        l.parallelism = 'Disabled'
      self.target_model.initialize(self.input_shape[:-1])
      self.save_weights_prefix = args.save_weights_prefix
    else:
      self.target_model = self.model

    self.callback = None

def sanity_check(conf_file, result, **be_args):
    experiment = deserialize(os.path.join(dir, conf_file))
    backend = gen_backend(model=experiment.model, **be_args)
    experiment.initialize(backend)
    res = experiment.run()
    print(float(res['test']['MisclassRate_TOP_1']))
    assert float(res['test']['MisclassRate_TOP_1']) == result

def backend(request):
    """
    Fixture to setup the backend before running a
    test.  Also registers the teardown function to clean
    up the backend after a test is done.  This had module
    scope, so this will be run once for each test in a
    given test file (module).

    This fixture is parameterized to run both the cpu and
    gpu backends for every test
    """
    be = gen_backend(backend=request.param, rng_seed=0)
    NervanaObject.be = be
    be.bsz = 128  # hardwires here to 128

    # add a cleanup call - will run after all
    # test in module are done
    def cleanup():
        be = request.getfuncargvalue("backend")
        del be

    request.addfinalizer(cleanup)

    # tests using this fixture can
    # access the backend object from
    # backend or use the NervanaObject.be global
    return be

def backend_default(request):
    '''
    Fixture to setup the backend before running a
    test.  Also registers the teardown function to clean
    up the backend after a test is done.  This had module
    scope, so this will be run once for each test in a
    given test file (module).

    This fixture is parameterized to run both the cpu and
    gpu backends for every test
    '''
    be = gen_backend(backend=request.param,
                     default_dtype=np.float32,
                     batch_size=128,
                     rng_seed=0)

    # add a cleanup call - will run after all
    # test in module are done
    def cleanup():
        be = request.getfuncargvalue('backend_default')
        del be
    request.addfinalizer(cleanup)

    # tests using this fixture can
    # access the backend object from
    # backend or use the NervanaObject.be global
    return be

    def __init__(self, env, args, rng, name = "DQNNeon"):
        """ Initializes a network based on the Neon framework.

        Args:
            env (AtariEnv): The envirnoment in which the agent actuates.
            args (argparse.Namespace): All settings either with a default value or set via command line arguments.
            rng (mtrand.RandomState): initialized Mersenne Twister pseudo-random number generator.
            name (str): The name of the network object.

        Note:
            This function should always call the base class first to initialize
            the common values for the networks.
        """
        _logger.info("Initializing new object of type " + str(type(self).__name__))
        super(DQNNeon, self).__init__(env, args, rng, name)
        self.input_shape = (self.sequence_length,) + self.frame_dims + (self.batch_size,)
        self.dummy_batch = np.zeros((self.batch_size, self.sequence_length) + self.frame_dims, dtype=np.uint8)
        self.batch_norm = args.batch_norm

        self.be = gen_backend(
                backend = args.backend,
                batch_size = args.batch_size,
                rng_seed = args.random_seed,
                device_id = args.device_id,
                datatype = np.dtype(args.datatype).type,
                stochastic_round = args.stochastic_round)

        # prepare tensors once and reuse them
        self.input = self.be.empty(self.input_shape)
        self.input.lshape = self.input_shape # HACK: needed for convolutional networks
        self.targets = self.be.empty((self.output_shape, self.batch_size))

        # create model
        layers = self._create_layer()
        self.model = Model(layers = layers)
        self.cost_func = GeneralizedCost(costfunc = SumSquared())
        # Bug fix
        for l in self.model.layers.layers:
            l.parallelism = 'Disabled'
        self.model.initialize(self.input_shape[:-1], self.cost_func)

        self._set_optimizer()

        if not self.args.load_weights == None:
            self.load_weights(self.args.load_weights)

        # create target model
        if self.target_update_frequency:
            layers = self._create_layer()
            self.target_model = Model(layers)
            # Bug fix
            for l in self.target_model.layers.layers:
                l.parallelism = 'Disabled'
            self.target_model.initialize(self.input_shape[:-1])
        else:
            self.target_model = self.model

        self.callback = None
        _logger.debug("%s" % self)

def serialize_check(conf_file, result, tol, res_string, **be_args):
    experiment = deserialize(conf_file)
    backend = gen_backend(model=experiment.model, **be_args)
    experiment.initialize(backend)
    res = experiment.run()
    print float(res[res_string]['MisclassPercentage_TOP_1']), result,
    assert abs(
        float(res[res_string]['MisclassPercentage_TOP_1']) - result) < tol

def speed_check(conf_file, num_epochs, **be_args):
    experiment = deserialize(os.path.join(dir, conf_file))
    experiment.model.num_epochs = num_epochs
    backend = gen_backend(model=experiment.model, **be_args)
    experiment.initialize(backend)
    start = time.time()
    experiment.run()
    return (time.time() - start)

def test_loader_exception_iter():
    # NOTE: manifest needs to stay in scope until DataLoader has read it.
    manifest = random_manifest(10, 2)
    config = generic_config(manifest.name)

    dl = DataLoader(config, gen_backend(backend='cpu'))

    assert len(list(iter(dl))) == 4

def run():
    model = create_model(nin=784)
    backend = gen_backend(rng_seed=0)
    dataset = MNIST(repo_path='~/data/')
    experiment = FitPredictErrorExperiment(model=model,
                                           backend=backend,
                                           dataset=dataset)
    experiment.run()

    def __init__(self, batch_size, its, layer_def, W, I, gradO):
        gen_backend(backend='gpu', batch_size=batch_size,
                datatype=np.float32, device_id=0)

        assert layer_def['iH'] == layer_def['iW']
        assert layer_def['kH'] == layer_def['kW']
        assert layer_def['dH'] == layer_def['dW']
        assert layer_def['padH'] == layer_def['padW']

        input_filters = layer_def['Ci']
        output_filters = layer_def['Co']
        image_size = layer_def['iW']
        filter_size = layer_def['kH']
        padding = layer_def['padH']
        stride = layer_def['dH']

        self.I = I
        self.W = W
        self.gradO = gradO

        I_cuda = gpuarray.to_gpu(I)
        gradO_cuda = gpuarray.to_gpu(gradO)
        W_cuda = gpuarray.to_gpu(W)

        conv = Convolution((filter_size, filter_size, output_filters), strides=stride, padding=padding, init=init)
        conv.configure((input_filters, image_size, image_size))
        conv.allocate()
#        conv.allocate_deltas()
        conv.W = W_cuda

        self.conv = conv
        deltas = np.zeros(I.shape, dtype=np.float32)
        deltas_cuda = gpuarray.to_gpu(deltas)
        conv.deltas = deltas_cuda

#        self.O = O
#        self.gradW = gradW
#        self.gradI = gradI

        self.I_cuda = I_cuda
        self.O_cuda = conv.outputs
        self.gradO_cuda = gradO_cuda

        self.gradW_cuda = conv.dW
        self.gradI_cuda = conv.deltas

def compare_helper(op, inA, inB, dtype):
    numpy_result = math_helper(np, op, inA, inB, dtype=np.float32)

    if np.dtype(dtype).kind == 'i' or np.dtype(dtype).kind == 'u':
        numpy_result = np.around(numpy_result)
        numpy_result = numpy_result.clip(np.iinfo(dtype).min, np.iinfo(dtype).max)
    numpy_result = numpy_result.astype(dtype)

    if dtype in (np.float32, np.float16):
        gpu = gen_backend(backend='gpu', default_dtype=dtype)
        nervanaGPU_result = math_helper(gpu, op, inA, inB, dtype=dtype)
        nervanaGPU_result = nervanaGPU_result.get()
        np.allclose(numpy_result, nervanaGPU_result, rtol=0, atol=1e-5)

    cpu = gen_backend(backend='cpu', default_dtype=dtype)
    nervanaCPU_result = math_helper(cpu, op, inA, inB, dtype=dtype)
    nervanaCPU_result = nervanaCPU_result.get()
    np.allclose(numpy_result, nervanaCPU_result, rtol=0, atol=1e-5)

def test_loader_exception_next():
    # NOTE: manifest needs to stay in scope until DataLoader has read it.
    manifest = random_manifest(10, 2)
    config = generic_config(manifest.name)

    dl = DataLoader(config, gen_backend(backend='cpu'))
    dl.next()
    with pytest.raises(LoaderRuntimeError):
        dl.next()

def get_backend(request, datatype=np.float32):
    be = gen_backend(backend=request.param,
                     datatype=datatype,
                     device_id=request.config.getoption("--device_id"),
                     batch_size=128,
                     rng_seed=0)
    if request.param == 'gpu':
        be.enable_winograd = 2 if be.enable_winograd else be.enable_winograd
    return be

def test_loader_invalid_config_type():
    manifest = random_manifest(10)
    config = generic_config(manifest.name)

    config['type'] = 'invalid type name'

    with pytest.raises(Exception) as ex:
        dl = DataLoader(config, gen_backend(backend='cpu'))

    assert 'invalid type name' in str(ex)

def test_loader_missing_config_field():
    manifest = random_manifest(10)
    config = generic_config(manifest.name)

    del config['image']

    with pytest.raises(Exception) as ex:
        dl = DataLoader(config, gen_backend(backend='cpu'))

    assert 'image' in str(ex)

def gen_model(backend_type):
    # setup backend
    gen_backend(
        backend=backend_type, batch_size=batch_size, rng_seed=2, device_id=args.device_id, default_dtype=args.datatype
    )

    init_uni = Uniform(low=-0.1, high=0.1)

    # Set up the model layers
    layers = []
    layers.append(Conv((5, 5, 16), init=init_uni, bias=Constant(0), activation=Rectlin()))
    layers.append(Pooling(2))
    layers.append(Conv((5, 5, 32), init=init_uni, activation=Rectlin()))
    layers.append(Pooling(2))
    layers.append(Affine(nout=500, init=init_uni, activation=Rectlin()))
    layers.append(Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True)))

    mlp = Model(layers=layers)
    return mlp

    def train(self, dataset, model=None):
        """Trains the passed model on the given dataset. If no model is passed, `generate_default_model` is used."""
        print "[%s] Starting training..." % self.model_name                                                              
        start = time.time()

        # The training will be run on the CPU. If a GPU is available it should be used instead.
        backend = gen_backend(backend='cpu',
                              batch_size=self.batch_size,
                              rng_seed=self.random_seed,
                              stochastic_round=False)

        cost = GeneralizedCost(
            name='cost',
            costfunc=CrossEntropyMulti())

        optimizer = GradientDescentMomentum(
            learning_rate=self.lrate,
            momentum_coef=0.9)

        # set up the model and experiment
        if not model:
            model = self.generate_default_model(dataset.num_labels)

        args = NeonCallbackParameters()
        args.output_file = os.path.join(self.root_path, self.Callback_Store_Filename)
        args.evaluation_freq = 1
        args.progress_bar = False
        args.epochs = self.max_epochs
        args.save_path = os.path.join(self.root_path, self.Intermediate_Model_Filename)
        args.serialize = 1
        args.history = 100
        args.model_file = None

        callbacks = Callbacks(model, dataset.train(), args, eval_set=dataset.test())

        # add a callback that saves the best model state
        callbacks.add_save_best_state_callback(self.model_path)

        # Uncomment line below to run on GPU using cudanet backend
        # backend = gen_backend(rng_seed=0, gpu='cudanet')
        model.fit(
            dataset.train(),
            optimizer=optimizer,
            num_epochs=self.max_epochs,
            cost=cost,
            callbacks=callbacks)

        print("[%s] Misclassification error = %.1f%%"
              % (self.model_name, model.eval(dataset.test(), metric=Misclassification()) * 100))
        print "[%s] Finished training!" % self.model_name
        end = time.time()
        print "[%s] Duration in seconds", end - start

        return model