def gradients_check(self, func, param, autograds, h=0.0005, df=1):
        # param: PyTensor
        # autograds: numpy_tensor
        p = tensor.to_numpy(param)
        it = np.nditer(p, flags=['multi_index'], op_flags=['readwrite'])
        while not it.finished:
            idx = it.multi_index
            diff = np.zeros_like(p)
            diff[idx] += h
            diff = tensor.from_numpy(diff)
            diff.to_device(gpu_dev)

            param += diff
            pos = func()
            pos = tensor.to_numpy(pos)

            param -= diff
            param -= diff
            neg = func()
            neg = tensor.to_numpy(neg)

            numerical_grad = np.sum((pos - neg) * df) / (2 * h)
            #print((autograds[idx] - numerical_grad)/numerical_grad)
            # threshold set as -5% to +5%
            #self.assertAlmostEqual((autograds[idx] - numerical_grad)/(numerical_grad+0.0000001), 0., places=1)
            self.assertAlmostEqual(
                autograds[idx] - numerical_grad, 0., places=2)

            it.iternext()

def onnx_to_singa(niter, use_cpu=False):
    if use_cpu:
        print("Using CPU")
        dev = device.get_default_device()
    else:
        print("Using GPU")
        dev = device.create_cuda_gpu()
    model = sonnx.load("mlp.onnx")
    backend = sonnx.prepare(model, device=dev)
    sgd = opt.SGD(0.1)
    inputs = Tensor(
        data=data,
        device=dev,
        requires_grad=False,
        stores_grad=False,
        name="input",
    )
    target = Tensor(
        data=label,
        device=dev,
        requires_grad=False,
        stores_grad=False,
        name="target",
    )

    for i in range(100):
        y = backend.run([inputs])[0]
        loss = autograd.softmax_cross_entropy(y, target)
        for p, gp in autograd.backward(loss):
            sgd.update(p, gp)
        loss_rate = tensor.to_numpy(loss)[0]
        accuracy_rate = accuracy(tensor.to_numpy(y), label)

        print("Iter {}, accurate={}, loss={}".format(i, accuracy_rate, loss_rate))

 def test_copy_data(self):
     t = self.t
     t += 1.23
     s = self.s
     s += 5.43
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 0], 1.23)
     tensor.copy_data_to_from(t, s, 2)
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 0], 5.43, 5)
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 1], 5.43, 5)
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 2], 1.23)

    def test_numpy_convert(self):
        a = np.asarray([[1, 0, 0], [0, 1, 0]], dtype=np.int)
        t = tensor.from_numpy(a)
        b = tensor.to_numpy(t)
        self.assertEqual(np.sum(a-b), 0)

        a = np.asarray([[1, 0, 0], [0, 1, 0]], dtype=np.float32)
        t = tensor.from_numpy(a)
        b = tensor.to_numpy(t)
        self.assertEqual(np.sum(a-b), 0.)

 def test_slice(self):
     t = np.zeros((3, 3))
     t[:, :2] = float(2)
     t[:, 2] = float(1)
     lyr = layer.Slice('slice', 1, [2], t.shape)
     out = lyr.forward(model_pb2.kTrain, [tensor.from_numpy(t)])
     t1 = tensor.to_numpy(out[0])
     t2 = tensor.to_numpy(out[1])
     self.assertEquals(np.average(t1), 2)
     self.assertEquals(np.average(t2), 1)

 def test_unary_operators(self):
     t = self.t
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 0], 0.0)
     t += 1.23
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 0], 1.23)
     t -= 0.23
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 0], 1.23-0.23)
     t *= 2.5
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 0], (1.23-0.23)*2.5)
     t /= 2
     self.assertAlmostEqual(tensor.to_numpy(t)[0, 0], (1.23-0.23)*2.5/2)

 def test_binary_operators(self):
     t = self.t
     t += 3.2
     s = self.s
     s += 2.1
     a = t + s
     self.assertAlmostEqual(tensor.to_numpy(a)[0, 0], 3.2+2.1, 5)
     a = t - s
     self.assertAlmostEqual(tensor.to_numpy(a)[0, 0], 3.2-2.1, 5)
     a = t * s
     self.assertAlmostEqual(tensor.to_numpy(a)[0, 0], 3.2*2.1, 5)
     ''' not implemented yet

def singa_to_onnx(epochs, use_cpu=False, batchsize=32):
    sgd = opt.SGD(lr=0.1)

    # operations initialization
    conv1 = autograd.Conv2d(1, 8, 3, 2, padding=1) # 28 - 14
    conv2 = autograd.Conv2d(8, 4, 3, 2, padding=1) # 14 - 7
    pooling = autograd.MaxPool2d(3, 2, padding=1) # 7 - 4
    linear = autograd.Linear(64, 10)

    def forward(x, t):
        y = conv1(x)
        y = autograd.relu(y)
        y = conv2(y)
        y = autograd.relu(y)
        y = pooling(y)
        y = autograd.flatten(y)
        y = linear(y)
        loss = autograd.softmax_cross_entropy(y, t)
        return loss, y

    autograd.training = True
    (x_train, y_train), (x_test, y_test), dev = common(use_cpu)

    niter = 1 # x_train.shape[0] // batchsize
    for epoch in range(epochs):
        accuracy_rate = 0.0
        loss_rate = 0.0
        for i in range(niter):
            inputs = tensor.Tensor(
                device=dev,
                data=x_train[i * batchsize : (i + 1) * batchsize],
                stores_grad=False,
                name="input",
            )
            targets = tensor.Tensor(
                device=dev,
                data=y_train[i * batchsize : (i + 1) * batchsize],
                requires_grad=False,
                stores_grad=False,
                name="target",
            )
            loss, y = forward(inputs, targets)
            accuracy_rate += accuracy(
                tensor.to_numpy(y), y_train[i * batchsize : (i + 1) * batchsize]
            )
            loss_rate += tensor.to_numpy(loss)[0]
            for p, gp in autograd.backward(loss):
                sgd.update(p, gp)
        print( "accuracy is {}, loss is {}".format( accuracy_rate / niter, loss_rate / niter))
    model = sonnx.to_onnx_model([inputs], [y])
    sonnx.save(model, "cnn.onnx")

 def test_concat(self):
     t1 = tensor.Tensor((2, 3))
     t2 = tensor.Tensor((1, 3))
     t1.set_value(1)
     t2.set_value(2)
     lyr = layer.Concat('concat', 0, [(3,), (3,)])
     t = lyr.forward(model_pb2.kTrain, [t1, t2])
     tnp = tensor.to_numpy(t)
     self.assertEqual(np.sum(tnp), 12)
     t3 = tensor.Tensor((3, 3))
     t3.set_value(1.5)
     grads, _ = lyr.backward(model_pb2.kTrain, [t3])
     gnp = tensor.to_numpy(grads[0])
     self.assertEqual(np.sum(gnp), 6 * 1.5)

    def test_transpose(self):
        a = np.array([1.1,1.1,1.1,1.1,1.4,1.3,1.1,1.6,1.1,1.1,1.1,1.2])
        a = np.reshape(a,(2,3,2))
        ta = tensor.from_numpy(a)

        A1 = np.transpose(a)
        tA1 = tensor.transpose(ta)
        TA1 = tensor.to_numpy(tA1)
        A2 = np.transpose(a,[0,2,1])
        tA2 = tensor.transpose(ta,[0,2,1])
        TA2 = tensor.to_numpy(tA2)

        self.assertAlmostEqual(np.sum(TA1 - A1), 0.,places=3)
        self.assertAlmostEqual(np.sum(TA2 - A2), 0.,places=3)

    def test_einsum(self):

        a = np.array([1.1,1.1,1.1,1.1,1.4,1.3,1.1,1.6,1.1,1.1,1.1,1.2])
        a = np.reshape(a,(2,3,2))
        ta = tensor.from_numpy(a)

        res1 = np.einsum('kij,kij->kij', a, a)
        tres1 = tensor.einsum('kij,kij->kij', ta, ta)
        Tres1 = tensor.to_numpy(tres1)
        res2 = np.einsum('kij,kih->kjh', a, a)
        tres2 = tensor.einsum('kij,kih->kjh', ta, ta)
        Tres2 = tensor.to_numpy(tres2)
        
        self.assertAlmostEqual(np.sum(Tres1 - res1), 0.,places=3)
        self.assertAlmostEqual(np.sum(Tres2 - res2), 0.,places=3)

    def test_repeat(self):

        a = np.array([1.1,1.1,1.1,1.1,1.4,1.3,1.1,1.6,1.1,1.1,1.1,1.2])
        a = np.reshape(a,(2,3,2))
        ta = tensor.from_numpy(a)

        ta_repeat1 = tensor.repeat(ta,2,axis = None)
        a_repeat1 = np.repeat(a,2,axis = None)
        Ta_repeat1 = tensor.to_numpy(ta_repeat1)
        ta_repeat2 = tensor.repeat(ta, 4, axis = 1)
        a_repeat2 = np.repeat(a, 4, axis = 1)
        Ta_repeat2 = tensor.to_numpy(ta_repeat2)

        self.assertAlmostEqual(np.sum(Ta_repeat1 - a_repeat1), 0., places=3)
        self.assertAlmostEqual(np.sum(Ta_repeat2 - a_repeat2), 0., places=3)

    def test_tensordot(self):
        a = np.array([1.1,1.1,1.1,1.1,1.4,1.3,1.1,1.6,1.1,1.1,1.1,1.2])
        a = np.reshape(a,(2,3,2))

        ta = tensor.from_numpy(a)

        res1 = np.tensordot(a, a, axes = 1)
        tres1 = tensor.tensordot(ta, ta, axes = 1)
        Tres1 = tensor.to_numpy(tres1)
        res2 = np.tensordot(a, a, axes = ([0,1],[2,1]))
        tres2 = tensor.tensordot(ta, ta, axes = ([0,1],[2,1]))
        Tres2 = tensor.to_numpy(tres2)

        self.assertAlmostEqual(np.sum(Tres1 - res1), 0., places=3)
        self.assertAlmostEqual(np.sum(Tres2 - res2), 0., places=3)

 def test_sgd(self):
     lr = 0.1
     sgd = opt.SGD(lr)
     sgd.apply(0, self.g, self.W, 'w')
     w = tensor.to_numpy(self.W)
     for i in range(self.W.size()):
         self.assertAlmostEqual(w[i], self.np_W[i] - lr * self.np_g[i])

def predict(net, dev, synset_list, topk=5):
    '''Predict the label of each image.

    Args:
        net, a pretrained neural net
        images, a batch of images [batch_size, 3, 32, 32], which have been
            pre-processed
        dev, the training device
        synset_list: the synset of labels
        topk, return the topk labels for each image.
    '''
    while True:
        img_path = eval(input("Enter input image path('quit' to exit): "))
        if img_path == 'quit':
            return
        if not os.path.exists(img_path):
            print('Path is invalid')
            continue
        img = read_image(img_path)
        x = tensor.from_numpy(img.astype(np.float32)[np.newaxis, :])
        x.to_device(dev)
        y = net.predict(x)
        y.to_host()
        prob = tensor.to_numpy(y)
        lbl = np.argsort(-prob[0])  # sort prob in descending order
        print([synset_list[lbl[i]] for i in range(topk)])

 def test_constraint(self):
     threshold = 0.02
     cons = opt.L2Constraint(threshold)
     cons.apply(0, self.W, self.g)
     g = tensor.to_numpy(self.g)
     nrm = np.linalg.norm(self.np_g) / self.np_g.size
     for i in range(g.size):
         self.assertAlmostEqual(g[i], self.np_g[i] * threshold / nrm)

 def test_slice(self):
     t = np.zeros((3, 3))
     t[:, :2] = float(2)
     t[:, 2] = float(1)
     lyr = layer.Slice('slice', 1, [2], (3,))
     out = lyr.forward(model_pb2.kTrain, [tensor.from_numpy(t)])
     t1 = tensor.to_numpy(out[0])
     t2 = tensor.to_numpy(out[1])
     self.assertEqual(np.average(t1), 2)
     self.assertEqual(np.average(t2), 1)
     t1 = tensor.Tensor((3, 2))
     t2 = tensor.Tensor((3, 1))
     t1.set_value(1)
     t2.set_value(2)
     grad, _ = lyr.backward(model_pb2.kTrain, [t1, t2])
     gnp = tensor.to_numpy(grad)
     self.assertEqual(np.sum(gnp), 12)

 def test_regularizer(self):
     coefficient = 0.0001
     reg = opt.L2Regularizer(coefficient)
     reg.apply(0, self.W, self.g)
     g = tensor.to_numpy(self.g)
     for i in range(g.size):
         self.assertAlmostEqual(g[i],
                                self.np_g[i] + coefficient * self.np_W[i])

def serve(agent, use_cpu, parameter_file, topk=5):
    if use_cpu:
        print('running with cpu')
        dev = device.get_default_device()
        layer.engine = 'singacpp'
    else:
        print("runing with gpu")
        dev = device.create_cuda_gpu()
    agent = agent

    print('Start intialization............')
    net = create_net((3, 224, 224), parameter_file)
    net.to_device(dev)
    print('End intialization............')

    labels = np.loadtxt('synset_words.txt', str, delimiter='\t ')
    while True:
        key, val = agent.pull()
        if key is None:
            time.sleep(0.1)
            continue
        msg_type = MsgType.parse(key)
        if msg_type.is_request():
            try:
                response = ""
                img = imread(val['image'], mode='RGB').astype(np.float32)
                height,width = img.shape[:2]
                img[:, :, 0] -= 123.68
                img[:, :, 1] -= 116.779
                img[:, :, 2] -= 103.939
                img[:,:,[0,1,2]] = img[:,:,[2,1,0]]
                img = img.transpose((2, 0, 1))
                img = img[:, (height-224)//2:(height+224)//2,\
                          (width-224)//2:(width+224)//2]
                images = np.expand_dims(img, axis=0)

                x = tensor.from_numpy(images.astype(np.float32))
                x.to_device(dev)
                y = net.predict(x)
                prob = np.average(tensor.to_numpy(y), 0)
                # sort and reverse
                idx = np.argsort(-prob)[0:topk]
                for i in idx:
                    response += "%s:%s<br/>" % (labels[i], prob[i])
            except:
                traceback.print_exc()
                response = "Sorry, system error during prediction."
            agent.push(MsgType.kResponse, response)
        elif MsgType.kCommandStop.equal(msg_type):
                print('get stop command')
                agent.push(MsgType.kStatus, "success")
                break
        else:
            print('get unsupported message %s' % str(msg_type))
            agent.push(MsgType.kStatus, "Unknown command")
            break
        # while loop
    print("server stop")

 def forward(self, flag, x):
     '''pad zeros'''
     tmp = tensor.to_numpy(x)
     shape = add_to_tuple(x.shape)
     ret = np.zeros(shape)
     ret[:,:,:-1, :-1] = tmp
     y = tensor.from_numpy(ret)
     y.to_device(x.device)
     return y