def train_init(init_net, meta_alpha, loss_fn, image, target_bbox, evaluator):
    init_net.train()
    
    # Draw pos/neg samples
    pos_examples = gen_samples(SampleGenerator('gaussian', image.size, 0.1, 1.2),
                               target_bbox, opts['n_pos_init'], opts['overlap_pos_init'])
    neg_examples = np.concatenate([
                    gen_samples(SampleGenerator('uniform', image.size, 1, 2, 1.1), 
                                target_bbox, opts['n_neg_init']//2, opts['overlap_neg_init']),
                    gen_samples(SampleGenerator('whole', image.size, 0, 1.2, 1.1),
                                target_bbox, opts['n_neg_init']//2, opts['overlap_neg_init'])])
    # Crop images
    crop_size = opts['img_size']
    padding = opts['padding']
    image = np.asarray(image)
    pos_regions = extract_regions(image, pos_examples, crop_size, padding)
    neg_regions = extract_regions(image, neg_examples, crop_size, padding)
    pos_regions_var = Variable(torch.from_numpy(pos_regions[:opts['batch_pos']]))
    neg_regions_var = Variable(torch.from_numpy(neg_regions[:opts['batch_neg']]))
    if opts['use_gpu']:
        pos_regions_var = pos_regions_var.cuda()
        neg_regions_var = neg_regions_var.cuda()
    
    # training
    tracker_init_weights = OrderedDict((name, param) for (name, param) in init_net.named_parameters())
    tracker_keys = [name for (name, _) in init_net.named_parameters()]
    # the first iteration
    pos_score = init_net.forward(pos_regions_var)
    neg_score = init_net.forward(neg_regions_var)
    init_loss = loss_fn(pos_score,neg_score)
    init_acc,init_acc_pos,init_acc_neg = evaluator(pos_score, neg_score)
    grads = torch.autograd.grad(init_loss, tracker_init_weights.values(), create_graph=True)
    tracker_weights = OrderedDict((name, param - torch.mul(alpha,grad)) for
                                  ((name, param),(_,alpha),grad) in
                                  zip(tracker_init_weights.items(),
                                      meta_alpha.items(), grads))
    # rest of iterations
    for i in range(opts['n_init_updates']-1):
        pos_score = init_net.forward(pos_regions_var, tracker_weights)
        neg_score = init_net.forward(neg_regions_var, tracker_weights)
        loss = loss_fn(pos_score,neg_score)
        grads = torch.autograd.grad(loss, tracker_weights.values(), create_graph=True)
        tracker_weights = OrderedDict((name, param - torch.mul(alpha,grad))
                                      for ((name, param),(_,alpha),grad) in
                                      zip(tracker_weights.items(),meta_alpha.items(), grads))
    # update tracker
    init_net.copy_meta_weights(tracker_weights)
    init_net.eval()
    pos_score = init_net.forward(pos_regions_var)
    neg_score = init_net.forward(neg_regions_var)
    acc,acc_pos,acc_neg = evaluator(pos_score, neg_score)

    pos_regions_var = Variable(torch.from_numpy(pos_regions))
    neg_regions_var = Variable(torch.from_numpy(neg_regions))
    if opts['use_gpu']:
        pos_regions_var = pos_regions_var.cuda()
        neg_regions_var = neg_regions_var.cuda()
    pos_feats = init_net(pos_regions_var, out_layer='features')
    neg_feats = init_net(neg_regions_var, out_layer='features')
    return pos_feats.data.clone(), neg_feats.data.clone(), init_acc, acc

    def forward(self, input1):
        self.batchgrid3d = torch.zeros(torch.Size([input1.size(0)]) + self.grid3d.size())

        for i in range(input1.size(0)):
            self.batchgrid3d[i] = self.grid3d

        self.batchgrid3d = Variable(self.batchgrid3d)
        #print(self.batchgrid3d)

        x = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,0:4]), 3)
        y = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,4:8]), 3)
        z = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,8:]), 3)
        #print(x)
        r = torch.sqrt(x**2 + y**2 + z**2) + 1e-5

        #print(r)
        theta = torch.acos(z/r)/(np.pi/2)  - 1
        #phi = torch.atan(y/x)
        phi = torch.atan(y/(x + 1e-5))  + np.pi * x.lt(0).type(torch.FloatTensor) * (y.ge(0).type(torch.FloatTensor) - y.lt(0).type(torch.FloatTensor))
        phi = phi/np.pi


        output = torch.cat([theta,phi], 3)

        return output

    def forward(self, images, questions):

        N, T, _, _, _ = images.size()

        # bs x 5 x 3 x 224 x 224
        img_feats = self.cnn(images.contiguous().view(
            -1, images.size(2), images.size(3), images.size(4)))
        img_feats = self.cnn_fc_layer(img_feats)

        img_feats_tr = self.img_tr(img_feats)

        ques_feats = self.q_rnn(questions)
        ques_feats_repl = ques_feats.view(N, 1, -1).repeat(1, T, 1)
        ques_feats_repl = ques_feats_repl.view(N * T, -1)

        ques_feats_tr = self.ques_tr(ques_feats_repl)

        ques_img_feats = torch.cat([ques_feats_tr, img_feats_tr], 1)

        att_feats = self.att(ques_img_feats)
        att_probs = F.softmax(att_feats.view(N, T), dim=1)
        att_probs2 = att_probs.view(N, T, 1).repeat(1, 1, 64)

        att_img_feats = torch.mul(att_probs2, img_feats.view(N, T, 64))
        att_img_feats = torch.sum(att_img_feats, dim=1)

        mul_feats = torch.mul(ques_feats, att_img_feats)

        scores = self.classifier(mul_feats)

        return scores, att_probs

    def __call__(self, image_batch, theta_aff, theta_aff_tps, use_cuda=True):
        
        sampling_grid_aff = self.affTnf(image_batch=None,
                                        theta_batch=theta_aff.view(-1,2,3),
                                        return_sampling_grid=True,
                                        return_warped_image=False)
      
        sampling_grid_aff_tps = self.tpsTnf(image_batch=None,
                                       theta_batch=theta_aff_tps,
                                       return_sampling_grid=True,
                                       return_warped_image=False)
        
        if self.padding_crop_factor is not None:
            sampling_grid_aff_tps = sampling_grid_aff_tps*self.padding_crop_factor;

        # put 1e10 value in region out of bounds of sampling_grid_aff
        in_bound_mask_aff = ((sampling_grid_aff[:,:,:,0]>-1) * (sampling_grid_aff[:,:,:,0]<1) * (sampling_grid_aff[:,:,:,1]>-1) * (sampling_grid_aff[:,:,:,1]<1)).unsqueeze(3)
        in_bound_mask_aff = in_bound_mask_aff.expand_as(sampling_grid_aff)
        sampling_grid_aff = torch.mul(in_bound_mask_aff.float(),sampling_grid_aff)
        sampling_grid_aff = torch.add((in_bound_mask_aff.float()-1)*(1e10),sampling_grid_aff)       
        
        # compose transformations
        sampling_grid_aff_tps_comp = F.grid_sample(sampling_grid_aff.transpose(2,3).transpose(1,2), sampling_grid_aff_tps).transpose(1,2).transpose(2,3)
            
        # put 1e10 value in region out of bounds of sampling_grid_aff_tps_comp
        in_bound_mask_aff_tps=((sampling_grid_aff_tps[:,:,:,0]>-1) * (sampling_grid_aff_tps[:,:,:,0]<1) * (sampling_grid_aff_tps[:,:,:,1]>-1) * (sampling_grid_aff_tps[:,:,:,1]<1)).unsqueeze(3)
        in_bound_mask_aff_tps=in_bound_mask_aff_tps.expand_as(sampling_grid_aff_tps_comp)
        sampling_grid_aff_tps_comp=torch.mul(in_bound_mask_aff_tps.float(),sampling_grid_aff_tps_comp)
        sampling_grid_aff_tps_comp = torch.add((in_bound_mask_aff_tps.float()-1)*(1e10),sampling_grid_aff_tps_comp)       

        # sample transformed image
        warped_image_batch = F.grid_sample(image_batch, sampling_grid_aff_tps_comp)
        
        return warped_image_batch

 def forward(self, theta, matches, return_outliers=False):
     if isinstance(theta,Variable): # handle normal batch transformations
         batch_size=theta.size()[0]
         theta=theta.clone()
         mask = self.geometricTnf(expand_dim(self.mask_id,0,batch_size),theta)
         if return_outliers:
             mask_outliers = self.geometricTnf(expand_dim(1.0-self.mask_id,0,batch_size),theta)
         if self.normalize:
             epsilon=1e-5
             mask = torch.div(mask,
                              torch.sum(torch.sum(torch.sum(mask+epsilon,3),2),1).unsqueeze(1).unsqueeze(2).unsqueeze(3).expand_as(mask))
             if return_outliers:
                 mask_outliers = torch.div(mask_outliers,
                                           torch.sum(torch.sum(torch.sum(mask_outliers+epsilon,3),2),1).unsqueeze(1).unsqueeze(2).unsqueeze(3).expand_as(mask_outliers))
         score = torch.sum(torch.sum(torch.sum(torch.mul(mask,matches),3),2),1)
         if return_outliers:
             score_outliers = torch.sum(torch.sum(torch.sum(torch.mul(mask_outliers,matches),3),2),1)
             return (score,score_outliers)
     elif isinstance(theta,list): # handle multiple transformations per batch item, batch is in list format (used for RANSAC)
         batch_size = len(theta)
         score = []
         for b in range(batch_size):
             sample_size=theta[b].size(0)
             s=self.forward(theta[b],expand_dim(matches[b,:,:,:].unsqueeze(0),0,sample_size))
             score.append(s)
     return score

    def forward(self, title, pg):

        r_gate = F.sigmoid(self.wrx(title) + self.wrh(pg))
        i_gate = F.sigmoid(self.wix(title) + self.wih(pg))
        n_gate = F.tanh(self.wnx(title) + torch.mul(r_gate, self.wnh(pg)))
        result =  torch.mul(i_gate, pg) + torch.mul(torch.add(-i_gate, 1), n_gate)
        return result

    def updateGradInput(self, input, gradOutput):
        v1 = input[0]
        v2 = input[1]
        v1, v2 = self._makeContiguous(v1, v2)

        if len(self.gradInput) != 2:
            if self.gradInput[0] is None:
                self.gradInput[0] = v1.new()
            if self.gradInput[1] is None:
                self.gradInput[1] = v1.new()
            self.gradInput = self.gradInput[:2]

        gw1 = self.gradInput[0]
        gw2 = self.gradInput[1]
        gw1.resize_as_(v1).copy_(v2)
        gw2.resize_as_(v1).copy_(v1)

        torch.mul(self.w1, self.w22, out=self.buffer)
        gw1.addcmul_(-1, self.buffer.expand_as(v1), v1)
        gw1.mul_(self.w.expand_as(v1))

        torch.mul(self.w1, self.w32, out=self.buffer)
        gw2.addcmul_(-1, self.buffer.expand_as(v1), v2)
        gw2.mul_(self.w.expand_as(v1))

        go = gradOutput.contiguous().view(-1, 1).expand_as(v1)
        gw1.mul_(go)
        gw2.mul_(go)

        return self.gradInput

    def updateGradInput(self, input, y):
        v1 = input[0]
        v2 = input[1]

        gw1 = self.gradInput[0]
        gw2 = self.gradInput[1]
        gw1.resize_as_(v1).copy_(v2)
        gw2.resize_as_(v1).copy_(v1)

        torch.mul(self.w1, self.w22, out=self.buffer)
        gw1.addcmul_(-1, self.buffer.expand_as(v1), v1)
        gw1.mul_(self.w.expand_as(v1))

        torch.mul(self.w1, self.w32, out=self.buffer)
        gw2.addcmul_(-1, self.buffer.expand_as(v1), v2)
        gw2.mul_(self.w.expand_as(v1))

        # self._idx = self._outputs <= 0
        torch.le(self._outputs, 0, out=self._idx)
        self._idx = self._idx.view(-1, 1).expand(gw1.size())
        gw1[self._idx] = 0
        gw2[self._idx] = 0

        torch.eq(y, 1, out=self._idx)
        self._idx = self._idx.view(-1, 1).expand(gw2.size())
        gw1[self._idx] = gw1[self._idx].mul_(-1)
        gw2[self._idx] = gw2[self._idx].mul_(-1)

        if self.sizeAverage:
            gw1.div_(y.size(0))
            gw2.div_(y.size(0))

        return self.gradInput

    def forward(self, x):
        x = self.embed(x)
        x = self.dropout(x)
        # x = x.view(len(x), x.size(1), -1)
        # x = embed.view(len(x), embed.size(1), -1)
        bilstm_out, self.hidden = self.bilstm(x, self.hidden)

        bilstm_out = torch.transpose(bilstm_out, 0, 1)
        bilstm_out = torch.transpose(bilstm_out, 1, 2)
        # bilstm_out = F.max_pool1d(bilstm_out, bilstm_out.size(2)).squeeze(2)
        bilstm_out = F.max_pool1d(bilstm_out, bilstm_out.size(2))
        bilstm_out = bilstm_out.squeeze(2)

        hidden2lable = self.hidden2label1(F.tanh(bilstm_out))

        gate_layer = F.sigmoid(self.gate_layer(bilstm_out))
        # calculate highway layer values
        gate_hidden_layer = torch.mul(hidden2lable, gate_layer)
        # if write like follow ,can run,but not equal the HighWay NetWorks formula
        # gate_input = torch.mul((1 - gate_layer), hidden2lable)
        gate_input = torch.mul((1 - gate_layer), bilstm_out)
        highway_output = torch.add(gate_hidden_layer, gate_input)

        logit = self.logit_layer(highway_output)

        return logit

 def forward(self, inputs, targets, step, weight_constraint_lambda, logger):
     n = inputs.size(0)
     # Compute pairwise distance, replace by the official when merged
     # features = F.normalize(inputs)
     features = inputs
     dist = torch.pow(features, 2).sum(dim=1, keepdim=True).expand(n, n)
     dist = dist + dist.t()
     dist.addmm_(1, -2, features, features.t())
     dist = dist.clamp(min=1e-12).sqrt()  # for numerical stability
     # get the positive label mask
     mask = targets.expand(n, n).eq(targets.expand(n, n).t())
     mask = mask.float()
     positive_dist = torch.mul(dist, mask)
     negative_dist = torch.mul(mask, dist.max()) + torch.mul(dist, 1 - mask)
     indexes_ap = []
     indexes_ng = []
     dist_ap = []
     dist_an = []
     for i in range(n):
         pos_dist, pos_index = positive_dist[i].max(0)
         neg_dist, neg_index = negative_dist[i].min(0)
         dist_ap.append(pos_dist)
         dist_an.append(neg_dist)
         indexes_ap.append(pos_index)
         indexes_ng.append(neg_index)
     dist_ap = torch.cat(dist_ap)
     dist_an = torch.cat(dist_an)
     indexes_ap = torch.cat(indexes_ap)
     indexes_ng = torch.cat(indexes_ng)
     pair_adp_inputs = []
     for i in range(n):
         pair_adp_inputs.append(torch.cat([inputs[i, :], inputs[indexes_ap.data[i], :]]))
     # for i in range(n):
         pair_adp_inputs.append(torch.cat([inputs[i, :], inputs[indexes_ng.data[i], :]]))
     pair_adp_inputs = torch.stack(pair_adp_inputs)
     # Compute adp_pairwise distance, replace by the official when merged
     dist_adp = self.AdpsubM(pair_adp_inputs, n)  # [2*batchsize] [ap,ng]*batchsize
     # dist_constraint = torch.norm(dist-dist.t())
     dist_ap_adp = dist_adp[::2]
     dist_an_adp = dist_adp[1::2]
     # Compute ranking hinge loss
     y = dist_an.data.new()
     y.resize_as_(dist_an.data)
     y.fill_(1)
     y = Variable(y)
     # dist_neg_constr = 1/torch.norm(dist[mask==0])
     trip_loss = self.softmargin_loss(dist_an - dist_ap, y)
     trip_loss_adp = self.softmargin_loss(dist_an_adp - dist_ap_adp, y)
     loss = trip_loss + trip_loss_adp
     # loss = trip_loss
     if logger:
         # logger.scalar_summary('Metric_constraint', Metric_constraint.data[0], step)
         # logger.scalar_summary('dist_constraint', dist_constraint.data[0], step)
         # logger.histo_summary('W',W.data.cpu().numpy(),step)
         logger.histo_summary('dist_apt', dist_adp.data.cpu().numpy(), step)
         logger.histo_summary('dist', dist.data.cpu().numpy(), step)
         logger.scalar_summary('trip_loss', trip_loss.data[0], step)
     prec = (dist_an.data > dist_ap.data).sum() * 1. / y.size(0)
     return trip_loss_adp, prec

 def forward(self, img, att_size=14):
     x0 = self.conv(img)
     x = self.pool_mil(x0)
     x = x.squeeze(2).squeeze(2)
     x = self.l1(x)
     x1 = torch.add(torch.mul(x.view(x.size(0), 1000, -1), -1), 1)
     cumprod = torch.cumprod(x1, 2)
     out = torch.max(x, torch.add(torch.mul(cumprod[:, :, -1], -1), 1))
     return out

 def forward(self, x):
     x0 = self.conv.forward(x.float())
     x = self.pool_mil(x0)
     x = x.squeeze(2).squeeze(2)
     x1 = torch.add(torch.mul(x0.view(x.size(0), 1000, -1), -1), 1)
     cumprod = torch.cumprod(x1, 2)
     out = torch.max(x, torch.add(torch.mul(cumprod[:, :, -1], -1), 1))
     #out = F.softmax(out)
     return out

    def updateOutput(self, input):
        input1, input2 = input[0], input[1]

        if self.buffer is None:
            self.buffer = input1.new()

        torch.mul(input1, input2, out=self.buffer)
        torch.sum(self.buffer, 1, True, out=self.output)
        self.output.resize_(input1.size(0))
        return self.output

    def custom_cross_entropy(x, y):
        sigmoid_x = torch.sigmoid(x)
        sigmoid_x2 = torch.sigmoid(x ** 2)
        neg_log_sigmoid_x = -1 * torch.log(sigmoid_x)
        neg_log_1_minus_sigmoid_x2 = -1 * torch.log(1 - sigmoid_x2)

        l1 = torch.mul(y, neg_log_sigmoid_x)
        l2 = torch.mul(1 - y, neg_log_1_minus_sigmoid_x2)

        return torch.sum(l1 + l2)

    def forward(self, x):
        bahs, chs, _, _ = x.size()

        # Returns a new tensor with the same data as the self tensor but of a different size.
        chn_se = self.avg_pool(x).view(bahs, chs)
        chn_se = self.channel_excitation(chn_se).view(bahs, chs, 1, 1)
        chn_se = torch.mul(x, chn_se)

        spa_se = self.spatial_se(x)
        spa_se = torch.mul(x, spa_se)
        return torch.add(chn_se, 1, spa_se)

    def updateOutput(self, input):
        gaterInput, expertInputs = input

        # buffers
        if self._gaterView is None:
            self._gaterView = input[0].new()
        if self._expert is None:
            self._expert = input[0].new()
        if self._expertView is None:
            self._expertView = input[0].new()

        self.dimG = 1
        batchSize = gaterInput.size(0)

        if self.table or isinstance(expertInputs, list):
            self.table = True
            if gaterInput.size(self.dimG) != len(expertInputs):
                raise RuntimeError("Should be one gater output per expert")

            expertInput = expertInputs[0]
            if self.batchSize != batchSize:
                size = [1] * (expertInput.dim() + 1)
                if self.dimG > 0:
                    size[0] = gaterInput.size(0)
                size[self.dim] = gaterInput.size(self.dimG)
                self.size = torch.Size(size)
                self.output.resize_as_(expertInput)
                self.backwardSetup = False
                self.batchSize = batchSize

            self._gaterView = gaterInput.view(self.size)
            self.output.zero_()
            # multiply accumulate gater outputs by their commensurate expert
            for i, expertInput in enumerate(expertInputs):
                gate = self._gaterView.select(self.dim, i).expand_as(expertInput)
                self.output.addcmul_(expertInput, gate)
        else:
            if self.batchSize != batchSize:
                size = [1] * expertInputs.dim()
                if self.dimG > 0:
                    size[0] = gaterInput.size(0)
                size[self.dim] = gaterInput.size(self.dimG)
                self.size = torch.Size(size)
                self.output.resize_as_(expertInputs.select(self.dim, 0))
                self.batchSize = batchSize
                self.backwardSetup = False

            self._gaterView = gaterInput.view(self.size)
            torch.mul(self._gaterView.expand_as(expertInputs), expertInputs, out=self._expert)
            torch.sum(self._expert, self.dim, True, out=self.output)
            self.output.resize_as_(expertInputs.select(self.dim, 0))

        return self.output

    def accGradParameters(self, input, gradOutput, scale=1):
        inputSize, outputSize = self.weight.size(0), self.weight.size(1)

        """
        dy_j   2 * c_j * c_j * (w_j - x)    c_j * c_j * (w_j - x)
        ---- = -------------------------- = ---------------------
        dw_j    2 || c_j * (w_j - x) ||             y_j

        dy_j    2 * c_j * (w_j - x)^2    c_j * (w_j - x)^2
        ---- = ----------------------- = -----------------
        dc_j   2 || c_j * (w_j - x) ||         y_j
        #"""
        # assumes a preceding call to updateGradInput
        if input.dim() == 1:
            self.gradWeight.add_(-scale, self._repeat2)

            self._repeat.div_(self.diagCov)
            self._repeat.mul_(self._repeat)
            self._repeat.mul_(self.diagCov)

            if torch.type(input) == 'torch.cuda.FloatTensor':
                self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
                self._repeat2.mul_(self._repeat)
            else:
                torch.mul(self._repeat, self._expand4, out=self._repeat2)

            self.gradDiagCov.add_(self._repeat2)
        elif input.dim() == 2:
            if self._sum is None:
                self._sum = input.new()
            torch.sum(self._repeat2, 0, True, out=self._sum)
            self._sum.resize_(inputSize, outputSize)
            self.gradWeight.add_(-scale, self._sum)

            if input.type() == 'torch.cuda.FloatTensor':
                # requires lots of memory, but minimizes cudaMallocs and loops
                self._repeat.div_(self._repeat3)
                self._repeat.mul_(self._repeat)
                self._repeat.mul_(self._repeat3)
                self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
                self._repeat.mul_(self._repeat2)
            else:
                self._repeat.div_(self._expand3)
                self._repeat.mul_(self._repeat)
                self._repeat.mul_(self._expand3)
                self._repeat.mul_(self._expand4)

            torch.sum(self._repeat, 0, True, out=self._sum)
            self._sum.resize_(inputSize, outputSize)
            self.gradDiagCov.add_(scale, self._sum)
        else:
            raise RuntimeError("1D or 2D input expected")

    def accGradParameters(self, input, gradOutput, scale=1):
        if self._input is None:
            self._input = input.new()
            self._gradWeight = input.new()
            self._sum = input.new()

        batchSize = input.size(0)
        contiguousView(self._input, input, batchSize, -1)
        contiguousView(self._gradOutput, gradOutput, batchSize, -1)
        self._gradWeight = self.gradWeight.view(1, -1)

        torch.mul(self._input, self._gradOutput, out=self._repeat)
        torch.sum(self._repeat, 0, True, out=self._sum)
        self._gradWeight.add_(scale, self._sum)

    def updateOutput(self, input):
        input1, input2 = input[0], input[1]
        input1, input2 = self._makeContiguous(input1, input2)

        if self.buffer is None:
            self.buffer = input1.new()
            self.w1 = input1.new()
            self.w22 = input1.new()
            self.w = input1.new()
            self.w32 = input1.new()
            self.ones = input1.new()

        torch.mul(input1, input2, out=self.buffer)
        torch.sum(self.buffer, 1, out=self.w1, keepdim=True)

        epsilon = 1e-12
        torch.mul(input1, input1, out=self.buffer)
        torch.sum(self.buffer, 1, out=self.w22, keepdim=True).add_(epsilon)
        self.w22.reciprocal_()
        self.w.resize_as_(self.w22).copy_(self.w22)

        torch.mul(input2, input2, out=self.buffer)
        torch.sum(self.buffer, 1, out=self.w32, keepdim=True).add_(epsilon)
        self.w32.reciprocal_()
        self.w.mul_(self.w32)
        self.w.sqrt_()

        torch.mul(self.w1, self.w, out=self.output)
        self.output.resize_(input1.size(0))

        return self.output

    def accGradParameters(self, input, gradOutput, scale=1):
        self._assertInputGradOutput(input, gradOutput)

        # make sure we have buffer:
        if self.buff1 is None:
            self.buff1 = input[0].new()
        self.buff1.resize_as_(input[0])

        # accumulate parameter gradients:
        for k in range(self.weight.size(0)):
            torch.mul(input[0], gradOutput.narrow(1, k, 1).expand_as(input[0]), out=self.buff1)
            self.gradWeight[k].addmm_(self.buff1.t(), input[1])

        if self.bias is not None:
            self.gradBias.add_(scale, gradOutput.sum(0, keepdim=False))