def forward(self, y_pred, y_true):
        torch.nn.modules.loss._assert_no_grad(y_true)

        y_pred_log = torch.log(y_pred)
        start_loss = F.nll_loss(y_pred_log[:, 0, :], y_true[:, 0])
        end_loss = F.nll_loss(y_pred_log[:, 1, :], y_true[:, 1])
        return start_loss + end_loss

def train(epoch):
    t = time.time()
    model.train()
    optimizer.zero_grad()
    output = model(features, adj)
    loss_train = F.nll_loss(output[idx_train], labels[idx_train])
    acc_train = accuracy(output[idx_train], labels[idx_train])
    loss_train.backward()
    optimizer.step()

    if not args.fastmode:
        # Evaluate validation set performance separately,
        # deactivates dropout during validation run.
        model.eval()
        output = model(features, adj)

    loss_val = F.nll_loss(output[idx_val], labels[idx_val])
    acc_val = accuracy(output[idx_val], labels[idx_val])
    print('Epoch: {:04d}'.format(epoch+1),
          'loss_train: {:.4f}'.format(loss_train.data[0]),
          'acc_train: {:.4f}'.format(acc_train.data[0]),
          'loss_val: {:.4f}'.format(loss_val.data[0]),
          'acc_val: {:.4f}'.format(acc_val.data[0]),
          'time: {:.4f}s'.format(time.time() - t))

    return loss_val.data[0]

def train(epoch, model):
    #最后的全连接层学习率为前面的10倍
    LEARNING_RATE = lr / math.pow((1 + 10 * (epoch - 1) / epochs), 0.75)
    print("learning rate：", LEARNING_RATE)
    optimizer_fea = torch.optim.SGD([
        {'params': model.sharedNet.parameters()},
        {'params': model.cls_fc.parameters(), 'lr': LEARNING_RATE},
    ], lr=LEARNING_RATE / 10, momentum=momentum, weight_decay=l2_decay)
    optimizer_critic = torch.optim.SGD([
        {'params': model.domain_fc.parameters(), 'lr': LEARNING_RATE}
    ], lr=LEARNING_RATE, momentum=momentum, weight_decay=l2_decay)

    data_source_iter = iter(source_loader)
    data_target_iter = iter(target_train_loader)
    dlabel_src = Variable(torch.ones(batch_size).long().cuda())
    dlabel_tgt = Variable(torch.zeros(batch_size).long().cuda())
    i = 1
    while i <= len_source_loader:
        model.train()

        source_data, source_label = data_source_iter.next()
        if cuda:
            source_data, source_label = source_data.cuda(), source_label.cuda()
        source_data, source_label = Variable(source_data), Variable(source_label)
        clabel_src, dlabel_pred_src = model(source_data)
        label_loss = F.nll_loss(F.log_softmax(clabel_src, dim=1), source_label)
        critic_loss_src = F.nll_loss(F.log_softmax(dlabel_pred_src, dim=1), dlabel_src)
        confusion_loss_src = 0.5 * ( F.nll_loss(F.log_softmax(dlabel_pred_src, dim=1), dlabel_src) + F.nll_loss(F.log_softmax(dlabel_pred_src, dim=1), dlabel_tgt) )

        target_data, target_label = data_target_iter.next()
        if i % len_target_loader == 0:
            data_target_iter = iter(target_train_loader)
        if cuda:
            target_data, target_label = target_data.cuda(), target_label.cuda()
        target_data = Variable(target_data)
        clabel_tgt, dlabel_pred_tgt = model(target_data)
        critic_loss_tgt = F.nll_loss(F.log_softmax(dlabel_pred_tgt, dim=1), dlabel_tgt)
        confusion_loss_tgt = 0.5 * (F.nll_loss(F.log_softmax(dlabel_pred_tgt, dim=1), dlabel_src) + F.nll_loss(
            F.log_softmax(dlabel_pred_tgt, dim=1), dlabel_tgt))

        confusion_loss_total = (confusion_loss_src + confusion_loss_tgt) / 2
        fea_loss_total = confusion_loss_total + label_loss
        critic_loss_total = (critic_loss_src + critic_loss_tgt) / 2

        optimizer_fea.zero_grad()
        fea_loss_total.backward(retain_graph=True)
        optimizer_fea.step()
        optimizer_fea.zero_grad()
        optimizer_critic.zero_grad()
        critic_loss_total.backward()
        optimizer_critic.step()

        if i % log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tconfusion_Loss: {:.6f}\tlabel_Loss: {:.6f}\tdomain_Loss: {:.6f}'.format(
                epoch, i * len(source_data),len_source_dataset,
                100. * i / len_source_loader, confusion_loss_total.data[0], label_loss.data[0], critic_loss_total.data[0]))
        i = i + 1

    def get_loss(cls, start_log_probs, end_log_probs, starts, ends):
        """
        Get the loss, $-\log P(s|p,q)P(e|p,q)$.
        The start and end labels are expected to be in span format,
        so that text[start:end] is the answer.
        """

        # Subtracts 1 from the end points, to get the exact indices, not 1
        # after the end.
        loss = nll_loss(start_log_probs, starts) +\
            nll_loss(end_log_probs, ends-1)
        return loss

def train(args, epoch, net, trainLoader, optimizer, trainF):
    net.train()
    nProcessed = 0
    nTrain = len(trainLoader.dataset)
    for batch_idx, (data, target) in enumerate(trainLoader):
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = net(data)
        loss = F.nll_loss(output, target)
        # make_graph.save('/tmp/t.dot', loss.creator); assert(False)
        loss.backward()
        optimizer.step()
        nProcessed += len(data)
        pred = output.data.max(1)[1] # get the index of the max log-probability
        incorrect = pred.ne(target.data).cpu().sum()
        err = 100.*incorrect/len(data)
        partialEpoch = epoch + batch_idx / len(trainLoader) - 1
        print('Train Epoch: {:.2f} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tError: {:.6f}'.format(
            partialEpoch, nProcessed, nTrain, 100. * batch_idx / len(trainLoader),
            loss.data[0], err))

        trainF.write('{},{},{}\n'.format(partialEpoch, loss.data[0], err))
        trainF.flush()

  def _test_pytorch(self, model):
    """
    Test pre-trained pytorch model using MNIST Dataset
    :param model: Pre-trained PytorchMNIST model
    :return: tuple(loss, accuracy)
    """
    data_loader = torch.utils.data.DataLoader(
      datasets.MNIST(self.dataDir, train=False, download=True,
                     transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))])),
      batch_size=BATCH_SIZE, shuffle=True)

    model.eval()
    loss = 0.0
    num_correct = 0.0
    with torch.no_grad():
      for data, target in data_loader:
        data = data.view(-1, 28 * 28)
        output = model(data)
        loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
        pred = output.max(1, keepdim=True)[1]  # get the index of the max log-probability
        num_correct += pred.eq(target.view_as(pred)).sum().item()

    loss /= len(data_loader.dataset)
    accuracy = num_correct / len(data_loader.dataset)

    return (loss, accuracy)

def test(model, device, test_loader, epoch):
    losses = AverageMeter()
    top1 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    for batch_idx, (data, target) in enumerate(test_loader):
        data, target = data.to(device), target.to(device)

        # compute output
        with torch.no_grad():
            output = model(data)
        loss = F.nll_loss(output, target)

        # measure accuracy and record loss
        prec1 = accuracy(output, target, topk=(1,))[0]
        losses.update(loss.item(), data.size(0))
        top1.update(prec1.item(), data.size(0))

        if batch_idx % args.print_freq == 0:
            print('Test: [{0}/{1}]\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  '[email protected] {top1.val:.3f} ({top1.avg:.3f})'.format(
                      batch_idx, len(test_loader), loss=losses,
                      top1=top1))

    print(' * [email protected] {top1.avg:.3f}'.format(top1=top1))
    return top1.avg

def test(epoch, best_acc):
    slope = get_slope(epoch)

    model.eval()
    test_loss = 0.0
    correct = 0.0
    for data, target in test_loader:
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model((data, slope))
        test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss
        pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
        correct += pred.eq(target.data.view_as(pred)).cpu().sum()

    test_loss /= len(test_loader.dataset)
    test_acc = correct / len(test_loader.dataset)
    print 'Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
          test_loss, int(correct), len(test_loader.dataset),
          100. * test_acc)

    if test_acc >= best_acc:
        torch.save(model.state_dict(), os.path.join('models','{}.pth'.format(model_name)))

    return test_loss, test_acc

def train(epoch, model):
    LEARNING_RATE = lr / math.pow((1 + 10 * (epoch - 1) / epochs), 0.75)
    print('learning rate{: .4f}'.format(LEARNING_RATE) )
    optimizer = torch.optim.SGD([
        {'params': model.sharedNet.parameters()},
        {'params': model.cls_fc.parameters(), 'lr': LEARNING_RATE},
        ], lr=LEARNING_RATE / 10, momentum=momentum, weight_decay=l2_decay)

    model.train()

    iter_source = iter(source_loader)
    iter_target = iter(target_train_loader)
    num_iter = len_source_loader
    for i in range(1, num_iter):
        data_source, label_source = iter_source.next()
        data_target, _ = iter_target.next()
        if i % len_target_loader == 0:
            iter_target = iter(target_train_loader)
        if cuda:
            data_source, label_source = data_source.cuda(), label_source.cuda()
            data_target = data_target.cuda()
        data_source, label_source = Variable(data_source), Variable(label_source)
        data_target = Variable(data_target)

        optimizer.zero_grad()
        label_source_pred, loss_mmd = model(data_source, data_target)
        loss_cls = F.nll_loss(F.log_softmax(label_source_pred, dim=1), label_source)
        gamma = 2 / (1 + math.exp(-10 * (epoch) / epochs)) - 1
        loss = loss_cls + gamma * loss_mmd
        loss.backward()
        optimizer.step()
        if i % log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tsoft_Loss: {:.6f}\tmmd_Loss: {:.6f}'.format(
                epoch, i * len(data_source), len_source_dataset,
                100. * i / len_source_loader, loss.data[0], loss_cls.data[0], loss_mmd.data[0]))

  def train(self, epoch):
    """
    Train one epoch of this model by iterating through mini batches. An epoch
    ends after one pass through the training set, or if the number of mini
    batches exceeds the parameter "batches_in_epoch".
    """

    self.logger.info("epoch: %s", epoch)

    t0 = time.time()
    self.preEpoch()

    self.logger.info("Learning rate: %s",
                     self.learningRate if self.lr_scheduler is None
                     else self.lr_scheduler.get_lr())

    self.model.train()
    for batch_idx, (batch, target) in enumerate(self.train_loader):
      data = batch["input"]
      if self.model_type in ["resnet9", "cnn"]:
        data = torch.unsqueeze(data, 1)
      data, target = data.to(self.device), target.to(self.device)
      self.optimizer.zero_grad()
      output = self.model(data)
      loss = F.nll_loss(output, target)
      loss.backward()
      self.optimizer.step()

      if batch_idx >= self.batches_in_epoch:
        break

    self.postEpoch()

    self.logger.info("training duration: %s", time.time() - t0)

def train(epoch):

    slope = get_slope(epoch)

    print '# Epoch : {} - Slope : {}'.format(epoch, slope)

    model.train()
    train_loss = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model((data, slope))
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()

        train_loss += loss.data

    train_loss /= len(train_loader)
    train_loss = train_loss[0]

    print 'Training Loss : {}'.format(train_loss)

    return train_loss

def train(model, device, train_loader, optimizer, epoch):
    """Train for one epoch on the training set"""
    losses = AverageMeter()
    top1 = AverageMeter()

    # switch to train mode
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)

        # compute output
        output = model(data)
        loss = F.nll_loss(output, target)

        # measure accuracy and record loss
        prec1 = accuracy(output, target, topk=(1,))[0]
        losses.update(loss.item(), data.size(0))
        top1.update(prec1.item(), data.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch_idx % args.print_freq == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  '[email protected] {top1.val:.3f} ({top1.avg:.3f})'.format(
                      epoch, batch_idx, len(train_loader), loss=losses, top1=top1))

def evaluate():
    should_stop = False
    model.eval()

    for name, loader in [('train', train_loader), ('test', test_loader)]:
        loss = 0
        correct = 0
        for data, target in loader:
            if args.cuda:
                data, target = data.cuda(), target.cuda()
            if isinstance(model, MLP):
                data = data.view(-1, 784)
            data, target = Variable(data, volatile=True), Variable(target)
            output = model(data)
            loss += F.nll_loss(output, target, size_average=False).data[0]
            # get the index of the max log-probability
            pred = output.data.max(1, keepdim=True)[1]
            correct += pred.eq(target.data.view_as(pred)).cpu().sum()

        loss /= len(loader.dataset)
        print('{} -- Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'
              .format(name.ljust(5), loss, correct, len(loader.dataset),
                      100. * correct / len(loader.dataset)))
        if name == 'test':
            scheduler.step(loss)
            should_stop = should_stop or correct == len(loader.dataset)
    return should_stop or optimizer.param_groups[0]['lr'] < args.lr / 1e2

def m_testxxx(epoch):
    # checkpoint = torch.load('checkpoint-1.pth.tar')
    # model.load_state_dict(checkpoint['state_dict'])
    # optimizer.load_state_dict(checkpoint['optimizer'])
    #
    model.eval()
    test_loss = 0
    correct = 0
    for data, target in test_loader:
        # x_data = data[0].numpy()
        # x_data = np.reshape(x_data, [28, 28])
        # np.savetxt('./data.csv', x_data)
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model(data)
        test_loss += F.nll_loss(output, target).data[0]
        pred = output.data.max(1)[1]  # get the index of the max log-probability
        #result = pred.numpy()
        #np.reshape(result, [-1, 1])
        #print(result.shape)
        # print(pred)
        correct += pred.eq(target.data).cpu().sum()

    test_loss = test_loss
    test_loss /= len(test_loader)  # loss function already averages over batch size
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

  def test(self, test_loader=None):
    """
    Test the model using the given loader and return test metrics
    """
    if test_loader is None:
      test_loader = self.test_loader

    self.model.eval()
    test_loss = 0
    correct = 0

    with torch.no_grad():
      for batch, target in test_loader:
        data = batch["input"]
        if self.model_type in ["resnet9", "cnn"]:
          data = torch.unsqueeze(data, 1)
        data, target = data.to(self.device), target.to(self.device)
        output = self.model(data)
        test_loss += F.nll_loss(output, target, reduction='sum').item()
        pred = output.max(1, keepdim=True)[1]
        correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.sampler)
    test_error = 100. * correct / len(test_loader.sampler)

    entropy = self.entropy()
    ret = {
      "total_correct": correct,
      "mean_loss": test_loss,
      "mean_accuracy": test_error,
      "entropy": float(entropy)}

    return ret

def cross_entropy2d(input, target, weight=None, size_average=True):
    """
    Function to compute pixelwise cross-entropy for 2D image. This is the segmentation loss.
    Args:
        input: input tensor of shape (minibatch x num_channels x h x w)
        target: 2D label map of shape (minibatch x h x w)
        weight (optional): tensor of size 'C' specifying the weights to be given to each class
        size_average (optional): boolean value indicating whether the NLL loss has to be normalized
            by the number of pixels in the image 
    """
    
    # input: (n, c, h, w), target: (n, h, w)
    n, c, h, w = input.size()
    
    # log_p: (n, c, h, w)
    log_p = F.log_softmax(input)
    
    # log_p: (n*h*w, c)
    log_p = log_p.transpose(1, 2).transpose(2, 3).contiguous().view(-1, c)
    try:
        log_p = log_p[target.view(n, h, w, 1).repeat(1, 1, 1, c) >= 0]
    except:
        print "Exception: ", target.size()
    log_p = log_p.view(-1, c)
    
    # target: (n*h*w,)
    mask = target >= 0
    target = target[mask]
    target = torch.squeeze(target)
    loss = F.nll_loss(log_p, target, weight=weight, size_average=False)
    if size_average:
        loss /= mask.data.sum()

    return loss

    def get_combined_loss(cls, combined, starts, ends):
        """
        Get the loss, $-\log P(s,e|p,q)$.
        In practice, with:
            1. $\Psi_s(s|p,q)$ the start logits,
            2. $\Psi_e(e|p,q)$ the end logits,
            3. $Z_s = \log\sum_{i}\exp\Psi_s(i|p,q)$, the start partition,
            4. $Z_e = \log\sum_{i}\exp\Psi_e(i|p,q)$, the end partition, and
            5. $Z_c = \log\sum_{i}\sum{j>=i}\exp(\Psi_s(i|p,q)+\Psi_e(i|p,q))$,
            the combined partition,
        the default loss is:
            $Z_s + Z_e - \Psi_s(s|p,q) - \Psi_e(e|p,q)$,
        and the combined loss is:
            $Z_c - \Psi_s(s|p,q) - \Psi_e(e|p,q)$.

        The combined loss uses a normalization that ignores invalid end points.
        This is not a major difference, and should only slow things down during
        training.
        This loss is only used to validate and to compare.
        """
        batch_size, num_tokens, _other = combined.size()
        assert num_tokens == _other
        mask = torch.zeros(batch_size, num_tokens, num_tokens).float()
        for start in range(1, num_tokens):
            mask[:, start, :start] = -1e20
        mask = mask.type_as(combined.data)
        combined = combined + Variable(mask)
        combined = combined.view(batch_size, num_tokens*num_tokens)
        combined = nn.functional.log_softmax(combined, dim=1)
        labels = starts * num_tokens + ends
        return nll_loss(combined, labels)

def train(args, model, device, train_loader, optimizer):
    model.train()
    start_time = time()

    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            percentage = 100. * batch_idx / len(train_loader)
            cur_length = int((tracker_length * int(percentage)) / 100)
            bar = '=' * cur_length + '>' + '-' * (tracker_length - cur_length)
            sys.stdout.write('\r{}/{} [{}] - loss: {:.4f}'.format(
                batch_idx * len(data), len(train_loader.dataset),
                bar, loss.item()))
            sys.stdout.flush()

    train_time = time() - start_time
    sys.stdout.write('\r{}/{} [{}] - {:.1f}s {:.1f}us/step - loss: {:.4f}'.format(
        len(train_loader.dataset), len(train_loader.dataset), '=' * tracker_length, 
        train_time, (train_time / len(train_loader.dataset)) * 1000000.0, loss.item()))
    sys.stdout.flush()

    return len(train_loader.dataset), train_time, loss.item()

def train(**kwargs):
    opt.parse(kwargs)
    vis = Visualizer(opt.env)

    #step1: config model
    model = getattr(Nets,opt.model)()
    if opt.load_model_path:
        model.load(opt.load_model_path)
    if opt.use_gpu:
        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        model.to(device)

    #step2: data
    train_data = imageSentiment(opt.train_path,train = True) #训练集
    val_data = imageSentiment(opt.train_path,train = False) #验证集
    train_dataloader = DataLoader(train_data,batch_size = opt.batch_size,shuffle=True,num_workers = opt.num_workers)
    val_dataloader = DataLoader(val_data,batch_size = opt.batch_size,shuffle=False,num_workers = opt.num_workers)

    #step3: 定义损失函数及优化器
    # criterion = nn.CrossEntropyLoss() #交叉熵损失函数 如果使用该损失函数 则网络最后无需使用softmax函数
    lr = opt.lr
    # optimizer = Optim.Adam(model.parameters(),lr = lr,weight_decay= opt.weight_decay)
    optimizer = Optim.SGD(model.parameters(),lr = 0.001,momentum=0.9,nesterov=True)
    #step4: 统计指标(计算平均损失以及混淆矩阵)
    loss_meter = meter.AverageValueMeter()
    confusion_matrix = meter.ConfusionMeter(7)
    previous_loss = 1e100

    #训练
    for i in range(opt.max_epoch):
        loss_meter.reset()
        confusion_matrix.reset()
        total_loss = 0.
        for ii,(label,data) in tqdm(enumerate(train_dataloader),total=len(train_dataloader)):
            if opt.use_gpu:
                label,data = label.to(device),data.to(device)

            optimizer.zero_grad()
            score = model(data)
            # ps:使用nll_loss和crossentropyloss进行多分类时 target为索引标签即可 无需转为one-hot
            loss = F.nll_loss(score,label)
            total_loss += loss.item()
            loss.backward()
            optimizer.step()

            #更新统计指标以及可视化
            loss_meter.add(loss.item())
            confusion_matrix.add(score.data,label.data)

            if ii%opt.print_freq==opt.print_freq-1:
                vis.plot('loss',loss_meter.value()[0])

        vis.plot('mach avgloss', total_loss/len(train_dataloader))
        model.save()

        #计算验证集上的指标
        val_accuracy = val(model,val_dataloader)

        vis.plot('val_accuracy',val_accuracy)

def compute_test():
    model.eval()
    output = model(features, adj)
    loss_test = F.nll_loss(output[idx_test], labels[idx_test])
    acc_test = accuracy(output[idx_test], labels[idx_test])
    print("Test set results:",
          "loss= {:.4f}".format(loss_test.data[0]),
          "accuracy= {:.4f}".format(acc_test.data[0]))