def forward(self, sentence):
        # print(sentence)       [torch.LongTensor of size 47x64]
        x = self.word_embeddings(sentence)      # [torch.FloatTensor of size 47x64x100]
        x = torch.transpose(x, 0, 1)
        # print(x)            # [torch.FloatTensor of size 32x43x300]
        x = x.unsqueeze(1)
        # x = F.relu(self.convl3(x).squeeze(3))
        x = [F.relu(conv(x)).squeeze(3) for conv in self.convsl]
        # print(x)            # [torch.FloatTensor of size 32x200x41] 40 39 38 37
        # print(type(x))
        # a = torch.cat((a1.data, a2.data), 2)
        x = torch.cat(x, 2)
        # print(x)            # [torch.FloatTensor of size 32x200x195]
        x = torch.transpose(x, 1, 2)
        embeds = torch.transpose(x, 0, 1)
        # print(embeds)       # [torch.FloatTensor of size 195x32x200]
        # embeds = self.word_embeddings()
        # x = embeds.view(len(sentence), self.batch_size, -1)  # torch.Size([43, 64, 300])
        lstm_out, self.hidden = self.lstm(embeds, self.hidden)

        lstm_out = torch.transpose(lstm_out, 0, 1)
        lstm_out = torch.transpose(lstm_out, 1, 2)
        # lstm_out = F.tanh(lstm_out)

        lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2))
        # print(lstm_out.size())
        lstm_out = lstm_out.squeeze(2)

        lstm_out = self.dropout(lstm_out)
        y = self.hidden2label(lstm_out)
        # log_probs = F.log_softmax(y)
        log_probs = y
        return log_probs

    def forward(self, sentence):
        # print(sentence)                                     # [torch.LongTensor of size 42x64]
        x = self.word_embeddings(sentence)
        x = self.dropout_embed(x)
        # print(embeds.size())                                # torch.Size([42, 64, 100])
        # x = embeds.view(len(sentence), self.batch_size, -1)
        print(x.size())                                     # torch.Size([42, 64, 100])
        print(self.hidden)
        lstm_out, self.hidden = self.bnlstm(x, self.hidden)   # lstm_out 10*5*50 hidden 1*5*50 *2
        # print(lstm_out)
        # lstm_out = [F.max_pool1d(i, len(lstm_out)).unsqueeze(2) for i in lstm_out]
        lstm_out = torch.transpose(lstm_out, 0, 1)
        lstm_out = torch.transpose(lstm_out, 1, 2)

        lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2))
        # print(lstm_out.size())
        lstm_out = lstm_out.squeeze(2)
        # y = self.hidden2label(lstm_out)

        #lstm_out = torch.cat(lstm_out, 1)
        # lstm_out = self.dropout(lstm_out)
        # lstm_out = lstm_out.view(len(sentence), -1)
        y = self.hidden2label(F.tanh(lstm_out))
        # log_probs = F.log_softmax(y)
        log_probs = y
        return log_probs

def get_paf_and_heatmap(model, img_raw, scale_search, param_stride=8, box_size=368):
    multiplier = [scale * box_size / img_raw.shape[0] for scale in scale_search]

    heatmap_avg = torch.zeros((len(multiplier), 19, img_raw.shape[0], img_raw.shape[1])).cuda()
    paf_avg = torch.zeros((len(multiplier), 38, img_raw.shape[0], img_raw.shape[1])).cuda()

    for i, scale in enumerate(multiplier):
        img_test = cv2.resize(img_raw, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
        img_test_pad, pad = pad_right_down_corner(img_test, param_stride, param_stride)
        img_test_pad = np.transpose(np.float32(img_test_pad[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5

        feed = Variable(torch.from_numpy(img_test_pad)).cuda()
        output1, output2 = model(feed)

        print(output1.size())
        print(output2.size())

        heatmap = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output2)

        paf = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output1)

        heatmap_avg[i] = heatmap[0].data
        paf_avg[i] = paf[0].data

    heatmap_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(heatmap_avg, 0)), 0, 1), 1, 2).cuda()
    heatmap_avg = heatmap_avg.cpu().numpy()

    paf_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(paf_avg, 0)), 0, 1), 1, 2).cuda()
    paf_avg = paf_avg.cpu().numpy()

    return paf_avg, heatmap_avg

 def score(self, hidden, encoder_output):
     
     if self.method == 'dot':            
         # hidden is 1 by 256
         # encoder_output is 22 by 256
         encoder_output = torch.transpose(encoder_output, 0, 1)
         # encoder_output is 256 by 22
         energy = torch.matmul(hidden, encoder_output)
         return energy
     
     elif self.method == 'general':
         # hidden is 1 by 256
         # encoder_output is 256 by 22
         # encoder_output = torch.transpose(encoder_output, 0, 1)
         hidden = hidden.view(1, -1)
         a = self.attn(encoder_output)
         a = torch.transpose(a, 0, 1)
         energy = torch.matmul(hidden, a)
         return energy
     
     elif self.method == 'concat':
         len_encoder_output = encoder_output.size()[1]
         # hidden is 1 by 256
         # encoder_output is 256 by 22
         hidden = torch.transpose(hidden, 0, 1)
         # hidden is 256 by 1
         hidden = hidden.repeat(hidden_size, len_encoder_output)
         # hidden is 256 by 22
         concat = torch.cat((hidden, encoder_output), dim=0)
         # concat is 512 by 22
         # self.attn(concat) --> 256 by 22
         energy = torch.matmul(self.v, F.tanh(self.attn(concat)))
         return energy

    def forward(self, x):
        x = self.embed(x)
        x = self.dropout(x)
        # x = x.view(len(x), x.size(1), -1)
        # x = embed.view(len(x), embed.size(1), -1)
        bilstm_out, self.hidden = self.bilstm(x, self.hidden)

        bilstm_out = torch.transpose(bilstm_out, 0, 1)
        bilstm_out = torch.transpose(bilstm_out, 1, 2)
        # bilstm_out = F.max_pool1d(bilstm_out, bilstm_out.size(2)).squeeze(2)
        bilstm_out = F.max_pool1d(bilstm_out, bilstm_out.size(2))
        bilstm_out = bilstm_out.squeeze(2)

        hidden2lable = self.hidden2label1(F.tanh(bilstm_out))

        gate_layer = F.sigmoid(self.gate_layer(bilstm_out))
        # calculate highway layer values
        gate_hidden_layer = torch.mul(hidden2lable, gate_layer)
        # if write like follow ,can run,but not equal the HighWay NetWorks formula
        # gate_input = torch.mul((1 - gate_layer), hidden2lable)
        gate_input = torch.mul((1 - gate_layer), bilstm_out)
        highway_output = torch.add(gate_hidden_layer, gate_input)

        logit = self.logit_layer(highway_output)

        return logit

    def forward(self, X, X_mask):
        #X: [m, Tx] m = batch size, Tx = word count
        #print(X.size(), type(X))
        m = X.size()[0]
        Tx = X.size()[1]
        
        #embedding layer
        X = self.embedding(X)
        #X: [m, Tx, embedding_dim] m = batch size, Tx = word count
        #print(X.size(), type(X.data))
        assert X.size() == torch.Size([m, Tx, self.embedding_dim])
        
        #conv layer
        #transpose
        X = torch.transpose(X, 1, 2)
        #print(X.size(), type(X.data))

        X = self.conv(X)
        #print(X.size(), type(X.data))

        #transpose back
        X = torch.transpose(X, 1, 2)
        #print(X.size(), type(X.data))

        assert X.size() == torch.Size([m, Tx, 256])

        #maxpool layer
        #transpose
        X = torch.transpose(X, 1, 2)
        X = self.maxpool(X)
        #print(X.size(), type(X.data))
        #remove dimension
        X = X.squeeze()
        #print(X.size(), type(X.data))
        assert X.size() == torch.Size([m, 256])

        #linear 
        X = self.linear(X)
        #X: [m, 1]
        #print(X.size(), type(X))
        if self.num_classes == 2:
            assert X.size() == torch.Size([m, 1])
        else:
            assert X.size() == torch.Size([m, self.num_classes])
            
        if self.num_classes == 2:
            X = torch.squeeze(X)
            X = self.sigmoid(X)
            #X: [m]
            #print(X.size(), type(X))
            assert X.size() == torch.Size([m])
            return X
        else:
            return F.softmax(X)

    def forward(self, x):
        '''
        :param x: batch * input_len * in_channels
        :return: batch * output_len * out_channels
        '''
        out= F.relu(self.maxPool(self.conv1(torch.transpose(x, 1, 2))))
        #print('out: ', out.size())

        out= F.relu(self.maxPool(self.conv2(out)))
        out= torch.transpose(F.relu(self.maxPool(self.conv3(out))), 1, 2)
        return out

    def forward(self, x):
        embed = self.embed(x)
        x = embed.view(len(x), embed.size(1), -1)
        bilstm_out, self.hidden = self.bilstm(x, self.hidden)

        bilstm_out = torch.transpose(bilstm_out, 0, 1)
        bilstm_out = torch.transpose(bilstm_out, 1, 2)
        bilstm_out = F.tanh(bilstm_out)
        bilstm_out = F.max_pool1d(bilstm_out, bilstm_out.size(2)).squeeze(2)
        y = self.hidden2label1(bilstm_out)
        y = self.hidden2label2(y)
        logit = y
        return logit

def myMatrixDivVector(matrix, vector):
    """
       matrix(N,M) / vector(N) = matrix(N,M)
       for each i,j: 
           matrix_result[i][j] = matrix_source[i][j] / vector[i]
    """
    matrix1 = torch.transpose(matrix, 0, 1)
    print("matrix transpose:", matrix1)
    for i, nm in enumerate(matrix1):
        matrix1[i] = nm / vector
    print("matrix after division:", matrix1)
    matrix = torch.transpose(matrix1, 0, 1)
    print("matrix (final result):", matrix)
    return matrix

def CORAL_loss(source, target):
    d = source.data.shape[1]

    # source covariance
    xm = torch.mean(source, 1, keepdim=True) - source
    xc = torch.matmul(torch.transpose(xm, 0, 1), xm)

    # target covariance
    xmt = torch.mean(target, 1, keepdim=True) - target
    xct = torch.matmul(torch.transpose(xmt, 0, 1), xmt)
    # frobenius norm between source and target
    loss = (xc - xct).pow(2).sum().sqrt()
    loss = loss/(4*d*d)
    return loss

def max_singular_value(W, u=None, Ip=1):
    """
    power iteration for weight parameter
    """
    #xp = W.data
    if u is None:
        u = torch.FloatTensor(1, W.size(0)).normal_(0, 1).cuda()
    _u = u
    for _ in range(Ip):
        #print(_u.size(), W.size())
        _v = _l2normalize(torch.matmul(_u, W.data), eps=1e-12)
        _u = _l2normalize(torch.matmul(_v, torch.transpose(W.data, 0, 1)), eps=1e-12)
    sigma = torch.matmul(torch.matmul(_v, torch.transpose(W.data, 0, 1)), torch.transpose(_u, 0, 1))
    return sigma, _v

 def forward(self, input):
     embed = self.embed(input)
     input = embed.view(len(input), embed.size(1), -1)
     # lstm
     lstm_out, hidden = self.gru(input, self.hidden)
     lstm_out = torch.transpose(lstm_out, 0, 1)
     lstm_out = torch.transpose(lstm_out, 1, 2)
     # pooling
     lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2)).squeeze(2)
     lstm_out = F.tanh(lstm_out)
     # linear
     y = self.hidden2label(lstm_out)
     logit = y
     return logit

 def forward(self, x):
     embed = self.embed(x)
     embed = self.dropout_embed(embed)
     x = embed.view(len(x), embed.size(1), -1)
     # lstm
     lstm_out, self.hidden = self.lstm(x, self.hidden)
     lstm_out = torch.transpose(lstm_out, 0, 1)
     lstm_out = torch.transpose(lstm_out, 1, 2)
     # pooling
     lstm_out = F.tanh(lstm_out)
     lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2)).squeeze(2)
     lstm_out = F.tanh(lstm_out)
     # linear
     logit = self.hidden2label(lstm_out)
     return logit

 def _get_lstm_features(self, names, lengths):
     self.hidden = self.init_hidden(names.size(-1))
     embeds = self.char_embeds(names)  # Figure 4
     packed_input = pack_padded_sequence(embeds, lengths)  # Figure 5
     packed_output, (ht, ct) = self.lstm(packed_input, self.hidden)  # Figure 6
     lstm_out, _ = pad_packed_sequence(packed_output)  # Figure 7
     lstm_out = torch.transpose(lstm_out, 0, 1)
     lstm_out = torch.transpose(lstm_out, 1, 2)
     lstm_out = F.tanh(lstm_out)  # Figure 8
     lstm_out, indices = F.max_pool1d(lstm_out, lstm_out.size(2), return_indices=True)  # Figure 9
     lstm_out = lstm_out.squeeze(2)  #对维度的修正，使其符合输入格式
     lstm_out = F.tanh(lstm_out)
     lstm_feats = self.fully_connected_layer(lstm_out)
     output = self.softmax(lstm_feats)  # Figure 10
     return output

 def forward(self, input):
     embed = self.embed(input)
     embed = self.dropout(embed)  # add this reduce the acc
     input = embed.view(len(input), embed.size(1), -1)
     # gru
     gru_out, hidden = self.bigru(input, self.hidden)
     gru_out = torch.transpose(gru_out, 0, 1)
     gru_out = torch.transpose(gru_out, 1, 2)
     # pooling
     # gru_out = F.tanh(gru_out)
     gru_out = F.max_pool1d(gru_out, gru_out.size(2)).squeeze(2)
     gru_out = F.tanh(gru_out)
     # linear
     y = self.hidden2label(gru_out)
     logit = y
     return logit

    def forward(self, inputs):
        '''inputs: batch_size, num_tokens, 50
        return: batch_size, num_tokens + 2, output_dim'''
        mask = ((inputs > 0).long().sum(dim=-1) > 0).long()
        inputs_with_boundary, mask_with_boundary = _add_sentence_boundary_token_ids(inputs, mask,
                                                                                    self._beginning_of_sentence,
                                                                                    self._end_of_sentence)
        max_chars_per_token = self._options["char_cnn"]["max_characters_per_token"]
        character_embedding = torch.nn.functional.embedding(inputs_with_boundary.view(-1, max_chars_per_token),
                                                            self._char_embedding_weights)

        assert self._options["char_cnn"]["activation"] == "relu"

        # shape after transpose: (batch_size * (num_tokens + 2), output_dim, max_chars_per_token)
        character_embedding = torch.transpose(character_embedding, 1, 2)
        convs = []
        for i in range(len(self._convolutions)):
            conv = getattr(self, "char_conv_%d" % i)
            convolved = conv(character_embedding)
            # for each width, (batch_size * (num_tokens + 2), n_filters)
            convolved, _ = torch.max(convolved, dim=-1)
            convolved = torch.nn.functional.relu(convolved)
            convs.append(convolved)

        token_embedding = torch.cat(convs, dim=-1)
        token_embedding = self._highways(token_embedding)
        token_embedding = self._projection(token_embedding)
        batch_size, sequence_length_with_boundary, _ = inputs_with_boundary.size()
        return {"mask": mask_with_boundary,
                "token_embedding": token_embedding.view(batch_size, sequence_length_with_boundary, -1)}

    def forward(self, tokens: torch.Tensor, mask: torch.Tensor):  # pylint: disable=arguments-differ
        if mask is not None:
            tokens = tokens * mask.unsqueeze(-1).float()

        # Our input is expected to have shape `(batch_size, num_tokens, embedding_dim)`.  The
        # convolution layers expect input of shape `(batch_size, in_channels, sequence_length)`,
        # where the conv layer `in_channels` is our `embedding_dim`.  We thus need to transpose the
        # tensor first.
        tokens = torch.transpose(tokens, 1, 2)
        # Each convolution layer returns output of size `(batch_size, num_filters, pool_length)`,
        # where `pool_length = num_tokens - ngram_size + 1`.  We then do an activation function,
        # then do max pooling over each filter for the whole input sequence.  Because our max
        # pooling is simple, we just use `torch.max`.  The resultant tensor of has shape
        # `(batch_size, num_conv_layers * num_filters)`, which then gets projected using the
        # projection layer, if requested.

        filter_outputs = []
        for i in range(len(self._convolution_layers)):
            convolution_layer = getattr(self, 'conv_layer_{}'.format(i))
            filter_outputs.append(
                    self._activation(convolution_layer(tokens)).max(dim=2)[0]
            )

        # Now we have a list of `num_conv_layers` tensors of shape `(batch_size, num_filters)`.
        # Concatenating them gives us a tensor of shape `(batch_size, num_filters * num_conv_layers)`.
        maxpool_output = torch.cat(filter_outputs, dim=1) if len(filter_outputs) > 1 else filter_outputs[0]

        if self.projection_layer:
            result = self.projection_layer(maxpool_output)
        else:
            result = maxpool_output
        return result

 def forward(self, x):
     x_no_static = self.embed_no_static(x)
     # x_no_static = self.dropout(x_no_static)
     x_static = self.embed_static(x)
     # fix the embedding
     x_static = Variable(x_static.data)
     # x_static = self.dropout(x_static)
     x = torch.stack([x_static, x_no_static], 1)
     one_layer = x  # (N,W,D) #  torch.Size([64, 43, 300])
     # print("one_layer {}".format(one_layer.size()))
     # one_layer = self.dropout(one_layer)
     # one_layer = one_layer.unsqueeze(1)  # (N,Ci,W,D)  #  torch.Size([64, 1, 43, 300])
     # one layer
     one_layer = [torch.transpose(F.relu(conv(one_layer)).squeeze(3), 1, 2).unsqueeze(1) for conv in self.convs1] # torch.Size([64, 100, 36])
     # one_layer = [F.relu(conv(one_layer)).squeeze(3).unsqueeze(1) for conv in self.convs1] # torch.Size([64, 100, 36])
     # print(one_layer[0].size())
     # print(one_layer[1].size())
     # two layer
     two_layer = [F.relu(conv(one_layer)).squeeze(3) for (conv, one_layer) in zip(self.convs2, one_layer)]
     # print("two_layer {}".format(two_layer[0].size()))
     # print("two_layer {}".format(two_layer[1].size()))
     # pooling
     output = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in two_layer]   #  torch.Size([64, 100]) torch.Size([64, 100])
     output = torch.cat(output, 1)  # torch.Size([64, 300])
     # dropout
     output = self.dropout(output)
     # linear
     output = self.fc1(output)
     logit = self.fc2(F.relu(output))
     return logit

    def forward(self, question,length):
        length = list(length.data.cpu().numpy())
        
        
        emb = self.drop(self.encoder(question))
        emb = self.tanh(emb)

        hidden = self.init_hidden(len(length))
        seqs = trnn.pack_padded_sequence(emb, length, batch_first=True)

        seqs, hidden = self.rnn(seqs, hidden)
        h,_ = trnn.pad_packed_sequence(seqs, batch_first=True)

        #attention
        weights = self.softmax(self.att2(torch.transpose(h, 1, 2)).squeeze(1)).unsqueeze(-1)
        weights = weights.expand_as(h)
        
        bilstmout = torch.sum(h*weights, 1).squeeze(1)


        #bilstmout = torch.cat([hidden[0][0],hidden[0][1]],-1)


        fc1fea = self.fc1(bilstmout)

        return fc1fea

    def forward(self, x):
        # print(x)                              # <class 'torch.autograd.variable.Variable'> [torch.LongTensor of size 64x35]
        # x_size = x.data.size(1)
        # print(x_size)
        x = self.embed(x)
        x = x.unsqueeze(1)                      # (N,Ci,W,D) 在索引1处增加一维
        # print(x)                             # [torch.FloatTensor of size 64x1x35x128]
        # x = Variable(torch.transpose(x.data, 0, 1))
        # x = self.bn(x)
        # x = Variable(torch.transpose(x.data, 0, 1))
        # print(x)                             # [torch.FloatTensor of size 64x35x128]
        # if self.args.static:
        #     x = Variable(x.data)
        # print(x)                             # [torch.FloatTensor of size 64x35x128]

        # x2 = [F.relu(conv(x)).squeeze(3) for conv in self.convs1]  # [(N,Co,W), ...]*len(Ks)
        # print(x2)
        a = []
        for conv in self.convs1:
            xx = conv(x)                        # variable [torch.FloatTensor of size 16x200x35x1]
            # print(xx)
            xx = Variable(torch.transpose(xx.data, 2, 3))
            xx = Variable(torch.transpose(xx.data, 1, 2))
            xx = self.bn(xx)
            xx = F.relu(xx)
            xx = xx.squeeze(1)
            a.append(xx)
        # print(a)
        x = a
        # print(x)                             # [torch.FloatTensor of size 64x100x31],32,33,34
        x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]  # [(N,Co), ...]*len(Ks)
        # print(x)                             # [torch.FloatTensor of size 64x100]*4

        x = torch.cat(x, 1)
        # print(x)                             # [torch.FloatTensor of size 64x400]
        '''
        x1 = self.conv_and_pool(x,self.conv13) #(N,Co)
        x2 = self.conv_and_pool(x,self.conv14) #(N,Co)
        x3 = self.conv_and_pool(x,self.conv15) #(N,Co)
        x = torch.cat((x1, x2, x3), 1) # (N,len(Ks)*Co)
        '''

        x = self.dropout(x)  # (N,len(Ks)*Co)
        # print(x)                            # [torch.FloatTensor of size 64x400]
        logit = self.fc1(x)  # (N,C)
        # print(logit)                        # [torch.FloatTensor of size 64x2]
        return logit