def forward(self, x):
        x1 = self.layers1(x)
        x2 = self.layers2(x1)
        x3 = self.layers3(x2)
        x4 = self.layers4(x3)
        x5a = self.layers5a(x4)
        x5b = self.layers5b(x5a)
        x5c = self.layers5c(x5b)

        x5a_feat = F.avg_pool2d(x5a, x5a.size()[2:]).view(x5a.size(0), x5a.size(1))
        x5b_feat = F.avg_pool2d(x5b, x5b.size()[2:]).view(x5b.size(0), x5b.size(1))
        x5c_feat = F.avg_pool2d(x5c, x5c.size()[2:]).view(x5c.size(0), x5c.size(1))

        midfeat = torch.cat((x5a_feat, x5b_feat), dim=1)
        midfeat = self.fc_fuse(midfeat)

        combofeat = torch.cat((x5c_feat, midfeat), dim=1)
        
        if not self.training:
            return combofeat
        
        prelogits = self.classifier(combofeat)
        
        if self.loss == {'xent'}:
            return prelogits
        elif self.loss == {'xent', 'htri'}:
            return prelogits, combofeat
        else:
            raise KeyError("Unsupported loss: {}".format(self.loss))

    def forward(self, x):
        x = self.features(x)

        if hasattr(self, 'pooling_kernel_size') and self.pooling_kernel_size is not None:

            # 2D image output
            if hasattr(self, 'pooling_dropped') and self.pooling_dropped:
                x = x[:, :, self.pooling_dropped:-self.pooling_dropped, self.pooling_dropped:-self.pooling_dropped]
            x = F.avg_pool2d(x, kernel_size=self.pooling_kernel_size, stride=1)
            out = self.classifier(x)
            if self.raster_size is not None:
                out = out.view(out.size(0),
                               self.raster_size, self.raster_size, -1,
                               out.size(2), out.size(3))
                out = out.permute(0, 3, 4, 1, 5, 2).contiguous()
                out = out.view(out.size(0), out.size(1),
                               out.size(2)*out.size(3), out.size(4)*out.size(5))
            return out

        else:

            # Scalar output
            assert not hasattr(self, 'raster_size') or self.raster_size is None
            if not self.training and self.test_time_pool:
                x = F.avg_pool2d(x, kernel_size=7, stride=1)
                out = self.classifier(x)
                # The extra test time pool should be pooling an img_size//32 - 6 size patch
                out = adaptive_avgmax_pool2d(out, pool_type='avgmax')
            else:
                x = adaptive_avgmax_pool2d(x, pool_type='avg')
                out = self.classifier(x)
            return out.view(out.size(0), -1)

    def forward(self, x, class_em=True):
        for name, module in self.base._modules.items():
            if name == 'avgpool':
                break
            x = module(x)

        if self.cut_at_pooling:
            x = F.avg_pool2d(x, x.size()[2:])
            # x = x.max(1)[0][0]
            return x

        x = F.avg_pool2d(x, x.size()[2:])
        x = x.view(x.size(0), -1)
        # if self.dropout > 0:
        #     x = self.drop(x)
        if self.num_classes > 0 and not self.has_embedding:
            if class_em:
                x_class = self.classifier(x)
            else:
                x_class = x
        if self.has_embedding:
            x = self.feat(x)
            x = self.feat_bn(x)
            x = self.relu(x)
            if self.dropout > 0:
                x = self.drop(x)
        if self.num_diff_features > 0:
            x = self.diff_feat(x)
        if self.num_diff_features > 0 and self.num_classes > 0:
            return x, x_class
        elif self.num_classes > 0 and self.has_embedding:
            x_class = self.classifier(x)
            return x_class
        else:
            return x

    def forward(self, x):
        x = self.conv1(x)
        x = F.max_pool2d(x, 2) + F.avg_pool2d(x, 2)

        x = self.block1(x)
        x = self.group1(x)
        x = F.max_pool2d(x, 2) + F.avg_pool2d(x, 2)

        x = self.block2(x)
        x = self.group2(x)
        x = F.max_pool2d(x, 2) + F.avg_pool2d(x, 2)

        x = self.block3(x)
        x = self.group3(x)
        x = self.block4(x)
        x = self.group4(x)
        x = F.max_pool2d(x, 2) + F.avg_pool2d(x, 2)

        x = x.view(x.size(0), -1)
        fc = self.fc(x)
        x = F.dropout(fc, training=self.training)
        
        output = list()
        for name, fun in self.fc_dict.iteritems():
            out = fun(x)
            output.append(out)

        return output, fc

 def forward(self, x):
     out = F.relu(self.bn1(self.conv1(x)))
     out = self.layers(out)
     out = F.avg_pool2d(out, 2)
     out = out.view(out.size(0), -1)
     out = self.linear(out)
     return out

def preprocess2(img):
    # numpy to pytoch, float, scale, cuda, downsample
    img = torch.from_numpy(img).cuda().float() / 255.
    img = F.avg_pool2d(img, kernel_size=2, stride=None, padding=0, ceil_mode=False, count_include_pad=True)
    # print (img.shape) #[3,240,320]
    # fsdaf
    return img 

    def features(self, input):
        x_conv0 = self.conv0(input)
        x_stem_0 = self.cell_stem_0(x_conv0)
        x_stem_1 = self.cell_stem_1(x_conv0, x_stem_0)

        x_cell_0 = self.cell_0(x_stem_1, x_stem_0)
        x_cell_1 = self.cell_1(x_cell_0, x_stem_1)
        x_cell_2 = self.cell_2(x_cell_1, x_cell_0)
        x_cell_3 = self.cell_3(x_cell_2, x_cell_1)

        x_reduction_cell_0 = self.reduction_cell_0(x_cell_3, x_cell_2)

        x_cell_6 = self.cell_6(x_reduction_cell_0, x_cell_3)
        x_cell_7 = self.cell_7(x_cell_6, x_reduction_cell_0)
        x_cell_8 = self.cell_8(x_cell_7, x_cell_6)
        x_cell_9 = self.cell_9(x_cell_8, x_cell_7)

        x_reduction_cell_1 = self.reduction_cell_1(x_cell_9, x_cell_8)

        x_cell_12 = self.cell_12(x_reduction_cell_1, x_cell_9)
        x_cell_13 = self.cell_13(x_cell_12, x_reduction_cell_1)
        x_cell_14 = self.cell_14(x_cell_13, x_cell_12)
        x_cell_15 = self.cell_15(x_cell_14, x_cell_13)

        x_cell_15 = self.relu(x_cell_15)
        x_cell_15 = F.avg_pool2d(x_cell_15, x_cell_15.size()[2:])
        x_cell_15 = x_cell_15.view(x_cell_15.size(0), -1)
        x_cell_15 = self.dropout(x_cell_15)

        return x_cell_15

 def forward(self, x):
     if self.transform_input:
         x = x.clone()
         x[:, 0] = x[:, 0] * (0.229 / 0.5) + (0.485 - 0.5) / 0.5
         x[:, 1] = x[:, 1] * (0.224 / 0.5) + (0.456 - 0.5) / 0.5
         x[:, 2] = x[:, 2] * (0.225 / 0.5) + (0.406 - 0.5) / 0.5
     else: warn("Input isn't transformed")
     x = self.Conv2d_1a_3x3(x)
     x = self.Conv2d_2a_3x3(x)
     x = self.Conv2d_2b_3x3(x)
     x = F.max_pool2d(x, kernel_size=3, stride=2)
     x = self.Conv2d_3b_1x1(x)
     x = self.Conv2d_4a_3x3(x)
     x = F.max_pool2d(x, kernel_size=3, stride=2)
     x = self.Mixed_5b(x)
     x = self.Mixed_5c(x)
     x = self.Mixed_5d(x)
     x = self.Mixed_6a(x)
     x = self.Mixed_6b(x)
     x = self.Mixed_6c(x)
     x = self.Mixed_6d(x)
     x = self.Mixed_6e(x)
     x = self.Mixed_7a(x)
     x = self.Mixed_7b(x)
     x_for_attn = x = self.Mixed_7c(x)
     # 8 x 8 x 2048
     x = F.avg_pool2d(x, kernel_size=8)
     # 1 x 1 x 2048
     x_for_capt = x = x.view(x.size(0), -1)
     # 2048
     x = self.fc(x)
     # 1000 (num_classes)
     return x_for_attn, x_for_capt, x

    def forward(self, x):
        x = F.relu(self.bn1(self.conv1(x)), True)
        x = F.avg_pool2d(self.conv2(x), 2, stride=2)
        x = self.conv3(x)
        x = self.conv4(x)

        previous = x

        outputs = []
        for i in range(self.num_modules):
            hg = self._modules['m' + str(i)](previous)

            ll = hg
            ll = self._modules['top_m_' + str(i)](ll)

            ll = F.relu(self._modules['bn_end' + str(i)]
                        (self._modules['conv_last' + str(i)](ll)), True)

            # Predict heatmaps
            tmp_out = self._modules['l' + str(i)](ll)
            outputs.append(tmp_out)

            if i < self.num_modules - 1:
                ll = self._modules['bl' + str(i)](ll)
                tmp_out_ = self._modules['al' + str(i)](tmp_out)
                previous = previous + ll + tmp_out_

        return outputs

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)


        ## senet
        out2 = F.avg_pool2d(out, kernel_size=out.size(2))
        out2 = self.conv4(out2)
        out2 = self.relu(out2)
        out2 = self.conv5(out2)
        out2 = self.sigmoid(out2)
        # out2 = self.se_block.forward(out)  # not used

        if self.downsample is not None:
            residual = self.downsample(x)

        out = out2 * out + residual
        # out = out2 + residual  # not used
        out = self.relu(out)
        return out

    def forward(self, x):
        x1 = self.conv1(x)
        x1 = F.max_pool2d(x1, 3, stride=2)
        x2 = self.fire2(x1)
        x3 = self.fire3(x2)
        if self.bypass:
            x3 = x3 + x2
        x4 = self.fire4(x3)
        x4 = F.max_pool2d(x4, 3, stride=2)
        x5 = self.fire5(x4)
        if self.bypass:
            x5 = x5 + x4
        x6 = self.fire6(x5)
        x7 = self.fire7(x6)
        if self.bypass:
            x7 = x7 + x6
        x8 = self.fire8(x7)
        x8 = F.max_pool2d(x8, 3, stride=2)
        x9 = self.fire9(x8)
        if self.bypass:
            x9 = x9 + x8
        x9 = F.dropout(x9, training=self.training)
        x10 = F.relu(self.conv10(x9))
        f = F.avg_pool2d(x10, x10.size()[2:]).view(x10.size(0), -1)

        if not self.training:
            return f

        if self.loss == {'xent'}:
            return f
        elif self.loss == {'xent', 'htri'}:
            return f, f
        else:
            raise KeyError("Unsupported loss: {}".format(self.loss))

    def forward(self, x):
        xhighres = x
        h = self.blocks[-(self.depth + 1)](xhighres, True)
        if self.depth > 0:
            h = F.avg_pool2d(h, 2)
            if self.alpha < 1.0:
                xlowres = F.avg_pool2d(xhighres, 2)
                preult_rgb = self.blocks[-self.depth].fromRGB(xlowres)
                h = h * self.alpha + (1 - self.alpha) * preult_rgb

        for i in range(self.depth, 0, -1):
            h = self.blocks[-i](h)
            if i > 1:
                h = F.avg_pool2d(h, 2)
        h = self.linear(h.squeeze(-1).squeeze(-1))
        return h

    def forward(self, x):
        # reshape input first with batch size tracked
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.activation(x)
        x = self.dropout(x)
        x = self.main_avg_pool(x)

        x = self.conv2(x)
        x = self.bn2(x)
        x = self.activation(x)
        x = self.dropout(x)
        x = self.main_avg_pool(x)

        x = self.conv3(x)
        x = self.bn3(x)
        x = self.activation(x)
        x = self.dropout(x)
        x = self.main_avg_pool(x)

        x = self.conv4(x)
        x = self.bn4(x)
        x = self.activation(x)
        x = F.avg_pool2d(x, kernel_size=x.size()[-1])

        x = self.conv_to_class(x)
        x = x.view(x.size(0), -1)
        return x

 def forward(self, x):
     features = self.features(x)
     out = F.relu(features, inplace=True)
     out = F.avg_pool2d(out, kernel_size=self.avgpool_size).view(
         features.size(0), -1)
     out = self.classifier(out)
     return out

  def forward(self, x):
    """
    Returns:
      local_feat_list: each member with shape [N, c]
      logits_list: each member with shape [N, num_classes]
    """
    # shape [N, C, H, W]
    feat = self.base(x)
    assert feat.size(2) % self.num_stripes == 0
    stripe_h = int(feat.size(2) / self.num_stripes)
    local_feat_list = []
    logits_list = []
    for i in range(self.num_stripes):
      # shape [N, C, 1, 1]
      local_feat = F.avg_pool2d(
        feat[:, :, i * stripe_h: (i + 1) * stripe_h, :],
        (stripe_h, feat.size(-1)))
      # shape [N, c, 1, 1]
      local_feat = self.local_relu(self.local_bn(self.local_conv(local_feat)))
      # shape [N, c]
      local_feat = local_feat.view(local_feat.size(0), -1)
      local_feat_list.append(local_feat)
      if hasattr(self, 'fc_list'):
        logits_list.append(self.fc_list[i](local_feat))

    if hasattr(self, 'fc_list'):
      return local_feat_list, logits_list

    return local_feat_list

    def on_step_validation(self, state):
        if not self.done:
            x = state[torchbearer.X].data.clone()

            if len(x.size()) == 3:
                x = x.unsqueeze(1)

            x = F.avg_pool2d(x, self.avg_pool_size).data

            data = None

            if state[torchbearer.EPOCH] == 0 and self.write_data:
                if self.avg_data_channels:
                    data = torch.mean(x, 1)
                else:
                    data = x

                data = data.view(data.size(0), -1)

            feature = None

            if self.write_features:
                feature = state[self.features_key].data.clone()
                feature = feature.view(feature.size(0), -1)

            label = state[torchbearer.Y_TRUE].data.clone()

            if state[torchbearer.BATCH] == 0:
                remaining = self.num_images if self.num_images < label.size(0) else label.size(0)

                self._images = x[:remaining].to('cpu')
                self._labels = label[:remaining].to('cpu')

                if data is not None:
                    self._data = data[:remaining].to('cpu')

                if feature is not None:
                    self._features = feature[:remaining].to('cpu')
            else:
                remaining = self.num_images - self._labels.size(0)

                if remaining > label.size(0):
                    remaining = label.size(0)

                self._images = torch.cat((self._images, x[:remaining].to('cpu')), dim=0)
                self._labels = torch.cat((self._labels, label[:remaining].to('cpu')), dim=0)

                if data is not None:
                    self._data = torch.cat((self._data, data[:remaining].to('cpu')), dim=0)

                if feature is not None:
                    self._features = torch.cat((self._features, feature[:remaining].to('cpu')), dim=0)

            if self._labels.size(0) >= self.num_images:
                if state[torchbearer.EPOCH] == 0 and self.write_data:
                    self._writer.add_embedding(self._data, metadata=self._labels, label_img=self._images, tag='data', global_step=-1)
                if self.write_features:
                    self._writer.add_embedding(self._features, metadata=self._labels, label_img=self._images, tag='features', global_step=state[torchbearer.EPOCH])
                self.done = True

 def forward(self, x):
     out = self.conv1(x)
     out = self.trans1(self.dense1(out))
     out = self.trans2(self.dense2(out))
     out = self.dense3(out)
     out = torch.squeeze(F.avg_pool2d(F.relu(self.bn1(out)), 8))
     out = F.log_softmax(self.fc(out))
     return out

 def forward(self, x):
     B = x.data.size(0)
     C = x.data.size(1)
     H = x.data.size(2)
     W = x.data.size(3)
     x = F.avg_pool2d(x, (H, W))
     x = x.view(B, C)
     return x

 def forward(self, x):
     x = self.conv_1_3x3.forward(x)
     x = F.relu(self.bn_1.forward(x), inplace=True)
     x = self.stage_1.forward(x)
     x = self.stage_2.forward(x)
     x = self.stage_3.forward(x)
     x = F.avg_pool2d(x, 8, 1)
     x = x.view(-1, 1024)
     return self.classifier(x)

 def forward(self, x):
     out = F.relu(self.bn1(self.conv1(x)))
     out = self.layers(out)
     out = F.relu(self.bn2(self.conv2(out)))
     # NOTE: change pooling kernel_size 7 -> 4 for CIFAR10
     out = F.avg_pool2d(out, 4)
     out = out.view(out.size(0), -1)
     out = self.linear(out)
     return out