def test_3d_reg_shape(self):
        x = self.get_3d_input()

        unet_block_op = UNetBlock(
            'DOWNSAMPLE', (32, 64), (3, 3), with_downsample_branch=True,
            w_regularizer=regularizers.l2_regularizer(0.3))
        out_1, out_2 = unet_block_op(x, is_training=True)
        print(unet_block_op)
        print(out_1)
        print(out_2)

        unet_block_op = UNetBlock(
            'UPSAMPLE', (32, 64), (3, 3), with_downsample_branch=False,
            w_regularizer=regularizers.l2_regularizer(0.3))
        out_3, _ = unet_block_op(x, is_training=True)
        print(unet_block_op)
        print(out_3)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            out_1 = sess.run(out_1)
            self.assertAllClose((2, 8, 8, 8, 64), out_1.shape)
            out_2 = sess.run(out_2)
            self.assertAllClose((2, 16, 16, 16, 64), out_2.shape)
            out_3 = sess.run(out_3)
            self.assertAllClose((2, 32, 32, 32, 64), out_3.shape)

    def test_3d_reg_shape(self):
        x = self.get_3d_data()
        vnet_block_op = VNetBlock('DOWNSAMPLE', 2, 16, 8,
                                  w_regularizer=regularizers.l2_regularizer(
                                      0.2))
        out_1, out_2 = vnet_block_op(x, x)
        print(vnet_block_op)

        vnet_block_op = VNetBlock('UPSAMPLE', 2, 16, 8,
                                  w_regularizer=regularizers.l2_regularizer(
                                      0.2))
        out_3, out_4 = vnet_block_op(x, x)
        print(vnet_block_op)

        vnet_block_op = VNetBlock('SAME', 2, 16, 8,
                                  w_regularizer=regularizers.l2_regularizer(
                                      0.2))
        out_5, out_6 = vnet_block_op(x, x)
        print(vnet_block_op)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            out_1 = sess.run(out_1)
            self.assertAllClose((2, 16, 16, 16, 16), out_1.shape)
            out_2 = sess.run(out_2)
            self.assertAllClose((2, 8, 8, 8, 8), out_2.shape)
            out_3 = sess.run(out_3)
            self.assertAllClose((2, 16, 16, 16, 16), out_3.shape)
            out_4 = sess.run(out_4)
            self.assertAllClose((2, 32, 32, 32, 8), out_4.shape)
            out_5 = sess.run(out_5)
            self.assertAllClose((2, 16, 16, 16, 16), out_5.shape)
            out_6 = sess.run(out_6)
            self.assertAllClose((2, 16, 16, 16, 8), out_6.shape)

    def __init__(self,
                 decay=1e-6,
                 affine_w_initializer=None,
                 affine_b_initializer=None,
                 acti_func='relu',
                 name='inet-affine'):
        """
        This network estimates affine transformations from
        a pair of moving and fixed image:

            Hu et al., Label-driven weakly-supervised learning for
            multimodal deformable image registration, arXiv:1711.01666
            https://arxiv.org/abs/1711.01666

        :param decay:
        :param affine_w_initializer:
        :param affine_b_initializer:
        :param acti_func:
        :param name:
        """

        BaseNet.__init__(self, name=name)

        self.fea = [4, 8, 16, 32, 64]
        self.k_conv = 3
        self.affine_w_initializer = affine_w_initializer
        self.affine_b_initializer = affine_b_initializer
        self.res_param = {
            'w_initializer': GlorotUniform.get_instance(''),
            'w_regularizer': regularizers.l2_regularizer(decay),
            'acti_func': acti_func}
        self.affine_param = {
            'w_regularizer': regularizers.l2_regularizer(decay),
            'b_regularizer': None}

 def test_fc_2d_bias_reg_shape(self):
     input_param = {'n_output_chns': 10,
                    'with_bias': True,
                    'w_regularizer': regularizers.l2_regularizer(0.5),
                    'b_regularizer': regularizers.l2_regularizer(0.5)}
     self._test_fc_output_shape(rank=2,
                                param_dict=input_param,
                                output_shape=(2, 10))

 def test_fclayer_3d_bias_reg_shape(self):
     input_param = {'n_output_chns': 10,
                    'with_bn': False,
                    'w_regularizer': regularizers.l2_regularizer(0.5),
                    'b_regularizer': regularizers.l2_regularizer(0.5)}
     self._test_fc_layer_output_shape(rank=3,
                                      param_dict=input_param,
                                      output_shape=(2, 10))

 def test_deconv_3d_bias_reg_shape(self):
     input_param = {'n_output_chns': 10,
                    'kernel_size': 3,
                    'stride': [2, 2, 1],
                    'with_bias': True,
                    'w_regularizer': regularizers.l2_regularizer(0.5),
                    'b_regularizer': regularizers.l2_regularizer(0.5)}
     self._test_deconv_output_shape(rank=3,
                                    param_dict=input_param,
                                    output_shape=(2, 32, 32, 16, 10))

 def test_fclayer_2d_bn_reg_shape(self):
     input_param = {'n_output_chns': 10,
                    'with_bias': False,
                    'with_bn': True,
                    'w_regularizer': regularizers.l2_regularizer(0.5),
                    'b_regularizer': regularizers.l2_regularizer(0.5)}
     self._test_fc_layer_output_shape(rank=2,
                                      param_dict=input_param,
                                      output_shape=(2, 10),
                                      is_training=True)

 def test_convlayer_2d_bias_reg_shape(self):
     input_param = {'n_output_chns': 10,
                    'kernel_size': [3, 5],
                    'stride': [2, 1],
                    'with_bias': True,
                    'with_bn': False,
                    'w_regularizer': regularizers.l2_regularizer(0.5),
                    'b_regularizer': regularizers.l2_regularizer(0.5)}
     self._test_conv_layer_output_shape(rank=2,
                                        param_dict=input_param,
                                        output_shape=(2, 8, 16, 10))

 def test_convlayer_3d_bn_reg_shape(self):
     input_param = {'n_output_chns': 10,
                    'kernel_size': [5, 1, 2],
                    'stride': 1,
                    'with_bias': False,
                    'with_bn': True,
                    'w_regularizer': regularizers.l2_regularizer(0.5),
                    'b_regularizer': regularizers.l2_regularizer(0.5)}
     self._test_conv_layer_output_shape(rank=3,
                                        param_dict=input_param,
                                        output_shape=(2, 16, 16, 16, 10),
                                        is_training=True)

    def test_2d_reg_shape(self):
        input_shape = (2, 57, 57, 1)
        x = tf.ones(input_shape)

        deepmedic_instance = DeepMedic(
            num_classes=160,
            w_regularizer=regularizers.l2_regularizer(0.5),
            b_regularizer=regularizers.l2_regularizer(0.5))
        out = deepmedic_instance(x, is_training=True)
        # print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            out = sess.run(out)
            self.assertAllClose((2, 9, 9, 160), out.shape)

    def test_2d_reg_shape(self):
        input_shape = (2, 20, 20, 1)
        x = tf.ones(input_shape)

        holistic_net_instance = HolisticNet(
            num_classes=3,
            w_regularizer=regularizers.l2_regularizer(0.5),
            b_regularizer=regularizers.l2_regularizer(0.5))
        out = holistic_net_instance(x, is_training=False)
        # print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            out = sess.run(out)
            self.assertAllClose((2, 20, 20, 3), out.shape)

def slim_net_original(image, keep_prob):
    with arg_scope([layers.conv2d, layers.fully_connected], biases_initializer=tf.random_normal_initializer(stddev=0.1)):

        # conv2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME',
        # activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None,
        # weights_initializer=initializers.xavier_initializer(), weights_regularizer=None,
        # biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, scope=None):
        net = layers.conv2d(image, 32, [5, 5], scope='conv1', weights_regularizer=regularizers.l1_regularizer(0.5))

        # max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None)
        net = layers.max_pool2d(net, 2, scope='pool1')

        net = layers.conv2d(net, 64, [5, 5], scope='conv2', weights_regularizer=regularizers.l2_regularizer(0.5))
        summaries.summarize_tensor(net, tag='conv2')

        net = layers.max_pool2d(net, 2, scope='pool2')

        net = layers.flatten(net, scope='flatten1')

        # fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None,
        # normalizer_params=None, weights_initializer=initializers.xavier_initializer(),
        # weights_regularizer=None, biases_initializer=init_ops.zeros_initializer,
        # biases_regularizer=None, scope=None):
        net = layers.fully_connected(net, 1024, scope='fc1')

        # dropout(inputs, keep_prob=0.5, is_training=True, scope=None)
        net = layers.dropout(net, keep_prob=keep_prob, scope='dropout1')

        net = layers.fully_connected(net, 10, scope='fc2')
    return net

def resnet_arg_scope(is_training=True,
                     weight_decay=cfg.TRAIN.WEIGHT_DECAY,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
  batch_norm_params = {
    # NOTE 'is_training' here does not work because inside resnet it gets reset:
    # https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187
    'is_training': False,
    'decay': batch_norm_decay,
    'epsilon': batch_norm_epsilon,
    'scale': batch_norm_scale,
    'trainable': cfg.RESNET.BN_TRAIN,
    'updates_collections': ops.GraphKeys.UPDATE_OPS
  }

  with arg_scope(
      [slim.conv2d],
      weights_regularizer=regularizers.l2_regularizer(weight_decay),
      weights_initializer=initializers.variance_scaling_initializer(),
      trainable=is_training,
      activation_fn=nn_ops.relu,
      normalizer_fn=layers.batch_norm,
      normalizer_params=batch_norm_params):
    with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
      return arg_sc

    def test_3d_reg_shape(self):
        input_shape = (2, 32, 32, 32, 1)
        x = tf.ones(input_shape)

        # vnet_instance = VNet(num_classes=160)
        vnet_instance = VNet(
            num_classes=160,
            w_regularizer=regularizers.l2_regularizer(0.4),
            b_regularizer=regularizers.l2_regularizer(0.4))
        out = vnet_instance(x, is_training=True)
        print(vnet_instance.num_trainable_params())
        # print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            out = sess.run(out)
            self.assertAllClose((2, 32, 32, 32, 160), out.shape)

    def initialise_network(self):
        w_regularizer = None
        b_regularizer = None
        reg_type = self.net_param.reg_type.lower()
        decay = self.net_param.decay
        if reg_type == 'l2' and decay > 0:
            from tensorflow.contrib.layers.python.layers import regularizers
            w_regularizer = regularizers.l2_regularizer(decay)
            b_regularizer = regularizers.l2_regularizer(decay)
        elif reg_type == 'l1' and decay > 0:
            from tensorflow.contrib.layers.python.layers import regularizers
            w_regularizer = regularizers.l1_regularizer(decay)
            b_regularizer = regularizers.l1_regularizer(decay)

        self.net = ApplicationNetFactory.create(self.net_param.name)(
            w_regularizer=w_regularizer,
            b_regularizer=b_regularizer)

 def test_apply_zero_regularization(self):
   regularizer = regularizers.l2_regularizer(0.0)
   array_weights_list = [[1.5], [2, 3, 4.2], [10, 42, 666.6]]
   tensor_weights_list = [constant_op.constant(x) for x in array_weights_list]
   with self.cached_session():
     result = regularizers.apply_regularization(regularizer,
                                                tensor_weights_list)
     self.assertAllClose(0.0, result.eval())

def alexnet_v2_arg_scope(weight_decay=0.0005):
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      activation_fn=nn_ops.relu,
      biases_initializer=init_ops.constant_initializer(0.1),
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope([layers.conv2d], padding='SAME'):
      with arg_scope([layers_lib.max_pool2d], padding='VALID') as arg_sc:
        return arg_sc

 def test_fclayer_3d_bn_reg_dropout_valid_shape(self):
     input_param = {'n_output_chns': 10,
                    'with_bias': False,
                    'with_bn': True,
                    'w_regularizer': regularizers.l2_regularizer(0.5),
                    'acti_func': 'prelu', }
     self._test_fc_layer_output_shape(rank=3,
                                      param_dict=input_param,
                                      output_shape=(2, 10),
                                      is_training=True,
                                      dropout_prob=0.4)

 def test_3d_prelu_reg_shape(self):
     x = self.get_3d_input()
     prelu_layer = ActiLayer(func='prelu',
                             regularizer=regularizers.l2_regularizer(0.5),
                             name='regularized')
     out_prelu = prelu_layer(x)
     print(prelu_layer)
     with self.test_session() as sess:
         sess.run(tf.global_variables_initializer())
         out = sess.run(out_prelu)
         self.assertAllClose((2, 16, 16, 16, 8), out.shape)

 def test_convlayer_2d_bn_reg_prelu_shape(self):
     input_param = {'n_output_chns': 10,
                    'kernel_size': 3,
                    'stride': 1,
                    'with_bias': False,
                    'with_bn': True,
                    'acti_func': 'prelu',
                    'w_regularizer': regularizers.l2_regularizer(0.5)}
     self._test_conv_layer_output_shape(rank=2,
                                        param_dict=input_param,
                                        output_shape=(2, 16, 16, 10),
                                        is_training=True)