def total_tensor_depth(tensor=None, tensor_shape=None):
    """Returns the size of a tensor without the first (batch) dimension"""
    if tensor is None and tensor_shape is None:
        raise ValueError('a tensor or a tensor shape is required.')
    if tensor_shape:
        return int(np.prod(tensor_shape[1:]))
    return int(np.prod(get_shape(tensor)[1:]))

 def reshape(self, *shape):
     if len(shape) == 1 and isinstance(shape[0], (list, tuple)):
         shape = shape[0]
     if self.elemstrides == (1,):
         size = int(np.prod(shape))
         if size != self.size:
             raise ShapeMismatch(shape, self.shape)
         elemstrides = [1]
         for si in reversed(shape[1:]):
             elemstrides = [si * elemstrides[0]] + elemstrides
         return SignalView(
             base=self.base,
             shape=shape,
             elemstrides=elemstrides,
             offset=self.offset)
     elif self.size == 1:
         # -- scalars can be reshaped to any number of (1, 1, 1...)
         size = int(np.prod(shape))
         if size != self.size:
             raise ShapeMismatch(shape, self.shape)
         elemstrides = [1] * len(shape)
         return SignalView(
             base=self.base,
             shape=shape,
             elemstrides=elemstrides,
             offset=self.offset)
     else:
         # -- there are cases where reshaping can still work
         #    but there are limits too, because we can only
         #    support view-based reshapes. So the strides have
         #    to work.
         raise NotImplementedError('reshape of strided view')

    def init_conv_filters(self, numpy_rng, D, poolsize):
        ''' Convolutional Filters '''
        # there are "num input feature maps * filter height * filter width"
        # inputs to each hidden unit
        fan_in = np.prod(self.filter_shape[1:])

        # each unit in the lower layer receives a gradient from:
        # "num output feature maps * filter height * filter width" pooling size
        fan_out = (self.filter_shape[0] * np.prod(self.filter_shape[2:]) /
                   np.prod(poolsize))

        # initialize weights with random weights
        W_bound = np.sqrt(6. / (fan_in + fan_out))

        self.W = theano.shared(
                init_conv_weights(-W_bound, W_bound, \
                        self.filter_shape, numpy_rng),borrow=True, name='W_conv')

        #b_values = np.zeros((self.filter_shape[0],), dtype=theano.config.floatX)
        #self.b = theano.shared(value=b_values, borrow=True, name='b_conv')

        c_values = np.zeros((self.filter_shape[1],), dtype=theano.config.floatX)
        self.c = theano.shared(value=c_values, borrow=True, name='b_conv')

        self.params = [self.W, self.c]

 def process(self, image, out=None):
     # 0.25 is the default value used in Ng's paper
     alpha = self.specs.get('alpha', 0.25)
     # check if we would like to do two-side thresholding. Default yes.
     if self.specs.get('twoside', True):
         # concatenate, and make sure the output is C_CONTIGUOUS
         # for the temporary product, we check if we can utilize the
         # buffer to save allocation time
         product = mathutil.dot_image(image, self.dictionary.T)
         imshape = product.shape[:-1]
         N = product.shape[-1]
         product.resize((np.prod(imshape), N))
         if out is None:
             out = np.empty((np.prod(imshape), N*2))
         else:
             out.resize((np.prod(imshape), N*2))
         out[:,:N] = product
         out[:,N:] = -product
         out.resize(imshape + (N*2,))
     elif self.specs['twoside'] == 'abs':
         out = mathutil.dot_image(image, self.dictionary.T, out=out)
         np.abs(out, out=out)
     else:
         out = mathutil.dot_image(image, self.dictionary.T, out=out)
     # do threshold
     out -= alpha
     np.clip(out, 0., np.inf, out=out)
     return out

    def sample(self, n, d=None, rng=np.random):
        if d is not None and np.prod(self.options.shape[1:]) != d:
            raise ValueError("Options must be of dimensionality %d "
                             "(got %d)" % (d, np.prod(self.options.shape[1:])))

        i = np.searchsorted(np.cumsum(self.p), rng.rand(n))
        return self.options[i]

    def annotate_bn(self, var, id, var_type, mb_size, size, norm_ax):
        var_shape = np.array((1,) + size)
        out_dim = np.prod(var_shape) / np.prod(var_shape[list(norm_ax)])
        # Flatten the var - shared variable updating is not trivial otherwise,
        # as theano seems to believe a row vector is a matrix and will complain
        # about the updates
        orig_shape = var.shape
        var = var.flatten()
        # Here we add the name and role, the variables will later be identified
        # by these values
        var.name = id + '_%s_clean' % var_type
        add_role(var, BNPARAM)
        shared_var = self.shared(np.zeros(out_dim),
                                 name='shared_%s' % var.name, role=None)

        # Update running average estimates. When the counter is reset to 1, it
        # will clear its memory
        cntr, c_up = self.counter()
        one = np.float32(1)
        run_avg = lambda new, old: one / cntr * new + (one - one / cntr) * old
        if var_type == 'mean':
            new_value = run_avg(var, shared_var)
        elif var_type == 'var':
            mb_size = T.cast(mb_size, 'float32')
            new_value = run_avg(mb_size / (mb_size - one) * var, shared_var)
        else:
            raise NotImplemented('Unknown batch norm var %s' % var_type)
        # Add the counter update to the annotated update if it is the first
        # instance of a counter
        self.annotate_update([(shared_var, new_value)] + c_up, var)

        return var.reshape(orig_shape)

def checker(input_var,desire_size):

    if input_var is None:
        print('input_variable does not exist!')
      
    if desire_size is None:
        print('desire_size does not exist!')  
             
    dd=numpy.size(desire_size)
    dims = numpy.shape(input_var)
#     print('dd=',dd,'dims=',dims)
    if numpy.isnan(numpy.sum(input_var[:])):
        print('input has NaN')
      
    if numpy.ndim(input_var) < dd:
        print('input signal has too few dimensions')
      
    if dd > 1:
        if dims[0:dd] != desire_size[0:dd]:
            print(dims[0:dd])
            print(desire_size)
            print('input signal has wrong size1')
    elif dd == 1:
        if dims[0] != desire_size:
            print(dims[0])
            print(desire_size)
            print('input signal has wrong size2')
       
    if numpy.mod(numpy.prod(dims),numpy.prod(desire_size)) != 0:
        print('input signal shape is not multiples of desired size!')

    def _compute_shape(self, include_bc):
        """ Precomputes the shape of a mesh, both as a grid and as a vector.

        The shape as a grid means the result is a tuple containing the number
        of nodes in each dimension.  As a vector means a column vector (an nx1
        `~numpy.ndarray`) where n is the total degrees of freedom.

        Parameters
        ----------

        include_bc : bool
            Indicates if the boundary padding is included.

        """

        sh = []
        for i in range(self.dim):
            p = self.parameters[i]

            n = p.n
            if include_bc:
                n += p.lbc.n
                n += p.rbc.n

            sh.append(n)

        # self._shapes dict has a tuple for an index: (include_bc, as_grid)
        self._shapes[(include_bc, True)] = sh
        self._shapes[(include_bc, False)] = (int(np.prod(np.array(sh))), 1)

        # precompute the degrees of freedom dict too
        self._dofs[include_bc] = int(np.prod(np.array(sh)))

    def make_workspace_ijv(self):
        module = C.ConvertToImage()
        shape = (14, 16)
        r = np.random.RandomState()
        r.seed(0)
        i = r.randint(0, shape[0], size = np.prod(shape))
        j = r.randint(0, shape[1], size = np.prod(shape))
        v = r.randint(1, 8, size = np.prod(shape))
        order = np.lexsort((i, j, v))
        ijv = np.column_stack((i, j, v))
        ijv = ijv[order, :]
        same = np.all(ijv[:-1, :] == ijv[1:, :], 1)
        ijv = ijv[~same, :]

        pipeline = cpp.Pipeline()
        object_set = cpo.ObjectSet()
        image_set_list = cpi.ImageSetList()
        image_set = image_set_list.get_image_set(0)
        workspace = cpw.Workspace(pipeline,
                                  module,
                                  image_set,
                                  object_set,
                                  cpmeas.Measurements(),
                                  image_set_list)
        objects = cpo.Objects()
        objects.set_ijv(ijv, shape)
        object_set.add_objects(objects, OBJECTS_NAME)
        self.assertGreater(len(objects.get_labels()), 1)
        module.image_name.value = IMAGE_NAME
        module.object_name.value = OBJECTS_NAME
        return (workspace, module, ijv)

def dwts2(x, p, e, Id, Jd, Ip, Jp):

    q = zeros(p)
    ix = zeros(p, dtype=int)
    ix[(arange(p) + p - e) % p] = arange(p)
    d = x[ix[arange(1, p)]] - x[ix[0]]

    if p == 3:

        d1_2 = d[0] - d[1]
        q[ix[0]] = 2.0 / (d[0] * d[1])
        q[ix[1]] = 2.0 / (d[0] * d1_2)
        q[ix[2]] = -2.0 / (d[1] * d1_2)

    else:

        # compute difference terms
        dd = d[Id[:, 0]] - d[Id[:, 1]]

        # compute permutation terms
        dp = prod(d[Ip], axis=1)

        # compute weights
        q[ix[0]] = 2.0 * sum(dp) / prod(d)
        for i in range(p - 1):
            fac = 2.0 * (-1) ** (p + i + 1)
            num = sum(dp[Jp[i, :]])  # all dp without i
            den = d[i] * prod(dd[Jd[i, :]])  # all dd with i
            q[ix[i + 1]] = fac * num / den

    return q

    def _initialize_theta(self):
        filter_shape = self.filter_shape
        image_shape = self.image_shape
        poolsize = self.poolsize

        conv_in = np.prod(filter_shape[1:])
        conv_out = filter_shape[0] * np.prod(filter_shape[2:])
        pool_out = conv_out / poolsize**2

        conv_map_size = image_shape[-1] - filter_shape[-1] + 1
        assert conv_map_size > 0
        pool_map_size = int(conv_map_size / poolsize)
        assert pool_map_size > 0

        self.conv_w = shared(
            nn_random_paramters(conv_in, conv_out, shape=filter_shape))
        self.conv_b = shared(
            nn_random_paramters(conv_in, conv_out,
                                shape=(filter_shape[0], 1, 1)))
        self.pool_w = shared(
            nn_random_paramters(conv_out, pool_out,
                                shape=(filter_shape[0], 1, 1)))
        self.pool_b = shared(
            nn_random_paramters(conv_out, pool_out,
                                shape=(filter_shape[0], 1, 1)))
        self.output_shape = (image_shape[0], filter_shape[0],
                             pool_map_size, pool_map_size)

        return [self.conv_w, self.conv_b, self.pool_w, self.pool_b]

    def test_neibs_bad_shape_wrap_centered(self):
        shape = (2, 3, 10, 10)

        for dtype in self.dtypes:
            images = shared(numpy.arange(
                numpy.prod(shape), dtype=dtype
                ).reshape(shape))

            for neib_shape in [(3, 2), (2, 3)]:
                neib_shape = T.as_tensor_variable(neib_shape)

                f = function([], images2neibs(images, neib_shape,
                                              mode="wrap_centered"),
                             mode=self.mode)
                self.assertRaises(TypeError, f)

            for shape in [(2, 3, 2, 3), (2, 3, 3, 2)]:
                images = shared(numpy.arange(numpy.prod(shape)).reshape(shape))
                neib_shape = T.as_tensor_variable((3, 3))
                f = function([], images2neibs(images, neib_shape,
                                              mode="wrap_centered"),
                             mode=self.mode)
                self.assertRaises(TypeError, f)

            # Test a valid shapes
            shape = (2, 3, 3, 3)
            images = shared(numpy.arange(numpy.prod(shape)).reshape(shape))
            neib_shape = T.as_tensor_variable((3, 3))

            f = function([],
                         images2neibs(images, neib_shape, mode="wrap_centered"),
                         mode=self.mode)
            f()

def varying_weight_orderplots(p,wkey,xvals):
    """
    recieves a wTOPopulation (weighted 2 objectives) and makes a plot for each weight setting value given in xvals;
    works with both, weighted ranking or weighted sum of objectives, depending on argument wkey being 'r' or 's'
    """
    # remember old setting
    if wkey=='s': xold=p.sumcoeffs[0]
    elif wkey=='r': xold=p.rankweights[0]
    # make the plots
    for i,x in enumerate(xvals):
        if wkey=='s':
            p.set_sumcoeffs([x,1-x]); p.update_scores(); p.sort()
        if wkey=='r':
            p.set_rankweights([x,1-x]); p.update_overall_ranks(); p.sort_for('overall_rank')
        sqdft,sqdlt=p.ranking_triangles_twoobj(x,1,wkey)
        ttxt='weighting factors: '+str(p.sumcoeffs)+'\n'
        ttxt+='sqdft = '+str(sqdft)+',  sqdlf = '+str(sqdlt)
        ttxt+=',  crit 1 '+str(prod(sqdft)/prod(sqdlt))+'\n'
        r1,r2=p.correlations_criterion(x,1,wkey)
        ttxt+='$r_{P,1}$ = '+str(r1)+',  $r_{P,2}$ = '+str(r2)
        ttxt+=',  $crit(r_{P,1},r_{P,2})$ = '+str(abs(r1-r2)*max(abs(r1),abs(r2)))
        MOorderplot(p,join(p.path,'plots2'),title=ttxt,
                    picname='var_'+wkey+'w_orderplot_nc'+str(p.ncase)+'_g'+str(p.gg).zfill(3)+'_op'+str(i).zfill(2)+'.png')
    # restore old order
    if wkey=='s':
        p.set_sumcoeffs([xold,1-xold]); p.update_scores(); p.sort()
    if wkey=='r':
        p.set_rankweights([xold,1-xold]); p.update_overall_ranks(); p.sort_for('overall_rank')

    def case(S, n_trials=50):        
        S = to_super(S)
        left_dims, right_dims = S.dims
        
        # Assume for the purposes of the test that S maps square operators to square operators.
        in_dim = np.prod(right_dims[0])
        out_dim = np.prod(left_dims[0])
        
        S_dual = to_super(S.dual_chan())
        
        primals = []
        duals = []
    
        for idx_trial in range(n_trials):
            X = rand_dm_ginibre(out_dim)
            X.dims = left_dims
            X = operator_to_vector(X)
            Y = rand_dm_ginibre(in_dim)
            Y.dims = right_dims
            Y = operator_to_vector(Y)

            primals.append((X.dag() * S * Y)[0, 0])
            duals.append((X.dag() * S_dual.dag() * Y)[0, 0])
    
        np.testing.assert_array_almost_equal(primals, duals)

def boardProd(chessboard, Q, K, R, B, N):
    # remove 0
    chessboard[Q[0]] = 1
    chessboard[K[0]] = 1
    chessboard[R[0]] = 1
    chessboard[B[0]] = 1
    chessboard[N[0]] = 1
    
    # remove -1
    negind = np.where(chessboard < 0)
    chessboard[negind] = 1
    
    # -1 is taken care of by abs
    rprodcurr = abs(np.prod(chessboard % 10, axis=1))
    cprodcurr = abs(np.prod(chessboard % 10, axis=0))
    
    # put back 0 and -1
    chessboard[Q[0]] = 0
    chessboard[K[0]] = 0
    chessboard[R[0]] = 0
    chessboard[B[0]] = 0
    chessboard[N[0]] = 0
    chessboard[negind] = -1
    
    return rprodcurr, cprodcurr

def prod(x, axis=None, keepdims=False):
    if isinstance(axis, list):
        for a in axis:
            x = np.prod(x, axis=a, keepdims=keepdims)
        return x
    else:
        return np.prod(x, axis=axis, keepdims=keepdims)

    def check_log_prob(self, is_gpu):
        smp = self.sample_for_test()
        if is_gpu:
            log_prob1 = self.gpu_dist.log_prob(cuda.to_gpu(smp)).data
        else:
            log_prob1 = self.cpu_dist.log_prob(smp).data

        if self.continuous:
            scipy_prob = self.scipy_dist.logpdf
        else:
            scipy_prob = self.scipy_dist.logpmf

        if self.scipy_onebyone:
            onebyone_smp = smp.reshape(
                (int(numpy.prod(self.sample_shape)),
                 numpy.prod(self.shape),
                 int(numpy.prod(self.event_shape))))
            onebyone_smp = numpy.swapaxes(onebyone_smp, 0, 1)
            onebyone_smp = onebyone_smp.reshape((-1,) + self.sample_shape
                                                + self.event_shape)
            log_prob2 = []
            for one_params, one_smp in zip(
                    self.scipy_onebyone_params_iter(), onebyone_smp):
                log_prob2.append(scipy_prob(one_smp, **one_params))
            log_prob2 = numpy.vstack(log_prob2)
            log_prob2 = log_prob2.reshape(numpy.prod(self.shape), -1).T
            log_prob2 = log_prob2.reshape(self.sample_shape + self.shape)
        else:
            log_prob2 = scipy_prob(smp, **self.scipy_params)
        array.assert_allclose(log_prob1, log_prob2)

  def testContribSignalSTFT(self):
    ws = 512
    hs = 128
    dims = (ws * 20,)
    shape = BATCH_DIMS + dims
    data = np.arange(np.prod(shape)) / np.prod(dims)
    np.random.seed(123)
    np.random.shuffle(data)
    data = np.reshape(data.astype(np.float32), shape)
    window = sps.get_window("hann", ws)
    expected = sps.stft(
        data, nperseg=ws, noverlap=ws - hs, boundary=None, window=window)[2]
    expected = np.swapaxes(expected, -1, -2)
    expected *= window.sum()  # scipy divides by window sum
    with self.test_session() as sess:
      with self.test_scope():
        ph = array_ops.placeholder(
            dtypes.as_dtype(data.dtype), shape=data.shape)
        out = signal.stft(ph, ws, hs)
        grad = gradients_impl.gradients(out, ph,
                                        grad_ys=array_ops.ones_like(out))

      # For gradients, we simply verify that they compile & execute.
      value, _ = sess.run([out, grad], {ph: data})
      self.assertAllClose(expected, value, rtol=RTOL, atol=ATOL)

def test_valid_1_3_11_12():
    seed_rng()
    shapes = get_valid_shapes()
    version = [1, 3, 11, 12]
    verbose = 0

    random = True
    print_ = False
    ones = False
    if ones:
        random = False
    shapes2 = []

    for id, (ishape, kshape, subshape, istride, kstride) in enumerate(shapes):
        oshape = [ishape[0]] + [kshape[0]] + list(numpy.asarray(ishape[2:]) -
                                                  numpy.asarray(kshape[2:]) +
                                                  numpy.asarray([1, 1]))
        if oshape[3] > device_prop['maxThreadsDim0']:
            continue
        if ((numpy.prod(ishape[2:]) + numpy.prod(kshape[2:])) * 4 >
            (16 * 1024 - 150)):
            continue
        if subshape == (1, 1):
            shapes2.append((ishape, kshape, subshape, istride, kstride))
    shapes = shapes2

    exec_conv(version, shapes, verbose, random, 'valid',
              print_=print_, ones=ones, rtol=1.1e-5)

def analyze_param(net, layers):
#   plt.figure()
    print '\n=============analyze_param start==============='
    total_nonzero = 0
    total_allparam = 0
    percentage_list = []
    for i, layer in enumerate(layers):
        i += 1
        W = net.params[layer][0].data
        b = net.params[layer][1].data
#       plt.subplot(3, 1, i);
#       numBins = 2 ^ 8
#       plt.hist(W.flatten(), numBins, color='blue', alpha=0.8)
#       plt.show()
        print 'W(%d) range = [%f, %f]' % (i, min(W.flatten()), max(W.flatten()))
        print 'W(%d) mean = %f, std = %f' % (i, np.mean(W.flatten()), np.std(W.flatten()))
        non_zero = (np.count_nonzero(W.flatten()) + np.count_nonzero(b.flatten()))
        all_param = (np.prod(W.shape) + np.prod(b.shape))
        this_layer_percentage = non_zero / float(all_param)
        total_nonzero += non_zero
        total_allparam += all_param
        print 'non-zero W and b cnt = %d' % non_zero
        print 'total W and b cnt = %d' % all_param
        print 'percentage = %f\n' % (this_layer_percentage)
        percentage_list.append(this_layer_percentage)
    print '=====> summary:'
    print 'non-zero W and b cnt = %d' % total_nonzero
    print 'total W and b cnt = %d' % total_allparam
    print 'percentage = %f' % (total_nonzero / float(total_allparam))
    print '=============analyze_param ends ==============='
    return (total_nonzero / float(total_allparam), percentage_list)