def call_feature_demon(imageurl):
  result_dic = {}
  #import pdb; pdb.set_trace()
  try:
    fe_starttime = time.time()
    url = url_prefix % imageurl
    response = urllib2.urlopen(url)
    logging.info('extract_feature done, %.4f', time.time() - fe_starttime)
    if response <> None:
      retrieved_items = json.loads(response.read())
      if retrieved_items['result']:
        result_dic['url'] = retrieved_items['url']
        result_dic['predicted_category'] = retrieved_items['category']
        result_dic['scores'] = \
          np.trim_zeros((np.asarray(retrieved_items['score']) * 100).astype(np.uint8))
        #print(result_dic['scores'].shape)
        result_dic['predicted_category'] = result_dic['predicted_category'][0:result_dic['scores'].size]
        result_dic['feature'] = np.asarray(retrieved_items['feature'])

        result_dic['predicted_category_gc'] = retrieved_items['category_gc']
        result_dic['scores_gc'] = \
          np.trim_zeros((np.asarray(retrieved_items['score_gc']) * 100).astype(np.uint8))
        #print(result_dic['scores'].shape)
        result_dic['predicted_category_gc'] = result_dic['predicted_category_gc'][0:result_dic['scores_gc'].size]
        result_dic['sentence'] = retrieved_items['sentence']
        
  except Exception as err:
    logging.info('call_feature_demon error: %s', err)
    return {'result': False, 'feature': None}

  return result_dic

def computePrecision(R_cv, B_cv, predictions, threshold, n):			
	'''
	Computes [email protected] metric on the cross-validation set.
	[email protected] is the percentage of movies the user rated above threshold in the recommendation list of size n
	'''
	
	cv_predictions = np.multiply(predictions, B_cv)
	sorted_predictions = np.fliplr(np.sort(cv_predictions))[:,:n]	
	top_indices = np.fliplr(np.argsort(cv_predictions))[:,:n]
	
	num_users = R_cv.shape[0]
	precision = np.zeros(num_users, dtype=float)
	
	for user_id in range(num_users):
		user_liked = np.ceil(threshold)<= np.trim_zeros(R_cv[user_id,top_indices[user_id]]) 
		user_disliked = np.trim_zeros(R_cv[user_id,top_indices[user_id]]) <= np.floor(threshold)
		
		# we think that a recommendation is good if the predicted rating 
		# is grater than threshold
		not_recommended = np.trim_zeros(sorted_predictions[user_id]) < threshold
		recommended = np.trim_zeros(sorted_predictions[user_id]) >= threshold
		
		tp = np.count_nonzero(np.logical_or(not_recommended, user_liked))
		fp = np.count_nonzero(np.logical_and(recommended, user_disliked))
		precision[user_id] = float(tp) / (tp+fp)
	
	mean_precision = np.mean(precision)
	return mean_precision

def redundancyAnalysis(m):
  CHECK = False
  dims = m.shape
  vals = np.zeros((dims[1]*dims[2]*dims[3], (1<<PRECISION+1)))
  
  #if CHECK:
  aux = np.zeros(257)
  auxVal = np.zeros(257)

  index = 0
  for c in range(dims[1]):
    for x in range(dims[2]):
      for y in range(dims[3]):
        for i in range(dims[0]):
          if CHECK:
            if aux[ toInteger(m[i][c][x][y]) ] > 0:
              if m[i][c][x][y] != auxVal[  toInteger(m[i][c][x][y]) ]:
                print "ALARM"
            auxVal[ toInteger(m[i][c][x][y]) ] = m[i][c][x][y]
            aux[ toInteger(m[i][c][x][y]) ] += 1
          
          vals[index][ toInteger(m[i][c][x][y]) ] += 1;
        index += 1 # same value for all the filters
  vals = vals.flatten()
  vals = np.sort(vals)
  #print vals
  vals = np.trim_zeros(vals)
  vals -= 1
  vals = np.trim_zeros(vals)
  vals = vals[::-1]
  #print vals
  vals = np.cumsum(vals)
  vals /= dims[1]*dims[2]*dims[3]*dims[0]
  vals = sample(vals,SAMPLES)
  return vals

 def axis_data(axis):
     """Gets the bounds of a masked area along a certain axis"""
     x = mask.sum(axis)
     trimmed_front = N.trim_zeros(x,"f")
     offset = len(x)-len(trimmed_front)
     size = len(N.trim_zeros(trimmed_front,"b"))
     return offset,size

 def trim(self, min_qual, clone = True):
     """return a new object with DNA trimmed according to some min_qual
         
     call using my_object.trim(clone=False) if you wish to 
     mask, trim, and return the current object.
     
     """
     self._qual_check()
     if clone: # pragma: no cover
         rval = self.clone()
     else: # pragma: no cover
         rval = self
     rval.min_qual = min_qual
     trim, comparison = rval._check()
     if trim:
         rval.snapshot()
         # use temp array so we don't change inner quality values (we are *not* masking)
         temp = deepcopy(rval.quality)
         temp[numpy.where(comparison)[0]] = 0
         l = len(rval.quality) - len(numpy.trim_zeros(temp,'f'))
         r = len(rval.quality) - len(numpy.trim_zeros(temp,'b'))
         rval.quality = rval.quality[l:len(rval.quality) - r]
         rval.sequence = rval.sequence[l:len(rval.sequence) - r]
         rval.trimming = 't'
     return rval

    def plot_abilities_one_components(self, team_ids, **kwargs):
        import matplotlib.pyplot as plt
        import seaborn as sns

        figsize = kwargs.get('figsize',(15,5))

        if self.latent_variables.estimated is False:
            raise Exception("No latent variables estimated!")
        else:
            plt.figure(figsize=figsize)

            if type(team_ids) == type([]):
                if type(team_ids[0]) == str:
                    for team_id in team_ids:
                        plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[self.team_dict[team_id]],
                            trim='b'), label=self.team_strings[self.team_dict[team_id]])
                else:
                    for team_id in team_ids:
                        plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[team_id],
                            trim='b'), label=self.team_strings[team_id])
            else:
                if type(team_ids) == str:
                    plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[self.team_dict[team_ids]],
                        trim='b'), label=self.team_strings[self.team_dict[team_ids]])
                else:
                    plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[team_ids],
                        trim='b'), label=self.team_strings[team_ids])

            plt.legend()
            plt.ylabel("Power")
            plt.xlabel("Games")
            plt.show()

def calc_power(N, field, dk, Nshot, Lx, Ly, norm):

	dkx = (2.*np.pi)/Lx
	dky = (2.*np.pi)/Ly
	V= Lx*Ly

	power = np.zeros(N, dtype=float)
	Nmodes = np.zeros(N, dtype=int)
	
	for ix in range (0,N):
		if ix <=N/2.:
			kx = ix*dkx
		else:
			kx = (ix-N)*dkx

		for iy in range (0,N):
			if iy <=N/2.:
				ky = iy*dky
			else:
				ky = (iy-N)*dky
	
			kval = (kx*kx + ky*ky)**0.5

			if kval >0:
				power[int(kval/dk)] = power[int(kval/dk)] + field[ix][iy].real**2 + field[ix][iy].imag**2 - Nshot
				Nmodes[int(kval/dk)] = Nmodes[int(kval/dk)]+1
				
	iNonZeros = np.where(Nmodes != 0)	
	iZeros = np.where(Nmodes == 0)
	power[iNonZeros] = power[iNonZeros]/Nmodes[iNonZeros]

	k=np.linspace(dkx/2., ((N-1)*dkx)+dkx/2. ,num=N )	
	k[iZeros]= 0.
	return V*np.trim_zeros(power)/norm, np.trim_zeros(k)

 def trim_zeros(self):
     """Remove the leading and trailing zeros.
     """      
     tmp = self.numpy()
     f = len(self)-len(_numpy.trim_zeros(tmp, trim='f'))
     b = len(self)-len(_numpy.trim_zeros(tmp, trim='b'))
     return self[f:len(self)-b]

def parse_coverage_bam(fbam, genomecoveragebed="genomeCoverageBed"):
    """
    
    Arguments:
    - `fbam`: file to read coverage from. Needs samtools and bedtools (genomeCoverageBed)
    - `genomecoveragebed`: path to genomeCoverageBed binary
    """

  
    # forward and reverse coverage
    # int is essential; see note below
    fw_cov = numpy.zeros((MAX_LEN), dtype=numpy.int32)
    rv_cov = numpy.zeros((MAX_LEN), dtype=numpy.int32)

    
    file_genome = tempfile.NamedTemporaryFile(delete=False)
    genome_from_bam(fbam, file_genome)
    file_genome.close()

    basic_cmd = "%s -ibam %s -g %s -d" % (
        genomecoveragebed, fbam, file_genome.name)



    for strand in ["+", "-"]:
        cmd = "%s -strand %s" % (basic_cmd, strand)
        try:
            process = subprocess.Popen(cmd.split(),
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
        except OSError:
            raise OSError, ("It seems %s is not installed" % cmd.split()[0])
        (stdoutdata, stderrdata) =  process.communicate()
        retcode = process.returncode
        if retcode != 0:
            raise OSError("Called command exited with error code '%d'." \
                          " Command was '%s'. stderr was: '%s'" % (
                              retcode, cmd, stderrdata))
        for line in str.splitlines(stderrdata):
            if len(line.strip()) == 0:
                continue
            LOG.warn("Got following stderr message from genomeCoverageBed: %s" % line)
        for line in str.splitlines(stdoutdata):
            if len(line) == 0:
                continue
            (chr, pos, cov) = line.split('\t')
            pos = int(pos)-1 # we use zero offset
            cov = int(cov)

            assert pos < MAX_LEN

            if strand == '+':
                fw_cov[pos] = cov
            elif strand == '-':
                rv_cov[pos] = cov

    os.unlink(file_genome.name)

    return (numpy.trim_zeros(fw_cov, trim='b'),
            numpy.trim_zeros(rv_cov, trim='b'))

    def fit(self, data):
        magnitude = data[0]
        time = data[1]

        global m_21
        global m_31
        global m_32

        Nsf = 100
        Np = 100
        sf1 = np.zeros(Nsf)
        sf2 = np.zeros(Nsf)
        sf3 = np.zeros(Nsf)
        f = interp1d(time, magnitude)

        time_int = np.linspace(np.min(time), np.max(time), Np)
        mag_int = f(time_int)

        for tau in np.arange(1, Nsf):
            sf1[tau-1] = np.mean(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 1.0))
            sf2[tau-1] = np.mean(np.abs(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 2.0)))
            sf3[tau-1] = np.mean(np.abs(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 3.0)))
        sf1_log = np.log10(np.trim_zeros(sf1))
        sf2_log = np.log10(np.trim_zeros(sf2))
        sf3_log = np.log10(np.trim_zeros(sf3))

        m_21, b_21 = np.polyfit(sf1_log, sf2_log, 1)
        m_31, b_31 = np.polyfit(sf1_log, sf3_log, 1)
        m_32, b_32 = np.polyfit(sf2_log, sf3_log, 1)

        return m_21

 def avgEpisodeVValue(self):
     """ Returns the average V value on the episode (on time steps where a non-random action has been taken)
     """
     if (len(self._Vs_on_last_episode) == 0):
         return -1
     if(np.trim_zeros(self._Vs_on_last_episode)!=[]):
         return np.average(np.trim_zeros(self._Vs_on_last_episode))
     else:
         return 0

 def assert_numden_almost_equal(self, n1, n2, d1, d2):
     n1[np.abs(n1) < 1e-10] = 0.
     n1 = np.trim_zeros(n1)
     d1[np.abs(d1) < 1e-10] = 0.
     d1 = np.trim_zeros(d1)
     n2[np.abs(n2) < 1e-10] = 0.
     n2 = np.trim_zeros(n2)
     d2[np.abs(d2) < 1e-10] = 0.
     d2 = np.trim_zeros(d2)
     np.testing.assert_array_almost_equal(n1, n2)
     np.testing.assert_array_almost_equal(d2, d2)

def printSummaryOutput(maincounts):
    print OUTPUT + "\\summaryoutput.txt"
    with open(OUTPUT + "\\summaryoutput.txt", 'wb') as f:
        for index in maincounts.keys():
            f.write(index + "  ***************\n")
            f.write("Total Count: " + str(maincounts[index]['totnum']) + '\n')
            f.write("Total Females: " + str(maincounts[index]['numfemales']) + '\n')
            f.write("Total Males: " + str(maincounts[index]['nummales']) + '\n')

            f.write("Mean Length of Stay: " + str(np.mean(np.trim_zeros(np.nan_to_num(maincounts[index]['meanlengthofstay'])))) + '\n')
            f.write("Mean Age: " + str(np.mean(np.trim_zeros(maincounts[index]['meanage']))) + '\n')
            f.write("Mean Travel Events: " + str(np.mean(np.trim_zeros(maincounts[index]['meantravelevents']))) + '\n')

    def get_history(self, state, action):
        """
        :param state:
        :param action:
        :return: h(s,a), i.e. the number of times given action was selected in given state in the current episode
        """

        state = tuple(np.trim_zeros(state, 'f'))
        action = tuple(np.trim_zeros(action, 'f'))

        if (state, action) in self.state_action_history:
            return self.state_action_history[(state, action)]
        return 0

def largestDistrict(data):
    largestDis=0
    data_new = data.dropna(subset=['Location','PoliceDistrict'])
    policeDis=np.array(data_new['PoliceDistrict'])
    Location=np.array(data_new['Location'])

    dic= collections.defaultdict(list)

    for i in range(policeDis.shape[0]):
        dic[policeDis[i]]+=[Location[i]]
    for key,value in dic.items():
        x=np.zeros(len(value))
        y=np.zeros(len(value))
        xi=0
        for i in range(len(value)):
            l=value[i][1:-2]
            loc=l.split(",")

            if float(loc[0])>26 and float(loc[0])<30 and float(loc[1])>-93 and float(loc[1])<-87:
                x[xi]=float(loc[0])
                y[xi]=float(loc[1])
                xi+=1
        x=np.trim_zeros(x)
        y=np.trim_zeros(y)
        stdX=np.std(x)
        stdY=np.std(y)
        meanX=np.mean(x)
        meanY=np.mean(y)

        bootom_la=math.radians(meanX-stdX)
        top_la=math.radians(meanX+stdX)
        delta_la=math.radians(2*stdX)
        delta_long=0

        a=math.sin(delta_la)*math.sin(delta_la)+math.cos(bootom_la)*math.cos(top_la)*math.sin(delta_long/2)*math.sin(delta_long/2)
        c=2* math.atan2(math.sqrt(a),math.sqrt(1-a))
        d_x=6371*c/2

        delta_la=0
        delta_long=math.radians(2*stdY)
        la=math.radians(meanX)

        a=math.sin(delta_la)*math.sin(delta_la)+math.cos(la)*math.cos(la)*math.sin(delta_long/2)*math.sin(delta_long/2)
        c=2*math.atan2(math.sqrt(a),math.sqrt(1-a))
        d_y=6371*c/2

        area=math.pi*d_x*d_y
        print area, key, meanX,meanY,stdX,stdY
        largestDis=max(area,largestDis)
    return largestDis

    def predict_two_components(self, team_1, team_2, team_1b, team_2b, neutral=False):
        """
        Returns team 1's probability of winning
        """
        if self.latent_variables.estimated is False:
            raise Exception("No latent variables estimated!")
        else:
            if type(team_1) == str:
                team_1_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[self.team_dict[team_1]], trim='b')[-1]
                team_2_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[self.team_dict[team_2]], trim='b')[-1]
                team_1_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[self.team_dict[team_1]], trim='b')[-1]
                team_2_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[self.team_dict[team_2]], trim='b')[-1]
  
            else:
                team_1_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[team_1], trim='b')[-1]
                team_2_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[team_2], trim='b')[-1]
                team_1_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[team_1_b], trim='b')[-1]
                team_2_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[team_2_b], trim='b')[-1]

        t_z = self.transform_z()

        if neutral is False:
            return self.link(t_z[0] + team_1_ability - team_2_ability + team_1_b_ability - team_2_b_ability)
        else:
            return self.link(team_1_ability - team_2_ability + team_1_b_ability - team_2_b_ability)

    def clean_flux(self,flux, xDef = 1, lambdas = np.array([])):
        '''clean a flux array to cross corralate to determine RV shift
            eliminates NaNs
            moving median to reduce peaks
            optional: increase resolution by xDef times
            
        '''	
        
        #Copy to output in case of no resampling
        fluxHD = flux
        newLambdas = lambdas
        
        
        #clean NaNs and median outliers	
        fluxHD[np.isnan(fluxHD)] = 0
        fluxNeat = fluxHD	
        fluxMed = signal.medfilt(fluxHD,5)
        w = np.where(abs((fluxHD-fluxMed)/np.maximum(fluxMed,50)) > 0.4)
        for ix in w[0]:
            fluxNeat[ix] = fluxMed[ix]

        #if enough data -> resample
        if ((xDef>1) and (len(lambdas)>0)):
            fFluxHD = interpolate.interp1d(lambdas,fluxNeat) 
            lambdas = np.arange(min(lambdas), max(lambdas),(max(lambdas)-min(lambdas))/len(lambdas)/xDef)
            fluxNeat = fFluxHD(lambdas)
        
        #remove trailing zeros, devide by fitted curve (flatten) and apply tukey window
        fluxNeat = np.trim_zeros(fluxNeat,'f') 
        lambdas = lambdas[-len(fluxNeat):]
        fluxNeat = np.trim_zeros(fluxNeat,'b') 
        lambdas = lambdas[:len(fluxNeat)]
        
        if ((lambdas.shape[0]>0) &  (fluxNeat.shape[0]>0)):
            fFluxNeat = optimize.curve_fit(self.cubic, lambdas, fluxNeat, p0 = [1,1,1,1])
            fittedCurve = self.cubic(lambdas, fFluxNeat[0][0], fFluxNeat[0][1], fFluxNeat[0][2], fFluxNeat[0][3])
        # 	plt.plot(fittedCurve)
        # 	plt.plot(fluxNeat)
        # 	plt.show()
        # 	plt.plot(fluxNeat/fittedCurve-1)
        # 	plt.show()
            
            fluxFlat = fluxNeat/fittedCurve-1
            
            fluxWindow = fluxFlat * self.tukey(0.1, len(fluxFlat))
        else:
            fluxWindow = np.array([])
            print 'empty after removing zeros'
            
        return lambdas, fluxWindow

def trim_zeros(track, front=True, back=True):
    if front:
        values = np.trim_zeros(track.values, trim="f")
        times = track.times[len(track) - len(values):]
    else:
        values = track.values
        times = track.times
    if back:
        vallen = len(values)
        values = np.trim_zeros(values, trim="b")
        if vallen != len(values):
            times = times[:len(values) - vallen]

    track.values = values
    track.times = times

    def add_to_history(self, state, action):
        """
        Adds a state, action pair to the history of the current episode, or increases its counter if already present
        :param state:
        :param action:
        :return:
        """

        state = tuple(np.trim_zeros(state, 'f'))
        action = tuple(np.trim_zeros(action, 'f'))

        if (state, action) in self.state_action_history:
            self.state_action_history[(state, action)] += 1
        else:
            self.state_action_history[(state, action)] = 1

    def slotted_autocorrelation(self, data, time, T, K,
                                second_round=False, K1=100):

        slots = np.zeros((K, 1))
        i = 1

        # make time start from 0
        time = time - np.min(time)

        # subtract mean from mag values
        m = np.mean(data)
        data = data - m

        prod = np.zeros((K, 1))
        pairs = np.subtract.outer(time, time)
        pairs[np.tril_indices_from(pairs)] = 10000000

        ks = np.int64(np.floor(np.abs(pairs) / T + 0.5))

        # We calculate the slotted autocorrelation for k=0 separately
        idx = np.where(ks == 0)
        prod[0] = ((sum(data ** 2) + sum(data[idx[0]] *
                   data[idx[1]])) / (len(idx[0]) + len(data)))
        slots[0] = 0

        # We calculate it for the rest of the ks
        if second_round is False:
            for k in np.arange(1, K):
                idx = np.where(ks == k)
                if len(idx[0]) != 0:
                    prod[k] = sum(data[idx[0]] * data[idx[1]]) / (len(idx[0]))
                    slots[i] = k
                    i = i + 1
                else:
                    prod[k] = np.infty
        else:
            for k in np.arange(K1, K):
                idx = np.where(ks == k)
                if len(idx[0]) != 0:
                    prod[k] = sum(data[idx[0]] * data[idx[1]]) / (len(idx[0]))
                    slots[i - 1] = k
                    i = i + 1
                else:
                    prod[k] = np.infty
            np.trim_zeros(prod, trim='b')

        slots = np.trim_zeros(slots, trim='b')
        return prod / prod[0], np.int64(slots).flatten()