def readDGFrameFile(filename, interRRI_norRNA=1, support_read=3):
	fn_stat_dict = nested_dict()
	inter, intra = 0, 0
	with open(filename, 'r') as TXT:
		for line in TXT:
			line = line.strip()
			if not line or line.startswith('#'):
				continue
			arr = line.split('\t')
			if arr[1] == arr[5]:
				intra += 1
			else:
				inter += 1
	fn_stat_dict['inter'] = inter
	fn_stat_dict['intra'] = intra
	fn_stat_dict['all'] = intra + inter

	df = pd.read_csv(filename, header=0, sep='\t')
	df['type'] = ['intra' if i == j else 'inter' for i,j in zip(df['lchr'], df['rchr'])]
	df_inter_RRI = df[df['type']=='inter']
	nx_inter_RRI = nx.from_pandas_dataframe(df_inter_RRI, 'lchr', 'rchr')
	fn_stat_dict['uniq RRI']  = len(nx_inter_RRI.edges())
	if interRRI_norRNA:
		df_inter_RRI = df_inter_RRI[(df_inter_RRI['ltype'].isin(['mRNA', 'lncRNA'])) & (df_inter_RRI['rtype'].isin(['mRNA', 'lncRNA']))]
	df_inter_RRI = df_inter_RRI[df_inter_RRI['support']>=support_read]
	nx_inter_RRI = nx.from_pandas_dataframe(df_inter_RRI, 'lchr', 'rchr')
	nx_inter_RRI_info_dict, G_largest = RRI_network_property2(nx_inter_RRI)
	for i,j in nx_inter_RRI_info_dict.items():
		fn_stat_dict[i] = j
	# fn_stat_df['sampling'] = ''
	fn_stat_df = pd.DataFrame(fn_stat_dict, index=[0])
	return fn_stat_df

 def test_from_dataframe_all_attr(self, ):
     Gtrue = nx.Graph([('E', 'C', {'cost': 9, 'weight': 10}),
                       ('B', 'A', {'cost': 1, 'weight': 7}),
                       ('A', 'D', {'cost': 7, 'weight': 4})])
     G = nx.from_pandas_dataframe(self.df, 0, 'b', True)
     self.assert_equal(G, Gtrue)
     # MultiGraph
     MGtrue = nx.MultiGraph(Gtrue)
     MGtrue.add_edge('A', 'D', cost=16, weight=4)
     MG = nx.from_pandas_dataframe(self.mdf, 0, 'b', True, nx.MultiGraph())
     self.assert_equal(MG, MGtrue)

def read_dir(dir='/Share/home/zhangqf7/gongjing/zebrafish/data/paris/shi-zp-5-rep-combine/downsampling_N', to_dgframe=0, get_inter_intra=1, read_nx=1, interRRI_norRNA=1, support_read=3):
	fn_ls = os.listdir(dir)
	# print fn_ls

	fn_stat_dict = nested_dict()
	downsampling_N_draw = dir + '.subnetwork.draw.pdf'
	fig,ax=plt.subplots(10,1)
	for n,fn in enumerate(fn_ls):
		print "process: %s"%(fn)
		dfFile = dir + '/' + fn + '/' + '27-DG'
		frameFile = dfFile + '.txt'
		if to_dgframe:
			paris_dg2frame.DG2Frame(dfFile=dfFile, frameFile=frameFile)
		if get_inter_intra:
			inter, intra = 0, 0
			with open(frameFile, 'r') as TXT:
				for line in TXT:
					line = line.strip()
					if not line or line.startswith('#'):
						continue
					arr = line.split('\t')
					if arr[1] == arr[5]:
						intra += 1
					else:
						inter += 1
			fn_stat_dict[fn]['inter'] = inter
			fn_stat_dict[fn]['intra'] = intra
			fn_stat_dict[fn]['all'] = intra + inter
		if read_nx:
			df = pd.read_csv(frameFile, header=0, sep='\t')
			df['type'] = ['intra' if i == j else 'inter' for i,j in zip(df['lchr'], df['rchr'])]
			df_inter_RRI = df[df['type']=='inter']
			nx_inter_RRI = nx.from_pandas_dataframe(df_inter_RRI, 'lchr', 'rchr')
			fn_stat_dict[fn]['uniq RRI']  = len(nx_inter_RRI.edges())
			if interRRI_norRNA:
				df_inter_RRI = df_inter_RRI[(df_inter_RRI['ltype'].isin(['mRNA', 'lncRNA'])) & (df_inter_RRI['rtype'].isin(['mRNA', 'lncRNA']))]
			df_inter_RRI = df_inter_RRI[df_inter_RRI['support']>=support_read]
			nx_inter_RRI = nx.from_pandas_dataframe(df_inter_RRI, 'lchr', 'rchr')
			nx_inter_RRI_info_dict, G_largest = RRI_network_property2(nx_inter_RRI)
			for i,j in nx_inter_RRI_info_dict.items():
				fn_stat_dict[fn][i] = j
			# fn_stat_dict[fn]['uniq RRI']  = len(nx_inter_RRI.edges())
			if n < 10:
				draw_graph(G_largest, ax=ax[n])
	plt.savefig(downsampling_N_draw)
	savefn = dir + '.stat.txt'
	fn_stat_df = pd.DataFrame.from_dict(fn_stat_dict)
	fn_stat_df = fn_stat_df.T
	fn_stat_df['sampling'] = fn_stat_df.index
	print fn_stat_df.head()

	fn_stat_df.to_csv(savefn, header=True, index=False, sep='\t')

	return fn_stat_df

def read_graph(filename, directed=True, sep=' ', header = None):
    """
    Create networkx graph using pandas.
    :param filename: every line (u, v)
    :param directed: boolean
    :param sep: separator in file
    :return
    """
    df = pd.read_csv(filename, sep=sep, header = header)
    if directed:
        G = nx.from_pandas_dataframe(df, 0, 1, create_using=nx.DiGraph())
    else:
        G = nx.from_pandas_dataframe(df, 0, 1)
    return G

 def test_from_dataframe_all_attr(self,):
     Gtrue = nx.Graph(
         [
             ("E", "C", {"cost": 9, "weight": 10}),
             ("B", "A", {"cost": 1, "weight": 7}),
             ("A", "D", {"cost": 7, "weight": 4}),
         ]
     )
     G = nx.from_pandas_dataframe(self.df, 0, "b", True)
     self.assert_equal(G, Gtrue)
     # MultiGraph
     MGtrue = nx.MultiGraph(Gtrue)
     MGtrue.add_edge("A", "D", cost=16, weight=4)
     MG = nx.from_pandas_dataframe(self.mdf, 0, "b", True, nx.MultiGraph())
     self.assert_equal(MG, MGtrue)

def make_graph(edgesdf, edge_attr='fdist', name=None):
	g = nx.from_pandas_dataframe(edgesdf, source='node1',
								 target='node2', edge_attr=edge_attr)
	if name:
		g.add_node('namenode', name=name)

	return g

def build_file_move_graph(file_frame):
    move_frame = file_frame[file_frame.move]
    move_graph = nx.from_pandas_dataframe(
        move_frame,
        source='file_path1', target='file_path2',
        edge_attr='hexsha', create_using=nx.DiGraph())
    return move_graph

 def __init__(self, path):
     '''Constructor'''
     edges = pd.read_csv(path, header=0)
     self.l = edges.columns.values.tolist()
     self.l.remove('source')
     self.l.remove('target')
     self.G = nx.from_pandas_dataframe(edges, 'source', 'target', self.l, create_using=nx.DiGraph())

def simple_visualization (airport_df, routes_df):
	if (airport_df is None) or (routes_df is None):
		print "Data cannot be retrieved and read"
	else:
		airport_us = airport_df[(airport_df.Country == "United States")][['Name','Lat', 'Long', 'IATA', 'ICAO']]
		us_airport_ix = airport_us.index.values
		routes_us = routes_df[(routes_df['Source Airport ID'].isin(us_airport_ix)) &
		                     (routes_df['Dest Airport ID'].isin(us_airport_ix))] #extract routes that flyies from AND to USA
		routes_us =  pd.DataFrame(routes_us.groupby(['Source Airport', 'Dest Airport']).size().reset_index(name='counts'))
		# to find number of flights in and out of an airport
		# it is similar to find number of rows in which each airport occur in either one of the 2 columns
		counts = routes_us['Source Airport'].append(routes_us.loc[routes_us['Source Airport'] != routes_us['Dest Airport'], 'Dest Airport']).value_counts()
		# create a data frame of position based on names in count
		counts = pd.DataFrame({'IATA': counts.index, 'total_flight': counts})
		pos_data = counts.merge(airport_us, on = 'IATA')

		# Create graph
		graph = nx.from_pandas_dataframe(routes_us, source = 'Source Airport', target = 'Dest Airport',
		                        edge_attr = 'counts',create_using = nx.DiGraph())

		# default graph using Networkx inbuilt graph tools
		plt.figure(figsize = (10,9))
		nx.draw_networkx(graph)
		plt.savefig("./images/networkx_basemap/map_0.png", format = "png", dpi = 300)
		plt.show()

		# Set up base map
		plt.figure(figsize=(15,20))
		m = Basemap(
		        projection='merc',
		        llcrnrlon=-180,
		        llcrnrlat=10,
		        urcrnrlon=-50,
		        urcrnrlat=70,
		        lat_ts=0,
		        resolution='l',
		        suppress_ticks=True)

		# import long lat as m attribute
		mx, my = m(pos_data['Long'].values, pos_data['Lat'].values)
		pos = {}
		for count, elem in enumerate (pos_data['IATA']):
		    pos[elem] = (mx[count], my[count])

		# draw nodes and edges and over aly on basemap
		nx.draw_networkx_nodes(G = graph, pos = pos, node_list = graph.nodes(), node_color = 'r', alpha = 0.8,
		                       node_size = [counts['total_flight'][s]*3 for s in graph.nodes()])
		nx.draw_networkx_edges(G = graph, pos = pos, edge_color='g', width = routes_us['counts']*0.75, 
		                       alpha=0.2, arrows = False)

		m.drawcountries(linewidth = 3)
		m.drawstates(linewidth = 0.2)
		m.drawcoastlines(linewidth=3)
		plt.tight_layout()
		plt.savefig("./images/networkx_basemap/map_2.png", format = "png", dpi = 300)
		plt.show()
		print ("successful visualization")
		return 0

 def test_from_dataframe_multi_attr(self,):
     Gtrue = nx.Graph(
         [
             ("E", "C", {"cost": 9, "weight": 10}),
             ("B", "A", {"cost": 1, "weight": 7}),
             ("A", "D", {"cost": 7, "weight": 4}),
         ]
     )
     G = nx.from_pandas_dataframe(self.df, 0, "b", ["weight", "cost"])
     self.assert_equal(G, Gtrue)

def make_graph(cutdf):
    """
    Convert dataframe of a list of edges into a networkx graph object

    Args:
        cutdf : pandas dataframe containing list of edges with
            features 'source' and 'target' containing node numbers
    Returns:
        networkx graph object
    """
    g = nx.from_pandas_dataframe(cutdf, 'source', 'target')
    return g

def calculate(words):
    # instantiate a dictionary to later be filled with word:miscores
    wc = defaultdict(float)
    frames = []
    print("...it will take a while. Wait a sec...")
    for word in words:
        payload = {'searchstring': word.encode('ascii'),
                   'searchpositional':'word',
                   'searchpostag':'all',
                   'contextsize':'60c',
                   'sort2':'right',
                   'terminate':'100',
                   'searchtype':'coll',
                   'mistat':'on',
                   'collocspanleft':'2',
                   'collocspanright':'2',
                   'collocfilter':'noun'}

        r = requests.get("http://clic.cimec.unitn.it/cgi-bin/cqp/cqp.pl?corpuslist=WEBBIT", params=payload)
        soup = BeautifulSoup(r.content, 'lxml')

        # parse the html table and extract words and miscores. Add scores
    
        temp = []
        for tr in soup.find_all('tr')[1:]:
            tds = tr.find_all('td')
            word = tds[0].text.split('~~')[1]
            mi = float(tds[4].text)
            wc[word] += mi
            temp.append(map(lambda x:x.text,tds[0:]))
        x = pd.DataFrame(temp)
        df = pd.DataFrame()
        df['coll'] = x.ix[0:,0].apply(lambda x: x.split('~~')[1])
        df['word'] = x.ix[0:,0].apply(lambda x: x.split('~~')[0])
        df['mi'] = x.ix[0:,4]
        frames.append(df)

    #sort the results in decreasing order        
    results = []
    for w in sorted(wc, key=wc.get, reverse=True):
        results.append((w, wc[w]))

    #spit out the top result. If using ipython you can check the rest of the list by tiping `results`
    #viz part
    results_df = pd.concat(frames)

    G=nx.from_pandas_dataframe(results_df, 'word','coll',['mi'])
    mat = nx.adjacency_matrix(G).todense()
    viz = lgn.force(mat)
    vid = viz.id
    print(vid)
    url = '<iframe src="http://public.lightning-viz.org/visualizations/'+vid+'/iframe/" width=100% height=400px>'
    return (results[0][0].strip(),url)

    def weightGraph(self, datacontacts, mi_threshold, time_treshold=0.6):
        if len(self.mol.get('resid', 'name CA')) != len(self.resids):
            raise Exception('The length of the protein doesn\'t match the Mutual Information data')
        contactcat = np.concatenate(datacontacts.dat)
        contacts_matrix = np.zeros([len(self.resids), len(self.resids)])
        for i in range(contactcat.shape[1]):
            counter = np.count_nonzero(contactcat[:, i])
            resid1 = self.residmap[self.mol.resid[datacontacts.description.atomIndexes[i][0]]]
            resid2 = self.residmap[self.mol.resid[datacontacts.description.atomIndexes[i][1]]]
            contacts_matrix[resid1][resid2] = counter

        self.graph_array = np.zeros([contacts_matrix.shape[0], contacts_matrix.shape[0]])
        mask = (self.mi_matrix > mi_threshold) & (contacts_matrix > (time_treshold * contactcat.shape[0]))
        self.graph_array[mask] = self.mi_matrix[mask]

        intermed = []
        for source in range(self.graph_array.shape[0]):
            for target in range(source, self.graph_array.shape[1]):
                if self.graph_array[source, target] != 0 and target > source:
                    intermed.append(
                        [int(self.resids[source]), int(self.resids[target]), float(self.graph_array[source, target])])
        import pandas as pd
        import networkx as nx
        from sklearn.cluster.spectral import SpectralClustering

        pd = pd.DataFrame(intermed, columns=['source', 'target', 'weight'])
        pd[['source', 'target']] = pd[['source', 'target']].astype(type('int', (int,), {}))
        pd['weight'] = pd['weight'].astype(type('float', (float,), {}))
        G = nx.from_pandas_dataframe(pd, 'source', 'target', ['weight'])
        ## setSegment
        segids = self.mol.get('segid', 'name CA')
        seg_res_dict = {key: value for (key, value) in zip(self.resids, segids) if
                        np.any(pd.loc[(pd['source'] == key)].index) or np.any(pd.loc[(pd['target'] == key)].index)}
        nx.set_node_attributes(G, 'Segment', seg_res_dict)
        ## set
        if not nx.is_connected(G):
            G = max(nx.connected_component_subgraphs(G), key=len)
        flow_cent = nx.current_flow_betweenness_centrality(G, weight='weight')
        nx.set_node_attributes(G, 'flowcent', flow_cent)
        Spectre = SpectralClustering(n_clusters=10, affinity='precomputed')
        model = Spectre.fit_predict(self.graph_array)
        model = model.astype(type('float', (float,), {}))
        spectral_dict = {key: value for (key, value) in zip(self.resids, model) if key in G.nodes()}
        nx.set_node_attributes(G, 'spectral', spectral_dict)
        self.graph = G

def corr_to_graph(roi_corrs, copy_corrs=False):
    """
    >>> import pandas as pd
    >>> import numpy as np
    >>> corrs = pd.DataFrame(np.random.rand(2,2))
    >>> corrs.index = ['A', 'B']
    >>> corrs.columns = ['A', 'B']
    >>> graph = corr_to_graph(corrs)
    >>> ab = graph['A']['B']
    >>> wt, prox, dist = ab['weight'], ab['proximity'], ab['distance']
    >>> assert wt == corrs['B']['A'] #upper triangular
    >>> assert prox == wt
    >>> assert dist == 1 - wt
    >>> assert len(graph) == 2
    """
    roi_corrs = create_convertible_corr_df(roi_corrs, copy_corrs)
    return nx.from_pandas_dataframe(roi_corrs, 'source', 'target',
                                    edge_attr=['distance', 'proximity', 'weight'])

 def test_from_datafram(self, ):
     # Pandas DataFrame
     g = nx.cycle_graph(10)
     G = nx.Graph()
     G.add_nodes_from(g)
     G.add_weighted_edges_from((u, v, u) for u, v in g.edges())
     edgelist = nx.to_edgelist(G)
     source = [s for s, t, d in edgelist]
     target = [t for s, t, d in edgelist]
     weight = [d['weight'] for s, t, d in edgelist]
     import pandas as pd
     edges = pd.DataFrame({'source': source,
                           'target': target,
                           'weight': weight})
     GG = nx.from_pandas_dataframe(edges, edge_attr='weight')
     assert_nodes_equal(sorted(G.nodes()), sorted(GG.nodes()))
     assert_edges_equal(sorted(G.edges()), sorted(GG.edges()))
     GW = nx.to_networkx_graph(edges, create_using=nx.Graph())
     assert_nodes_equal(sorted(G.nodes()), sorted(GW.nodes()))
     assert_edges_equal(sorted(G.edges()), sorted(GW.edges()))

def read_inter_RRI(inter_RRI=None, filter_rRNA=False, support_read_num=2, only_mRNA_lncRNA=False):
	if inter_RRI is None:
		inter_RRI = '/Share/home/zhangqf7/gongjing/zebrafish/data/paris/shi-zp-2/27-DG.inter.element.txt'
	df_inter_RRI = pd.read_csv(inter_RRI, header=None, sep='\t')
	if filter_rRNA:
		df_inter_RRI = df_inter_RRI[(df_inter_RRI[13] != 'rRNA') & (df_inter_RRI[14] != 'rRNA')]
	if only_mRNA_lncRNA:
		only_mRNA_lncRNA_index = (df_inter_RRI[13].isin(['mRNA', 'lncRNA'])) & (df_inter_RRI[14].isin(['mRNA', 'lncRNA']))
		df_inter_RRI = df_inter_RRI[only_mRNA_lncRNA_index]
	header_ls = ['Group', 'lchr', 'lstrand', 'lstart', 'lend', 'rchr', 'rstrand',
				'rstart', 'rend', 'support', 'lcount', 'rcount', 'score', 'ltype', 'rtype', 'RRI_type', 'lcontext', 'rcontext']
	df_inter_RRI.columns = header_ls
	df_inter_RRI = df_inter_RRI[df_inter_RRI['support'] >= support_read_num]
	print df_inter_RRI.head()
	nx_inter_RRI = nx.from_pandas_dataframe(df_inter_RRI, 'lchr', 'rchr', edge_attr=['support', 'Group'])
	print "\nread: %s"%(inter_RRI)
	print nx.info(nx_inter_RRI)
	print

	return nx_inter_RRI, df_inter_RRI

    def create_nwrk(self,
                    nodes_cols,
                    attribs_cols):
        """Return Ortho_Network.nwrk upon pandas.DataFrame.

        Parameters
        -------
        nodes_cols: list
            Columns to take as nodes.
        attribs_cols: list
            Columns to take as attributes.

        Returns
        -------
        P_CRAWLER.Ortho_Network.nwrk
            Interactions-based network. networkx.classes.graph.Graph derivate.
        """
        self.nwrk = nx.from_pandas_dataframe(self.inter_df,
                                             nodes_cols[0],
                                             nodes_cols[1],
                                             attribs_cols)

def main():
    parser = ArgumentParser()
    parser.add_argument("-i", "--infile", type=str,
            help="Table with one row per link between families, columns: ['Family1','Family2'].\
                    If not specified the program reads from stdin.")
    parser.add_argument("--maxlink", default=6, type=int,
            help="Maximum number of allowed outgoing links (edges) for a single protein family. Defaults to 6")
    parser.add_argument("--minoc", default=10, type=int,
            help="Minimum number of occurrences for linking two families. Defaults to 10")
    parser.add_argument("--trimmed_out", type=str,
            help="Write trimmed families to file")

    args = parser.parse_args()
    
    if args.infile: linkdf = pd.read_csv(args.infile, sep="\t", header=0)
    else: linkdf = pd.read_csv(sys.stdin, sep="\t", header=0)
    linkdf.fillna("",inplace=True)
    oc = count_occurrences(linkdf)
    
    ## Create graph from data frame
    g = nx.from_pandas_dataframe(linkdf,source="Node1",target="Node2")
    g.remove_node('')
    ## Trim nodes by outgoing edges and edges by occurrence
    [gt,trimmed_nodes,trimmed_edges] = trim_graph(g,args.maxlink, args.minoc, oc)
    if args.trimmed_out: pd.DataFrame(trimmed_nodes).to_csv(args.trimmed_out,sep="\t",index=False,header=False)
    logging.info("Removed "+str(len(trimmed_edges))+" links due to low occurrence")    
    logging.info("Removed "+str(len(trimmed_nodes))+" families with too many links ("+str(len(gt.nodes()))+" remaining)")
    
    ## Create clusters for graph
    clusters = cluster(gt)
    logging.info(str(len(clusters))+" clusters created")    
    cdf = pd.DataFrame(clusters).T
    ## Sort by number of families in cluster
    cdf.sort_values("num",ascending=False,inplace=True)
    cdf.index = list(range(1,len(cdf)+1))
    
    ## Write clusters sorted by size
    write(cdf)

# concatenate all these edges from all these events
# and write them to an edgelist.
# append multiple items in one go instead of this
e = e1.append(e2);
e = e.append(e3);
e.to_csv('edges.csv');

# I previously made the perp list by concatenating the perp1 perp2 and perp3 columns
# then taking unique strings
# so supposing we are starting from the point where we had perp and edge lists saved as files already
perp_list = pd.read_csv("perp_list.csv");
edges = pd.read_csv("edges.csv")
# make an empty graph
G=nx.Graph();
# feed it edges
edgelist = nx.from_pandas_dataframe(edges)
# feed it nodes
G.add_nodes_from(perp_list);
# [Commenting this code so late that I forgot what this does. To be filled in later.]
test = e.replace(to_replace=perp_list, )

# quantification of categorical traits 

# recalling what we have...
print("Total rows: {0}".format(len(data)));
print(list(data));
# quantify the traits. 1 is least governmental, 3 is most. 
data = data.replace(to_replace='Private Citizens & Property', value=1);
data = data.replace(to_replace='Journalists & Media', value=1);
data = data.replace(to_replace='Educational Institution', value=1);
data = data.replace(to_replace='Abortion Related', value=1);

        # add plot.ly plotting options
        traces.append(make_scatter(lon_cc,lat_cc))
     
    return traces

def get_coastline_traces():
    poly_paths = m.drawcoastlines().get_paths() # coastline polygon paths
    N_poly = 91  # use only the 91st biggest coastlines (i.e. no rivers)
    return polygons_to_traces(poly_paths, N_poly)

traces_cc = get_coastline_traces()


############################## FRAME ABOVE FOR MAP, FRAME BELOW FOR NODES

G=nx.from_pandas_dataframe(latLonPopulated_RT, source='tweetId',target='retweeter')

pos = {}
color = {}
for i,tweet in latLonPopulated_RT.iterrows():
    pos[tweet['tweetId'].upper()] = np.asarray([tweet['x'],tweet['y']])
    pos[tweet['retweeter'].upper()] = np.asarray([tweet['x'],tweet['y']])
    color[tweet['tweetId'].upper()] = tweet['node_color']
    color[tweet['retweeter'].upper()] = tweet['node_color']

for n,p in pos.iteritems():
    G.node[n]['pos'] = p

for n,c in color.iteritems():
    G.node[n]['color'] = c