def fetch(self, oid, format="graphml", max_age=0):
     # If the file was already fetched check the timestamp and overwrite
     graphml = os.path.join(self.cache_dir, oid+ ".graphml")
     graphpng = os.path.join(self.cache_dir, oid+ ".png")
     graphgexf  = os.path.join(self.cache_dir, oid+ ".gexf")
     logger.debug("Fetching "+graphml)
     if os.path.exists(graphml):
         ## cache hit was old and we have to refresh it
         if int(time.time()) - os.path.getmtime(graphml ) > max_age:
             DG=nx.read_graphml(graphml)
             labels=dict((n,d['label']) for n,d in DG.nodes(data=True))
             nx.draw_networkx(DG,labels=labels)
             logger.debug("Generated graph "+graphpng);
             plt.savefig(graphpng)
             nx.write_gexf(DG, graphgexf )
     else: 
         logger.debug("Cache miss");
         ## cache miss we have to generate the graph, this will take time!
         return None
     if format=="graphml": 
         return json.dumps(['URL', graphml])
     elif format=="png": 
         return json.dumps(['URL', graphpng])
     elif format=="png": 
         return json.dumps(['URL', graphgexf ])

def NXexportDoubleMatAsGraph(matrix,TElist,TEdict,TEfamilydict,nameout,namestatout):
	fileout=open(namestatout,'w')
	size=matrix.shape
	graph=nx.DiGraph()
	i=0
	j=0
	L=len(TElist)
	#k=0
	#l=0
	#m=0
	while i<size[0]:
		while j<size[1]: #safest call
			if matrix[i,j]>0:
				if i>=L and j>=L:
					graph.add_edge("-"+TElist[i-L],"-"+TElist[j-L],weight=matrix[i,j])
					fileout.write("-"+TElist[i-L]+"\t-"+TElist[j-L]+"\t"+TEfamilydict[TElist[i-L]]+"\t"+TEfamilydict[TElist[j-L]]+"\t"+str(matrix[i,j])+"\n")
					#k+=matrix[i,j]
				elif i>=L:
					graph.add_edge("-"+TElist[i-L],TElist[j],weight=matrix[i,j])
					fileout.write("-"+TElist[i-L]+"\t"+TElist[j]+"\t"+TEfamilydict[TElist[i-L]]+"\t"+TEfamilydict[TElist[j]]+"\t"+str(matrix[i,j])+"\n")
					#l+=matrix[i,j]
				elif j>=L:
					graph.add_edge(TElist[i],"-"+TElist[j-L],weight=matrix[i,j])
					fileout.write(TElist[i]+"\t-"+TElist[j-L]+"\t"+TEfamilydict[TElist[i]]+"\t"+TEfamilydict[TElist[j-L]]+"\t"+str(matrix[i,j])+"\n")
					#m+=matrix[i,j]
				else:
					graph.add_edge(TElist[i],TElist[j],weight=matrix[i,j])
					fileout.write(TElist[i]+"\t"+TElist[j]+"\t"+TEfamilydict[TElist[i]]+"\t"+TEfamilydict[TElist[j]]+"\t"+str(matrix[i,j])+"\n")
			j+=1
		j=0
		i+=1
	#print(k,l,m) #unitary test
	#pos=nx.spring_layout(G) # positions for all nodes : not need if export in gexf
	nx.write_gexf(graph,nameout)
	fileout.close()

def convertNetToGefx(input_file):
    G = None
    if input_file.endswith(Constants.GEXF_FORMAT):
        G = nx.read_gexf(input_file, None, True)
    elif input_file.endswith(Constants.NET_FORMAT):
        G=nx.Graph()
        f = file(input_file, 'r')
        # iterate over the lines in the file
        for line in f:
            # split the line into a list of column values
            columns = line.split('\t')
            # clean any whitespace off the items
            columns = [col.strip() for col in columns]
            if columns:
                G.add_edge(columns[0], columns[1])
        #write to a gexf file, so that GHOST can read it as well
        gexf_path = input_file[:-len(Constants.NET_FORMAT)]+Constants.GEXF_FORMAT
        #add attributes to nodes in gefx file
        for n,d in G.nodes_iter(data=True):
            G.node[n]["id"] = n
            G.node[n]["gname"] = n
        nx.write_gexf(G, gexf_path)
    else:
        print("Unsupported Format")
        exit(0)
    print("For "+input_file+" Number of Nodes =", G.number_of_nodes(), "No of edges = ", G.number_of_edges())

    return G

def build_thesis_genealogy():
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()    
    query = "SELECT thesis.author_id, advisor.person_id FROM thesis, advisor WHERE thesis.id = advisor.thesis_id"
    cursor.execute(query)
    G = nx.DiGraph()
    for thesis in cursor:
        G.add_edge(thesis[1], thesis[0])
        
    i = 0        
    for n in G.nodes():
        try:
            node = str(n)
            G.node[n]["name"] = persons_id[node]
            try:
                G.node[n]["university"] = persons_university[node]["university"]["name"]
                G.node[n]["location"] = persons_university[node]["university"]["location"]
                i += 1
            except:
                G.node[n]["university"] = "none"
                G.node[n]["location"] = "none"
        except:
            print n
    
    print "Total persons with a location:", i
    cursor.close()
    cnx.close()
        
    nx.write_gexf(G, "./networks/genealogy.gexf")
    return G

def scrap_dbpedia():
    G = nx.DiGraph()
    
    for querie in queries:
        print 'Executing querie: ' + querie
        payload = {'query': querie,
                    'format': 'json'}
        r = requests.get("http://dbpedia.org/sparql/", params=payload)
        results = r.json()['results']['bindings']
        for result in results:
            advisor = result['advisor']['value']
            advisorName = result['labelAdvisor']['value']
            student = result['student']['value']
            studentName = result['labelStudent']['value']
            print u'Advisor:', advisor, u'student:', student
            add_node(G, advisor, advisorName)
            add_node(G, student, studentName)
            add_edge(G, advisor, student)
    
    print ''
    print '-Nodes: '
    print len(G.nodes())
    print '-Edges: '
    print len(G.edges())
    print 'Writing file'
    nx.write_gexf(G, 'dbpedia_genealogy.gexf')
    print 'Done'

 def as_gexf(self):
     """
     Return the graph as GEXF for download
     """
     sio = StringIO.StringIO()
     nx.write_gexf(self.graph_with_metadata, sio)
     return sio.getvalue()

def main():
    n = int(sys.argv[1])
    out = sys.argv[2]
    ans = float(sys.argv[3]) if len(sys.argv) >= 4 else None
    G = nx.Graph()

    # create nodes randomly placed in [0,100)x[0,100)
    P = [None] * n
    for i in xrange(n):
        G.add_node(i, x=random.random() * 100, y=random.random() * 100)

    # create a complete weighted graph using Euclidian distances
    for i in xrange(n):
        for j in xrange(i + 1, n):
            G.add_edge(i, j, weight=euclidian(G, i, j))

    # embed a tour with edge weight = ans (usually 0)
    if ans is not None:
        T = list(G.nodes())
        random.shuffle(T)
        for i in xrange(1, len(T)):
            G.edge[T[i - 1]][T[i]]["weight"] = ans
        G.edge[T[-1]][T[0]]["weight"] = ans
        print T

    nx.write_gexf(G, out)
    print "n=%d m=%d" % (G.number_of_nodes(), G.number_of_edges())

def adjlist2gexf(fAdjlist, bIntNode=1):
    '''
    Converts a graph in the adjacency list format to the GEXF format.

    input parameters:
          fAdjlist:   The file name of the adjacency list
          bIntNode:   Indicates if the node type is integer. The default is 1
                      (i.e., nodes are interger type).
    
    returns:
          None

    output:
          This function generates an GEXF format file with the same name the 
          input file, with .gexf extension.

    '''
    # first, loading the graph
    if bIntNode==1:
        G = nx.read_adjlist(fAdjlist, nodetype=int)
    else:
        G = nx.read_adjlist(fAdjlist)

    # the output file name
    (fOutRoot,tmpExt) = os.path.splitext(fAdjlist)
    fOut = fOutRoot + '.gexf'

    # writing out
    nx.write_gexf(G, fOut)

def build(task):
    """Build network based on task"""
    try:
        graph = nx.DiGraph()
        with open(task["file"]) as df:
            for line in df:
                action = json.loads(line)
                user = action["user"]
                actions = action["actions"]
                graph.add_node(user, tweets=actions["tweets"])

                #handle mentions
                men_counter = Counter(actions["mentions"])
                for t_user, num in men_counter.iteritems():
                    graph.add_edge(user, t_user, mentions=num)

                #handle retweet
                retweet_counter = Counter(actions["retweets"])
                for t_user, num in retweet_counter.iteritems():
                    graph.add_edge(user, t_user, retweets=num)

                #handle reply
                reply_counter = Counter(actions['replies'])
                for t_user, num in reply_counter.iteritems():
                    graph.add_edge(user, t_user, replies=num)
        nx.write_gexf(graph, task["output"])
        print 'Done[%s]' % task['file']
    except:
        print "Error:", task, sys.exc_info()[0], line

 def write_gexf(self, path, ext = '.gexf', max_edges = None):
     """Write as a GEXF output, suitable for Gephi input."""
     filename = path + ext
     print "writing GeoGraph as GEXF to %s" % filename
     gexf = self.geo_gexf_graph(max_edges)
     #gexf_path = os.path.join(name, name + '_' + '_'.join(append) + ext)
     nx.write_gexf(gexf, filename)

def main(seed):
    depth = 0
    global g

    # Connect to Neo4j
    graph_db = neo4j.GraphDatabaseService(NEODB)

    print "Starting Node Export at {0}.".format(datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ"))

    for s in seed:
        query = q.format(s)
        neo_nodes, metadata = cypher.execute(graph_db, query)

        # add the node
        g.add_node(neo_nodes[0][0])
        attr = graph_db.node(neo_nodes[0][0]).get_properties()
        attr = addNodeAttributes(attr)
        g.node[neo_nodes[0][0]] = attr
        complete.add(neo_nodes[0][0])

        # pass them to the recursive DFS
        dfs_parse_nodes(neo_nodes, depth + 1)

    print "Saving File"
    nx.write_gexf(g, GEXF_FILE)

    print "Done at {0}.".format(datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ"))

def test_real_graph(nparts):
    logging.info('Reading author collab graph')
    author_graph = nx.read_graphml('/home/amir/az/io/spam/mgraph2.gexf')
    author_graph.name = 'author graph'
    logging.info('Reading the full author product graph')
    full_graph = nx.read_graphml('/home/amir/az/io/spam/spam_graph.graphml')
    full_graph.name = 'full graph'

    proper_author_graph = author_graph.subgraph([a for a in author_graph if 'revLen' in author_graph.node[a]
                                                                            and 'hlpful_fav_unfav' in author_graph.node[a]
                                                                            and 'vrf_prchs_fav_unfav' in author_graph.node[a]])
    # features = {'revLen': 0.0, 'hlpful_fav_unfav': False, 'vrf_prchs_fav_unfav': False}
    # for a in author_graph:
    #     for feat, def_val in features.items():
    #         if feat not in author_graph.node[a]:
    #             author_graph.node[a][feat] = def_val

    # sub sample proper_author_graph
    # proper_author_graph.remove_edges_from(random.sample(proper_author_graph.edges(), 2*proper_author_graph.size()/3))
    # degree = proper_author_graph.degree()
    # proper_author_graph.remove_nodes_from([n for n in proper_author_graph if degree[n] == 0])
    # author to the product reviewed by him mapping
    logging.debug('forming the product mapping')
    author_product_mapping = {}
    for a in proper_author_graph:
        author_product_mapping[a] = [p for p in full_graph[a] if 'starRating' in full_graph[a][p] and
                                                                 full_graph[a][p]['starRating'] >= 4]
    logging.debug('Running EM')
    ll, partition = HardEM.run_EM(proper_author_graph, author_product_mapping, nparts=nparts, parallel=True)
    print 'best loglikelihood: %s' % ll
    for n in partition:
        author_graph.node[n]['cLabel'] = int(partition[n])
    nx.write_gexf(author_graph, '/home/amir/az/io/spam/spam_graph_mgraph_sage_labeled.gexf')

def write_gexf(filename, graph=None, adjacency=None, attributes=None):
    """
    Output a matrix of nodal flows to GEXF format
    """
    if graph is None:
        graph = to_graph(adjacency, attributes)
    nx.write_gexf(graph, filename, prettyprint=True)

def spectral_clustering(G, graph_name, num_clusters):
    #Find a way to figure out clusters number automatically
    subgraphs = []
    write_directory = os.path.join(Constants.SPECTRAL_PATH,graph_name)
    if not os.path.exists(write_directory):
        os.makedirs(write_directory)
    nodeList = G.nodes()
    matrix_data = nx.to_numpy_matrix(G, nodelist = nodeList)
    spectral = SpectralClustering(n_clusters=2,
                                          eigen_solver='arpack',
                                          affinity="rbf")   
    spectral.fit(matrix_data)
    label = spectral.labels_
    clusters = {}
    
    for nodeIndex, nodeLabel in enumerate(label):
        if nodeLabel not in clusters:
            clusters[nodeLabel] = []
        clusters[nodeLabel].append(nodeList[nodeIndex])
        
    #countNodes is used to test whether we have all the nodes in the clusters 
   
    for clusterIndex, subGraphNodes in enumerate(clusters.keys()):
        subgraph = G.subgraph(clusters[subGraphNodes])
        subgraphs.append(subgraph)
        nx.write_gexf(subgraph, os.path.join(write_directory,graph_name+str(clusterIndex)+"_I"+Constants.GEXF_FORMAT))
        #countNodes = countNodes + len(clusters[subGraphNodes])
    return subgraphs

 def build_from_single_tweet(self, tweet):
     """
     This will save all the hashtags from a single tweet to the graph file
     """
     entities = Extractors.getEntities(tweet)
     if len(entities['hashtags']) >= 1:
         self.load_graph()
         try:
             list_of_hashtags = entities['hashtags']
             tweet_tuples = []
             # Format each of the hashtags and add the tuple
             for tag in list_of_hashtags:
                 tag_cleaned = str(tag['text'])
                 tag_cleaned = tag_cleaned.lower()
                 tweetid = tweet['id_str']
                 tweet_tuple = (tweetid, tag_cleaned)
                 tweet_tuples.append(tweet_tuple)
             try:
                 #Add tag to the graph. Done here so that one bad edge won't bring everything down
                 self.graph.add_edges_from(tweet_tuples)
             except Exception as e:
                 print 'Error adding edge for tweet id %s and hashtag %s. Details: %s' % (tweetid, tag_cleaned, e)
             #todo Add redis log of non recorded hashtags
         finally:  # record added edges
             nx.write_gexf(self.graph, self.graphFile)

def add_lamina_LPU(config, i, lamina, manager):
    output_filename = config["Lamina"]["output_file"]
    gexf_filename = config["Lamina"]["gexf_file"]
    suffix = config["General"]["file_suffix"]

    dt = config["General"]["dt"]
    debug = config["Lamina"]["debug"]
    time_sync = config["Lamina"]["time_sync"]

    output_file = "{}{}{}.h5".format(output_filename, i, suffix)
    gexf_file = "{}{}{}.gexf.gz".format(gexf_filename, i, suffix)
    G = lamina.get_graph()
    nx.write_gexf(G, gexf_file)

    n_dict_ret, s_dict_ret = LPU.lpu_parser(gexf_file)
    lamina_id = get_lamina_id(i)
    modules = []
    manager.add(
        LPU,
        lamina_id,
        dt,
        n_dict_ret,
        s_dict_ret,
        input_file=None,
        output_file=output_file,
        device=2 * i + 1,
        debug=debug,
        time_sync=time_sync,
        modules=modules,
        input_generator=None,
    )

def Gephi_Graph(r_serv, graphpath, mincard, maxcard, insert_type):
    """Create Gephi Graph by calling a "Sub function": Create_Graph

    :param r_serv: -- connexion to redis database
    :param graphpath: -- the absolute path of the .gephi graph created.
    :param mincard: -- the minimum links between 2 nodes to be created
    :param maxcard: -- the maximum links between 2 nodes to be created
    :param insert_type: -- the type of datastructure used to create the graph.

    In fact this function is juste here to be able to choose between two kind of
    Redis database structure: One which is a Sorted set and the other a simple
    set.

    """
    g = nx.Graph()

    if (insert_type == 0):

        for h in r_serv.smembers("hash"):
            Create_Graph(r_serv, g, h, graphpath, mincard, maxcard)

    elif (insert_type == 2):

        for h in r_serv.zrange("hash", 0, -1):
            Create_Graph(r_serv, g, h, graphpath, mincard, maxcard)

    nx.write_gexf(g,graphpath)
    print nx.info(g)

def create_tracker_graph():
    cur.execute(("SELECT r.url, r.referrer, r.top_url, c.name, c.value "
                "FROM http_requests as r, http_cookies as c "
                "WHERE r.id = c.header_id "
                "AND c.http_type = 'request' "
                "AND top_url IN "
                "(SELECT DISTINCT top_url FROM http_requests LIMIT 1500)"))
    for url, ref, top, name, value in cur.fetchall():
        if ref is None or ref == '': # Empty referrer
            continue
        
        req_host = psl.get_public_suffix(urlparse(url).hostname)
        ref_host = psl.get_public_suffix(urlparse(ref).hostname)
        top_host = psl.get_public_suffix(urlparse(top).hostname)

        if top_host != ref_host: # Request that doesn't have knowledge of top url
            continue
        if ref_host == req_host: # Self loops
            continue
        if req_host == 'facebook.com': # Facebook
            continue

        # Check if identifying cookie
        for item in id_cookies.keys():
            if req_host.endswith(item) and name in id_cookies[item]:
                # If so, add nodes and edge
                G.add_node(req_host)
                G.add_node(ref_host)
                G.add_edge(ref_host, req_host)
                break
    networkx.write_gexf(G,os.path.expanduser('~/Desktop/05062014_triton.gexf'))

def build_interaction():
    ids, inverse_ids = get_all_ids()
    G = nx.DiGraph()
    cnx = mysql.connector.connect(**config)
    cursor = cnx.cursor()    
    query = "SELECT user_id, target_id, weight FROM interactions"
    cursor.execute(query)
    for relation in cursor:
        source = inverse_ids[relation[0]]
        target = inverse_ids[relation[1]]
        weight = relation[2]
        G.add_edge(source, target, weight = weight)
            
    cnx.close()
    
    print 'Nodes:', len(G.nodes())
    print 'Edges:', len(G.edges())
    nx.write_gexf(G, './sna/interactions-nonfiltered-%s-%s.gexf' % (datetime.datetime.now().month, datetime.datetime.now().day))    
    
    filter_weight(G, weight_limit = 4, degree_limit = 0)
    
    print 'Nodes:', len(G.nodes())
    print 'Edges:', len(G.edges())

    
    nx.write_gexf(G, './sna/interactions-%s-%s.gexf' % (datetime.datetime.now().month, datetime.datetime.now().day))

def main(notify):

    g = nx.Graph()
    out_filename = "data/subreddits_edged_by_description_links.gexf"
    parser = HTMLParser()
    session = Session()
    query = session.query(Subreddit)
    dbi = DBIterator(query=query)

    for subreddit in dbi.results_iter():
        sub = subreddit.url.split("/")[2].lower()

        initialize_node(g, sub)

        if not subreddit.description_html:
            continue

        html = parser.unescape(subreddit.description_html)
        for linked_sub in find_sub_links(html):
            if g.has_edge(sub, linked_sub):
                g[sub][linked_sub]["weight"] += 1
            else:
                g.add_edge(sub, linked_sub, weight=1)

    nx.write_gexf(g, out_filename)