ARC5 - Import et formatage de données de graphe¶

print fichier_nodes

partenaires_types= {}

G = nx.Graph()

with open( os.path.join(data_dir, fichier_nodes), "r") as f :
    reader = csv.DictReader(f)
    for line in reader :
        start = int(line["Début"])
        end =  int(line["Fin"])
        
        info = {
            "ville" :  line["Ville"],
            "lien" :  line["Lien"]
        }
        
        node = create_node(line["Nom"], line["Type"], start, end, info=info)
        partenaires_types[slugify(line["Nom"].decode('utf-8'))] = line["Type"]

print "%s nodes"%len(G.nodes())
print "%s edges"%len(G.edges())

show_table( "Types de nodes dans le fichier d'origine ", [n[1]["type"] for n in G.nodes(data=True)])

stored_projects={}

print fichier_projets

with open( os.path.join(data_dir, fichier_projets), "r") as f :
    reader = csv.DictReader(f)
    for line in reader :
        
        if line["Nom Projet"] and line["Orga"] != "13" and line["Orga"] != "14":

            start = int(line["Année"])
            end =  start+3
            
            # create project
            projet = create_node(line["Nom Projet"], line["Type"], start, end, orga=line["Orga"])            
            
            # porteur de projet
            porteur = create_node(line["Porteurs (nom)"], "personne", start, end)
            
            # get existing
            etablissement = G.node[get_slug(line["Etablissement"], "etablissement")]["id"]
            laboratoire = G.node[get_slug(line["Labo"], "laboratoire")]["id"]
            
            # TODOs : ville !
#             ville = G.node[get_slug(line["Ville"], "localite")]

            edges = []            
            edges.append((projet, etablissement))
            edges.append((projet, laboratoire))
            edges.append((projet, porteur))
            edges.append((laboratoire, porteur))
            
#             edges.append((etablissement, ville))
#             edges.append((laboratoire, ville))

            for e in edges :
                merge_edge_data(G, e, { "type" : line["Type"], "name" : line["Nom Projet"] })
        
        elif line["Orga"] == "13" or line["Orga"] == "14":
            start = int(line["Année"])
            end =  start+3
            stored_projects[get_slug(line["Nom Projet"], line["Type"])] = { "name" : line["Nom Projet"], "type": line["Type"], "start" : start, "end" : end, "orga" : line["Orga"] }

print "%s nodes"%len(G.nodes())
print "%s edges"%len(G.edges())
show_table( "Types de nodes ", [n[1]["type"] for n in G.nodes(data=True)])
show_table( "Projets par organisation ", [n[1]["orga"] for n in G.nodes(data=True)])

print fichier_partenaires


with open( os.path.join(data_dir,  fichier_partenaires), "r") as f :
    reader = csv.DictReader(f) 
    for i, line in enumerate(reader):
        if line["Projet"] and line["Structure"] : 

            start = int(line["début"])
            end =  int(line["fin"])
            
            type = partenaires_types[slugify(line["Structure"].decode('utf-8'))]
            
            partenaire = G.node[ get_slug( line["Structure"], type)]
    
    #         TODO : ville
    #         ville = create_node(line["Ville"], "ville", start, end)

            # get project (only those with partners)
            projet = get_project(line["Projet"], line["Type"])

            e = (partenaire["id"], projet["id"])
            merge_edge_data(G, e, { "type" : projet["type"], "name" : projet["name"] })

print "%s nodes"%len(G.nodes())
print "%s edges"%len(G.edges())
show_table( "Types de nodes après avoir ajouté les partenariats", [n[1]["type"] for n in G.nodes(data=True)])

print "before : %s nodes / %s edges"%(len(G.nodes()),len(G.edges()))

G_without_people = G.copy()

# get all persons in the graph
persons = [node[0] for node in G_without_people.nodes(data=True) if node[1]["type"] == "personne"]

for person in persons:

    # edges for a single person
    person_edges = G_without_people.edges(person)
  
    # get all nodes linked by a single person
    list_of_person_nodes = []; map(list_of_person_nodes.extend, map(list,person_edges))
    assert len(list_of_person_nodes) == len(person_edges)*2 # make sure we have all nodes
    
    clean_nodes = [n for n in list_of_person_nodes if n != person]

    if len(person_edges) > 2 : # if have less than degree of 1 then remove node

        # get data from the node to add to the edge
        data = G_without_people.node[person]
                
        # create new edges between all those
        new_edges = list(itertools.combinations(clean_nodes, 2))

        # create new edges with merge data info
        for e in new_edges:
            merge_edge_data(G_without_people, e, { "type" : "personne", "name" : None })

    # remove person from the graph
    G_without_people.remove_node(person)

print "after : %s nodes / %s edges"%(len(G_without_people.nodes()),len(G_without_people.edges()))

show_table( "Types de nodes (sans les personnes) ", [n[1]["type"] for n in G_without_people.nodes(data=True)])

# save graph without people inside
save_graph(G_without_people, '../final/ARC5-nx-with-projects.pickle')

print "before : %s nodes / %s edges"%(len(G_without_people.nodes()),len(G_without_people.edges()))

G_without_people_and_projects = G_without_people.copy()

# get all projects in the graph
projects = [node[0] for node in G_without_people_and_projects.nodes(data=True) if node[1]["type"] == "projet" or node[1]["type"] == "ADR" or node[1]["type"] == "postdoc" ]

for project in projects:

    # edges for a single person
    project_edges = G_without_people_and_projects.edges(project)
  
    # get all nodes linked by a single person
    list_of_project_nodes = []; map(list_of_project_nodes.extend, map(list, project_edges))
    assert len(list_of_project_nodes) == len(project_edges)*2 # make sure we have all nodes
    
    clean_nodes = [n for n in list_of_project_nodes if n != project]

    if len(project_edges) > 2 : # if have less than degree of 1 then remove node

        # get data from the node to add to the edge
        data = G_without_people_and_projects.node[project]
        
        # create new edges between all those
        new_edges = list(itertools.combinations(clean_nodes, 2))
                
        # parse text properly
        
        # merge data into edge info
        for e in new_edges:
            
            proj=G.node[project]
            notes = { "type" : proj["type"], "name" : proj["name"]}
            merge_edge_data(G_without_people_and_projects, e, notes) 

    # remove person from the graph
    G_without_people_and_projects.remove_node(project)


print "after : %s nodes / %s edges"%(len(G_without_people_and_projects.nodes()),len(G_without_people_and_projects.edges()))

show_table( "Types de nodes (sans personnes ni projets) ", [n[1]["type"] for n in G_without_people_and_projects.nodes(data=True)])

# save graph without projects
save_graph(G_without_people_and_projects, '../final/ARC5-nx-without-projects.pickle')

# create the graph

nodes = []
for n in G_without_people_and_projects.nodes(data=True): 
    if G_without_people_and_projects.degree(n[0]) > 0: # ignore singletons
        node = n[1]
        node["id"] = n[0]
        node["group"] = n[1]["type"]
        
        # add website and city
        node["additionalInfo"] = "**Ville** : %s \n\n "%node["info"]["ville"]
        node["additionalInfo"] += "[Consulter le site](%s)"%node["info"]["lien"]
        nodes.append(node)

print "%s nodes"%len(nodes)
        
edges = []
for i, e in enumerate(G_without_people_and_projects.edges(data=True)): 
    
    edge = e[2]
    
    # calculate edge weight
    edge["weight"] = len(edge["edge_types"])
        
    notes = ""
    team = 0
    
    for t in edge["edge_types"]:
        if t["type"] == "ADR" or t["type"] == "projet" or t["type"] == "postdoc" : 
            notes = notes  + "* **%s** : %s \n"%(project_types[t["type"]], t["name"])
        elif t["type"] == "personne":
            team = team + 1
    
    if team != 0 : 
        notes = "* Membres d'équipe en commun \n" + notes
    
    edge["additionalInfo"] = notes    
    
    edge["source"] = e[0]
    edge["target"] = e[1]
    
    edges.append(edge)

print "%s edges"%len(edges)

from topogram_client import TopogramAPIClient
import logging 
import datetime

now=datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")

# passwords
TOPOGRAM_URL = "https://app.topogram.io" # http://localhost:3000
USER = "***"
PASSWORD = "***"

# connect to the topogram instance 
topogram = TopogramAPIClient(TOPOGRAM_URL)

# topogram.create_user(USER, PASSWORD)
topogram.user_login(USER, PASSWORD)

r = topogram.create_topogram("ARC 5 - Collaborations Culture / Recherche en Rhône-Alpes")
print r["message"]
topogram_ID = r["data"][0]["_id"]

# get and backup existing nodes and edges
existing_nodes = topogram.get_nodes(topogram_ID)["data"]
url = slugify(TOPOGRAM_URL.decode('utf-8'))
with open('data/ARC5-%s-nodes-%s.json'%(url,now), 'w') as outfile:
    json.dump(existing_nodes, outfile)

existing_edges = topogram.get_edges(topogram_ID)["data"]
with open('data/ARC5-%s-edges-%s.json'%(url,now), 'w') as outfile:
    json.dump(existing_edges, outfile)

print "%s existing edges, %s existing nodes"%(len(existing_edges), len(existing_nodes))

# clear existing graph
topogram.delete_nodes([n["_id"] for n in existing_nodes])
print "nodes deleted"
topogram.delete_edges([n["_id"] for n in existing_edges])
print "edges deleted"

r = topogram.create_nodes(topogram_ID, nodes)
print "%s nodes created."%len(r["data"])
r = topogram.create_edges(topogram_ID, edges)
print "%s edges created."%len(r["data"])

print "done. Topogram is online at %s/topograms/%s/view"%(TOPOGRAM_URL, topogram_ID)