Force-Directed Graph Visualization of Cloud Data

So, I've written about the process of loading the csv cloud data and sanatizing it in this post, but now I'm going to take those skills and try to visualize the data in some way using d3.js. Since I don't quite know how d3.js works, I'm going to try to reuse their example.

First, we load and clean-up the data in the same way as the earlier post.

In [1]:
var csv = require('csv');
var fs = require('fs');
var cloud_data = [];

var parser = csv.parse({delimiter: ',', columns: true}, function(err, data){
  cloud_data = data;
}
                      );

fs.createReadStream('2015-11-16-cloud_data.csv').pipe(parser);

So, we've loaded the csv, let's clean it up. First, we fill in blank sceduled times with actual times and second we delete the entirely blank records.

In [2]:
for (eachRowIndex in cloud_data) {
    if (cloud_data[eachRowIndex].Scheduled === "") {
        cloud_data[eachRowIndex].Scheduled = cloud_data[eachRowIndex].Observed;
    }
}

for (eachRowIndex in cloud_data) {
    if (cloud_data[eachRowIndex].Scheduled === "") {
        delete cloud_data[eachRowIndex];
    }
}

There are a lot of ways to do something with this data, but let's do something simple as a first experiment. Let's make a force-directed graph of the words in any description and draw an edge between words which both occur on the same day. Let's put "Present", "Location", "Observation", and "Conditions" into separately colored groups though.

In [3]:
var cloudJson = {"nodes":[], "links":[]}; //nodes and edges to draw

function getNodeIndex (cloudJsonIn, searchWordIn) {
    for ( var nodeIndex = 0 ; nodeIndex < cloudJsonIn.nodes.length ; nodeIndex++) {
        if (cloudJsonIn.nodes[nodeIndex].name == searchWordIn ) {
            return nodeIndex;
        }
    }
    return false;
}


//first, let's get the Present nodes out
for ( var eachRowIndex in cloud_data) { //iterate through each row of data
    
    var words = cloud_data[eachRowIndex].Present.match(/\w+/g); //extracts the words
    for ( var eachWordIndex in words) {
        if (getNodeIndex(cloudJson, words[eachWordIndex]) === false) {
        cloudJson.nodes.push({"name":words[eachWordIndex],"group":1}); //adds new words to the cloudNodes
    }}
    
    //now, count through each word and link to the later words
    for (var fromEachWordIndex = 0 ; fromEachWordIndex <= words.length ; fromEachWordIndex++) {
        var toEachWordIndex = fromEachWordIndex + 1;
        while (toEachWordIndex < words.length) {
        cloudJson.links.push({
            "source": getNodeIndex(cloudJson,words[fromEachWordIndex]),
            "target": getNodeIndex(cloudJson,words[toEachWordIndex]),
            "value": 1
        });
        toEachWordIndex++;    
    }}
}

Now we have an object with nodes and edges that matches the example code, let's open a file, stringify the json, and export.

In [4]:
JSON.stringify(cloudJson);
Out[4]:
'{"nodes":[{"name":"JA","group":1},{"name":"RC","group":1},{"name":"neighbors","group":1},{"name":"passing","group":1},{"name":"car","group":1},{"name":"AW","group":1},{"name":"birds","group":1},{"name":"traffic","group":1},{"name":"upstairs","group":1},{"name":"JD","group":1},{"name":"M","group":1},{"name":"BS","group":1},{"name":"and","group":1},{"name":"Lundy","group":1},{"name":"near","group":1},{"name":"crickets","group":1},{"name":"radio","group":1},{"name":"two","group":1},{"name":"trash","group":1},{"name":"dudes","group":1},{"name":"inside","group":1},{"name":"cars","group":1},{"name":"Tia","group":1},{"name":"MER","group":1},{"name":"8","group":1},{"name":"students","group":1},{"name":"6","group":1},{"name":"bikes","group":1},{"name":"plane","group":1},{"name":"ER","group":1},{"name":"JB","group":1},{"name":"PL","group":1},{"name":"LW","group":1},{"name":"Rosie","group":1},{"name":"music","group":1},{"name":"outside","group":1},{"name":"construction","group":1},{"name":"neighbor","group":1},{"name":"raking","group":1},{"name":"children","group":1},{"name":"EH","group":1},{"name":"playing","group":1},{"name":"squirrel","group":1},{"name":"passengers","group":1},{"name":"trains","group":1},{"name":"BH","group":1},{"name":"quiet","group":1},{"name":"_many_","group":1},{"name":"more","group":1},{"name":"train","group":1},{"name":"people","group":1},{"name":"on","group":1},{"name":"the","group":1},{"name":"street","group":1},{"name":"pedestrians","group":1},{"name":"siren","group":1},{"name":"Frank","group":1},{"name":"dude","group":1},{"name":"phone","group":1},{"name":"so","group":1},{"name":"many","group":1},{"name":"crossing","group":1},{"name":"etc","group":1},{"name":"conference","group":1},{"name":"smokers","group":1},{"name":"LO","group":1},{"name":"ES","group":1},{"name":"dogs","group":1},{"name":"horses","group":1}],"links":[{"source":0,"target":1,"value":1},{"source":0,"target":2,"value":1},{"source":0,"target":3,"value":1},{"source":0,"target":4,"value":1},{"source":2,"target":3,"value":1},{"source":2,"target":4,"value":1},{"source":3,"target":4,"value":1},{"source":0,"target":5,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":1,"value":1},{"source":0,"target":7,"value":1},{"source":5,"target":6,"value":1},{"source":5,"target":1,"value":1},{"source":5,"target":7,"value":1},{"source":6,"target":1,"value":1},{"source":6,"target":7,"value":1},{"source":1,"target":7,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":1,"value":1},{"source":0,"target":8,"value":1},{"source":6,"target":1,"value":1},{"source":6,"target":8,"value":1},{"source":1,"target":8,"value":1},{"source":0,"target":9,"value":1},{"source":0,"target":10,"value":1},{"source":0,"target":11,"value":1},{"source":0,"target":12,"value":1},{"source":0,"target":13,"value":1},{"source":0,"target":14,"value":1},{"source":0,"target":15,"value":1},{"source":9,"target":10,"value":1},{"source":9,"target":11,"value":1},{"source":9,"target":12,"value":1},{"source":9,"target":13,"value":1},{"source":9,"target":14,"value":1},{"source":9,"target":15,"value":1},{"source":10,"target":11,"value":1},{"source":10,"target":12,"value":1},{"source":10,"target":13,"value":1},{"source":10,"target":14,"value":1},{"source":10,"target":15,"value":1},{"source":11,"target":12,"value":1},{"source":11,"target":13,"value":1},{"source":11,"target":14,"value":1},{"source":11,"target":15,"value":1},{"source":12,"target":13,"value":1},{"source":12,"target":14,"value":1},{"source":12,"target":15,"value":1},{"source":13,"target":14,"value":1},{"source":13,"target":15,"value":1},{"source":14,"target":15,"value":1},{"source":0,"target":11,"value":1},{"source":0,"target":13,"value":1},{"source":0,"target":15,"value":1},{"source":11,"target":13,"value":1},{"source":11,"target":15,"value":1},{"source":13,"target":15,"value":1},{"source":0,"target":16,"value":1},{"source":0,"target":17,"value":1},{"source":0,"target":18,"value":1},{"source":0,"target":19,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":9,"value":1},{"source":0,"target":20,"value":1},{"source":17,"target":18,"value":1},{"source":17,"target":19,"value":1},{"source":17,"target":6,"value":1},{"source":17,"target":9,"value":1},{"source":17,"target":20,"value":1},{"source":18,"target":19,"value":1},{"source":18,"target":6,"value":1},{"source":18,"target":9,"value":1},{"source":18,"target":20,"value":1},{"source":19,"target":6,"value":1},{"source":19,"target":9,"value":1},{"source":19,"target":20,"value":1},{"source":6,"target":9,"value":1},{"source":6,"target":20,"value":1},{"source":9,"target":20,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":21,"value":1},{"source":6,"target":21,"value":1},{"source":0,"target":22,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":15,"value":1},{"source":22,"target":7,"value":1},{"source":22,"target":15,"value":1},{"source":7,"target":15,"value":1},{"source":0,"target":23,"value":1},{"source":0,"target":15,"value":1},{"source":0,"target":7,"value":1},{"source":23,"target":15,"value":1},{"source":23,"target":7,"value":1},{"source":15,"target":7,"value":1},{"source":0,"target":15,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":9,"value":1},{"source":0,"target":20,"value":1},{"source":0,"target":7,"value":1},{"source":6,"target":9,"value":1},{"source":6,"target":20,"value":1},{"source":6,"target":7,"value":1},{"source":9,"target":20,"value":1},{"source":9,"target":7,"value":1},{"source":20,"target":7,"value":1},{"source":0,"target":24,"value":1},{"source":0,"target":25,"value":1},{"source":0,"target":26,"value":1},{"source":0,"target":27,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":28,"value":1},{"source":0,"target":7,"value":1},{"source":24,"target":25,"value":1},{"source":24,"target":26,"value":1},{"source":24,"target":27,"value":1},{"source":24,"target":6,"value":1},{"source":24,"target":28,"value":1},{"source":24,"target":7,"value":1},{"source":25,"target":26,"value":1},{"source":25,"target":27,"value":1},{"source":25,"target":6,"value":1},{"source":25,"target":28,"value":1},{"source":25,"target":7,"value":1},{"source":26,"target":27,"value":1},{"source":26,"target":6,"value":1},{"source":26,"target":28,"value":1},{"source":26,"target":7,"value":1},{"source":27,"target":6,"value":1},{"source":27,"target":28,"value":1},{"source":27,"target":7,"value":1},{"source":6,"target":28,"value":1},{"source":6,"target":7,"value":1},{"source":28,"target":7,"value":1},{"source":0,"target":29,"value":1},{"source":0,"target":30,"value":1},{"source":0,"target":31,"value":1},{"source":0,"target":32,"value":1},{"source":0,"target":26,"value":1},{"source":0,"target":25,"value":1},{"source":29,"target":30,"value":1},{"source":29,"target":31,"value":1},{"source":29,"target":32,"value":1},{"source":29,"target":26,"value":1},{"source":29,"target":25,"value":1},{"source":30,"target":31,"value":1},{"source":30,"target":32,"value":1},{"source":30,"target":26,"value":1},{"source":30,"target":25,"value":1},{"source":31,"target":32,"value":1},{"source":31,"target":26,"value":1},{"source":31,"target":25,"value":1},{"source":32,"target":26,"value":1},{"source":32,"target":25,"value":1},{"source":26,"target":25,"value":1},{"source":0,"target":15,"value":1},{"source":0,"target":4,"value":1},{"source":0,"target":2,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":33,"value":1},{"source":0,"target":11,"value":1},{"source":15,"target":4,"value":1},{"source":15,"target":2,"value":1},{"source":15,"target":7,"value":1},{"source":15,"target":33,"value":1},{"source":15,"target":11,"value":1},{"source":4,"target":2,"value":1},{"source":4,"target":7,"value":1},{"source":4,"target":33,"value":1},{"source":4,"target":11,"value":1},{"source":2,"target":7,"value":1},{"source":2,"target":33,"value":1},{"source":2,"target":11,"value":1},{"source":7,"target":33,"value":1},{"source":7,"target":11,"value":1},{"source":33,"target":11,"value":1},{"source":0,"target":5,"value":1},{"source":0,"target":15,"value":1},{"source":0,"target":7,"value":1},{"source":5,"target":15,"value":1},{"source":5,"target":7,"value":1},{"source":15,"target":7,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":7,"value":1},{"source":6,"target":7,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":34,"value":1},{"source":0,"target":35,"value":1},{"source":7,"target":34,"value":1},{"source":7,"target":35,"value":1},{"source":34,"target":35,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":36,"value":1},{"source":0,"target":37,"value":1},{"source":0,"target":38,"value":1},{"source":0,"target":39,"value":1},{"source":7,"target":36,"value":1},{"source":7,"target":37,"value":1},{"source":7,"target":38,"value":1},{"source":7,"target":39,"value":1},{"source":36,"target":37,"value":1},{"source":36,"target":38,"value":1},{"source":36,"target":39,"value":1},{"source":37,"target":38,"value":1},{"source":37,"target":39,"value":1},{"source":38,"target":39,"value":1},{"source":0,"target":40,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":15,"value":1},{"source":0,"target":39,"value":1},{"source":0,"target":41,"value":1},{"source":0,"target":2,"value":1},{"source":0,"target":42,"value":1},{"source":40,"target":7,"value":1},{"source":40,"target":6,"value":1},{"source":40,"target":15,"value":1},{"source":40,"target":39,"value":1},{"source":40,"target":41,"value":1},{"source":40,"target":2,"value":1},{"source":40,"target":42,"value":1},{"source":7,"target":6,"value":1},{"source":7,"target":15,"value":1},{"source":7,"target":39,"value":1},{"source":7,"target":41,"value":1},{"source":7,"target":2,"value":1},{"source":7,"target":42,"value":1},{"source":6,"target":15,"value":1},{"source":6,"target":39,"value":1},{"source":6,"target":41,"value":1},{"source":6,"target":2,"value":1},{"source":6,"target":42,"value":1},{"source":15,"target":39,"value":1},{"source":15,"target":41,"value":1},{"source":15,"target":2,"value":1},{"source":15,"target":42,"value":1},{"source":39,"target":41,"value":1},{"source":39,"target":2,"value":1},{"source":39,"target":42,"value":1},{"source":41,"target":2,"value":1},{"source":41,"target":42,"value":1},{"source":2,"target":42,"value":1},{"source":0,"target":15,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":29,"value":1},{"source":15,"target":7,"value":1},{"source":15,"target":29,"value":1},{"source":7,"target":29,"value":1},{"source":0,"target":15,"value":1},{"source":0,"target":7,"value":1},{"source":15,"target":7,"value":1},{"source":0,"target":6,"value":1},{"source":0,"target":16,"value":1},{"source":0,"target":7,"value":1},{"source":6,"target":16,"value":1},{"source":6,"target":7,"value":1},{"source":16,"target":7,"value":1},{"source":0,"target":43,"value":1},{"source":0,"target":44,"value":1},{"source":0,"target":45,"value":1},{"source":43,"target":44,"value":1},{"source":43,"target":45,"value":1},{"source":44,"target":45,"value":1},{"source":0,"target":46,"value":1},{"source":0,"target":4,"value":1},{"source":0,"target":47,"value":1},{"source":0,"target":48,"value":1},{"source":0,"target":43,"value":1},{"source":46,"target":4,"value":1},{"source":46,"target":47,"value":1},{"source":46,"target":48,"value":1},{"source":46,"target":43,"value":1},{"source":4,"target":47,"value":1},{"source":4,"target":48,"value":1},{"source":4,"target":43,"value":1},{"source":47,"target":48,"value":1},{"source":47,"target":43,"value":1},{"source":48,"target":43,"value":1},{"source":0,"target":43,"value":1},{"source":0,"target":49,"value":1},{"source":43,"target":49,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":50,"value":1},{"source":0,"target":51,"value":1},{"source":0,"target":52,"value":1},{"source":0,"target":53,"value":1},{"source":7,"target":50,"value":1},{"source":7,"target":51,"value":1},{"source":7,"target":52,"value":1},{"source":7,"target":53,"value":1},{"source":50,"target":51,"value":1},{"source":50,"target":52,"value":1},{"source":50,"target":53,"value":1},{"source":51,"target":52,"value":1},{"source":51,"target":53,"value":1},{"source":52,"target":53,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":54,"value":1},{"source":7,"target":54,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":55,"value":1},{"source":0,"target":56,"value":1},{"source":7,"target":55,"value":1},{"source":7,"target":56,"value":1},{"source":55,"target":56,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":36,"value":1},{"source":0,"target":57,"value":1},{"source":0,"target":51,"value":1},{"source":0,"target":58,"value":1},{"source":7,"target":36,"value":1},{"source":7,"target":57,"value":1},{"source":7,"target":51,"value":1},{"source":7,"target":58,"value":1},{"source":36,"target":57,"value":1},{"source":36,"target":51,"value":1},{"source":36,"target":58,"value":1},{"source":57,"target":51,"value":1},{"source":57,"target":58,"value":1},{"source":51,"target":58,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":36,"value":1},{"source":0,"target":50,"value":1},{"source":7,"target":36,"value":1},{"source":7,"target":50,"value":1},{"source":36,"target":50,"value":1},{"source":0,"target":50,"value":1},{"source":0,"target":7,"value":1},{"source":50,"target":7,"value":1},{"source":0,"target":59,"value":1},{"source":0,"target":60,"value":1},{"source":0,"target":50,"value":1},{"source":59,"target":60,"value":1},{"source":59,"target":50,"value":1},{"source":60,"target":50,"value":1},{"source":0,"target":50,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":55,"value":1},{"source":50,"target":7,"value":1},{"source":50,"target":55,"value":1},{"source":7,"target":55,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":50,"value":1},{"source":7,"target":50,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":36,"value":1},{"source":0,"target":57,"value":1},{"source":0,"target":61,"value":1},{"source":0,"target":52,"value":1},{"source":0,"target":53,"value":1},{"source":0,"target":62,"value":1},{"source":7,"target":36,"value":1},{"source":7,"target":57,"value":1},{"source":7,"target":61,"value":1},{"source":7,"target":52,"value":1},{"source":7,"target":53,"value":1},{"source":7,"target":62,"value":1},{"source":36,"target":57,"value":1},{"source":36,"target":61,"value":1},{"source":36,"target":52,"value":1},{"source":36,"target":53,"value":1},{"source":36,"target":62,"value":1},{"source":57,"target":61,"value":1},{"source":57,"target":52,"value":1},{"source":57,"target":53,"value":1},{"source":57,"target":62,"value":1},{"source":61,"target":52,"value":1},{"source":61,"target":53,"value":1},{"source":61,"target":62,"value":1},{"source":52,"target":53,"value":1},{"source":52,"target":62,"value":1},{"source":53,"target":62,"value":1},{"source":0,"target":63,"value":1},{"source":0,"target":64,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":36,"value":1},{"source":63,"target":64,"value":1},{"source":63,"target":7,"value":1},{"source":63,"target":36,"value":1},{"source":64,"target":7,"value":1},{"source":64,"target":36,"value":1},{"source":7,"target":36,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":50,"value":1},{"source":0,"target":36,"value":1},{"source":0,"target":65,"value":1},{"source":0,"target":66,"value":1},{"source":7,"target":50,"value":1},{"source":7,"target":36,"value":1},{"source":7,"target":65,"value":1},{"source":7,"target":66,"value":1},{"source":50,"target":36,"value":1},{"source":50,"target":65,"value":1},{"source":50,"target":66,"value":1},{"source":36,"target":65,"value":1},{"source":36,"target":66,"value":1},{"source":65,"target":66,"value":1},{"source":0,"target":65,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":50,"value":1},{"source":0,"target":67,"value":1},{"source":0,"target":68,"value":1},{"source":65,"target":7,"value":1},{"source":65,"target":50,"value":1},{"source":65,"target":67,"value":1},{"source":65,"target":68,"value":1},{"source":7,"target":50,"value":1},{"source":7,"target":67,"value":1},{"source":7,"target":68,"value":1},{"source":50,"target":67,"value":1},{"source":50,"target":68,"value":1},{"source":67,"target":68,"value":1},{"source":0,"target":7,"value":1},{"source":0,"target":50,"value":1},{"source":7,"target":50,"value":1}]}'
In [ ]: