import re
filename_pattern = pattern = re.compile('^pop_(?P<pop>\d+)_G_(?P<G>\d+)_s_(?P<s>\d\.?\d*)_H_(?P<H>\d\.?\d*)_U_(?P<U>\d\.?\d*)_beta_(?P<beta>\d\.?\d*)_pi_(?P<pi>\d\.?\d*)_tau_(?P<tau>\d\.?\d*)_(?P<date>\d{4}-\w{3}-\d{1,2})_(?P<time>\d{2}-\d{2}-\d{2}-\d{6}).(?P<extension>\w+)$')
def parse_filename(fname):
    m = pattern.match(fname)
    if m:
         return m.groupdict()
    else:
        return dict()

import ujson as json
import gzip
folder = 'output/fixation/'

def process_data_file(fname):
    fpath = folder + fname
    params = parse_filename(fname)
    if not params:
        print "Failed parsing file name", fpath
        return {},[],[]
    with gzip.open(fpath) as f:
        data = json.load(f,precise_float=True)
    if not data:
        print "Failed reading data", fpath
        return {},[],[]
    data.update(params)
    W = data.pop('W')
    p = data.pop('p')
    data['fname'] = fname
    for k in ['tau',  'G',  'H',  'pop',  'beta',  'U',  'T',  'pop_size',  's',  'pi']:
        if str == type(data[k]):
            data[k] = eval(data[k])  
    return data, W, p

import glob, os, time
tic = time.clock()
file_list = glob.glob1(folder, '*.data')
all_data = [None] * len(file_list)
print "processing", len(file_list), "data files"
for i,fname in enumerate(file_list) :
    data,W,p = process_data_file(fname)
    all_data[i] = data
toc = time.clock()
print "processed all files in", (toc-tic), "seconds"

df = [[data['T'],data['tau'],data['s'],data['pi']] for data in all_data]

%load_ext rmagic

%%R -i df
df <- as.data.frame(df)
names(df) <- c("T","tau","s","pi")
library(ggplot2)
p <- ggplot(df, aes(tau, T))
p <- p + 
    geom_point(alpha=I(0.3)) + 
    scale_x_log10() + scale_y_log10() + 
    facet_grid(facets=s~pi, labeller=function(variable,value) {paste0(variable,'=',as.character(value))}) +
    labs(y="Adaptation time", x=expression(tau)) 
print(p)