This notebook exemplifies how to reproduce Figure 1 of the article. The annotations from all screens will be downloaded and parsed to build statistics on phenotypes, which will be displayed using Bokeh.
import omero
from idr import connection
import numpy as np
import matplotlib.pyplot as plt
from pandas import DataFrame
from pandas import concat
from pandas import read_csv
from bokeh.models import ColumnDataSource
from bokeh.plotting import figure
from bokeh.plotting import output_notebook
from bokeh.plotting import show
from bokeh.plotting import save
from bokeh.models.formatters import TickFormatter
from bokeh.models.formatters import String
from bokeh.models.formatters import List
from bokeh.models.formatters import Dict
from bokeh.models.formatters import Int
from bokeh.models import FixedTicker
from bokeh.models import HoverTool
import bokeh.palettes as bpal
output_notebook()
%matplotlib inline
def getBulkAnnotationAsDf(screenID, conn):
ofId = None
sc = conn.getObject('Screen', screenID)
for ann in sc.listAnnotations():
if isinstance(ann, omero.gateway.FileAnnotationWrapper):
if (ann.getFile().getName() == 'bulk_annotations'):
if (ann.getFile().getSize() > 1476250900):
print "that's a big file..."
return None
ofId = ann.getFile().getId()
break
if ofId is None:
return None
original_file = omero.model.OriginalFileI(ofId, False)
table = conn.c.sf.sharedResources().openTable(original_file)
try:
rowCount = table.getNumberOfRows()
column_names = [col.name for col in table.getHeaders()]
black_list = []
column_indices = []
for column_name in column_names:
if column_name in black_list:
continue
column_indices.append(column_names.index(column_name))
table_data = table.slice(column_indices, None)
finally:
table.close()
data = []
for index in range(rowCount):
row_values = [column.values[index] for column in table_data.columns]
data.append(row_values)
dfAnn = DataFrame(data)
dfAnn.columns = column_names
return dfAnn
def appendPhInfo(phall,screen,df):
"""
extract all phenotypes information from given bulk annotation file and
append it to the phall
"""
phcol = df.columns[[('Phenotype' in s) and ('Term Accession' in s) for s in df.columns ]]
for s in phcol:
ph = df[s].unique()
if ph[0]!='':
ph = ph[0]
desc = df[s.replace('Accession', 'Name')].unique()[0]
else:
ph = ph[1]
desc = df[s.replace('Accession', 'Name')].unique()[1]
dfph = df[df[s]!='']
try:
phall[ph]['n'] = phall[ph]['n']+len(dfph)
if not (screen in phall[ph]['screens']):
phall[ph]['screens'].append(screen)
except Exception as e:
# print(e)
phcur = {'n':len(dfph),'screens':[screen],'desc':desc}
phall[ph] = phcur
conn = connection()
Connected to IDR...
screens = list(conn.getObjects("Screen"))
screen_count = len(screens)
print screen_count
phall={}
# List of screens used for paper
screen_ids = [3, 102, 51, 202, 597, 253, 201, 154, 751, 206, 251, 803, 1351, 1501, 1551, 1601, 1602, 1603, 1202, 1101, 1302, 1201, 1251, 1151, 1203, 1204, 1651, 1652, 1653, 1654]
print "Iterating through screens..."
for sc in screens:
sc_id = sc.getId()
print 'loading ' + str(sc_id)
if sc_id in screen_ids:
df = getBulkAnnotationAsDf(sc_id, conn)
if df is not None:
appendPhInfo(phall, sc.getName(), df)
37 [============================================================] 100.0% ...Iterating through screens
conn.close()
dfColor = read_csv('./includes/CMPOAccessionToPhenotypeCategories.csv')
colors = {}
for i, grp in enumerate(dfColor.CmpoPhenotypeCategory.unique()):
colors[grp] = bpal.Set3_10[i%10]
#add the information to the data and sort it
for ph in phall.keys():
try:
phall[ph]['group'] = dfColor[dfColor['CmpoAcc']==ph]['CmpoPhenotypeCategory'].values[0]
phall[ph]['groupColor'] = colors[phall[ph]['group']]
phall[ph]['FigureCmpoName'] = dfColor[dfColor['CmpoAcc']==ph]['FigureCmpoName'].values[0]
except:
print 'pass:'+ph
del phall[ph]
phalls = sorted(phall.values(), key=lambda x: x['group'])
pass:CMPO_0000458 pass:CMPO_0000390 pass:CMPO_0000018 pass:CMPO_0000446 pass:CMPO_0000447 pass:CMPO_0000444 pass:CMPO_0000445 pass:CMPO_0000442 pass:CMPO_0000443 pass:CMPO_0000440 pass:CMPO_0000441 pass:CMPO_0000450 pass:CMPO_0000453 pass:CMPO_0000454 pass:CMPO_0000457 pass:CMPO_0000113 pass:CMPO_0000422 pass:CMPO_0000023
TOOLS = "pan,wheel_zoom,reset"
phenotypes = figure(title = "Fig 1",
tools=TOOLS,
y_axis_type="log",
width=900,
toolbar_location="above")
source = ColumnDataSource(
data=dict(
ph=[ph['FigureCmpoName'] for ph in phalls],
n=[ph['n'] for ph in phalls],
names=[ph['screens'] for ph in phalls],
desc=[ph['desc'] for ph in phalls],
x=[2*x for x in range(len(phall.keys()))],
r=[1*len(ph['screens']) for ph in phalls],
color=[ph['groupColor'] for ph in phalls],
groupName=[ph['group'] for ph in phalls]
))
label_data = {2*i:x for i,x in enumerate([ph['FigureCmpoName'] for ph in phalls])}
cir = phenotypes.circle('x', 'n', radius='r', source=source, color='color')
hover = HoverTool(
tooltips=[
("Term", "@ph"),
("Description", "@desc"),
("Number of samples", "@n"),
("Screens name", "@names"),
("group", "@groupName")
]
)
phenotypes.add_tools(hover)
phenotypes.xaxis.major_label_orientation = np.pi/4.
phenotypes.xaxis.axis_label_text_font_size = "10pt"
show(phenotypes)
Copyright (C) 2016 University of Dundee. All Rights Reserved.
This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.