This notebook exemplifies how to reproduce Figure 1 of the article. The annotations from all screens will be downloaded and parsed to build statistics on phenotypes, which will be displayed using Bokeh.
The cell below will install dependencies if you choose to run the notebook in Google Colab.
%pip install idr-py
from IPython import get_ipython
import omero
from idr import connection
import numpy as np
from pandas import DataFrame
from pandas import read_csv
from bokeh.models import ColumnDataSource
from bokeh.plotting import figure
from bokeh.plotting import output_notebook
from bokeh.plotting import show
from bokeh.models import HoverTool
import bokeh.palettes as bpal
output_notebook()
get_ipython().run_line_magic('matplotlib', 'inline')
def getBulkAnnotationAsDf(screenID, conn):
ofId = None
sc = conn.getObject('Screen', screenID)
for ann in sc.listAnnotations():
if isinstance(ann, omero.gateway.FileAnnotationWrapper):
if (ann.getFile().getName() == 'bulk_annotations'):
if (ann.getFile().getSize() > 1476250900):
print("that's a big file...")
return None
ofId = ann.getFile().getId()
break
if ofId is None:
return None
original_file = omero.model.OriginalFileI(ofId, False)
table = conn.c.sf.sharedResources().openTable(original_file)
try:
rowCount = table.getNumberOfRows()
column_names = [col.name for col in table.getHeaders()]
black_list = []
column_indices = []
for column_name in column_names:
if column_name in black_list:
continue
column_indices.append(column_names.index(column_name))
table_data = table.slice(column_indices, None)
finally:
table.close()
data = []
for index in range(rowCount):
row_values = [column.values[index] for column in table_data.columns]
data.append(row_values)
dfAnn = DataFrame(data)
dfAnn.columns = column_names
return dfAnn
def appendPhInfo(phall, screen, df):
"""
extract all phenotypes information from given bulk annotation file and
append it to the phall
"""
phcol = df.columns[[('Phenotype' in s) and ('Term Accession' in s)
for s in df.columns]]
for s in phcol:
ph = df[s].unique()
if ph[0] != '':
ph = ph[0]
desc = df[s.replace('Accession', 'Name')].unique()[0]
else:
ph = ph[1]
desc = df[s.replace('Accession', 'Name')].unique()[1]
dfph = df[df[s] != '']
try:
phall[ph]['n'] = phall[ph]['n']+len(dfph)
if not (screen in phall[ph]['screens']):
phall[ph]['screens'].append(screen)
except Exception:
phcur = {'n': len(dfph), 'screens': [screen], 'desc': desc}
phall[ph] = phcur
conn = connection('idr.openmicroscopy.org')
Connected to IDR...
screens = list(conn.getObjects("Screen"))
screen_count = len(screens)
print(screen_count)
phall = {}
# List of screens used for paper
screen_ids = [3, 102, 51, 202, 597, 253, 201, 154, 751, 206,
251, 803, 1351, 1501, 1551, 1601, 1602, 1603, 1202, 1101, 1302,
1201, 1251, 1151, 1203, 1204, 1651, 1652, 1653, 1654]
print("Iterating through screens...")
for sc in screens:
sc_id = sc.getId()
print('loading ' + str(sc_id))
if sc_id in screen_ids:
df = getBulkAnnotationAsDf(sc_id, conn)
if df is not None:
appendPhInfo(phall, sc.getName(), df)
37 [============================================================] 100.0% ...Iterating through screens
conn.close()
dfColor = read_csv('https://raw.githubusercontent.com/IDR/idr-notebooks/master/includes/CMPOAccessionToPhenotypeCategories.csv')
colors = {}
for i, grp in enumerate(dfColor.CmpoPhenotypeCategory.unique()):
colors[grp] = bpal.Set3_10[i % 10]
# add the information to the data and sort it
for ph in phall.keys():
try:
v = dfColor['CmpoAcc'] == ph
phall[ph]['group'] = dfColor[v]['CmpoPhenotypeCategory'].values[0]
phall[ph]['groupColor'] = colors[phall[ph]['group']]
phall[ph]['FigureCmpoName'] = dfColor[v]['FigureCmpoName'].values[0]
except Exception:
print('pass:'+ph)
del phall[ph]
phalls = sorted(phall.values(), key=lambda x: x['group'])
pass:CMPO_0000458 pass:CMPO_0000390 pass:CMPO_0000018 pass:CMPO_0000446 pass:CMPO_0000447 pass:CMPO_0000444 pass:CMPO_0000445 pass:CMPO_0000442 pass:CMPO_0000443 pass:CMPO_0000440 pass:CMPO_0000441 pass:CMPO_0000450 pass:CMPO_0000453 pass:CMPO_0000454 pass:CMPO_0000457 pass:CMPO_0000113 pass:CMPO_0000422 pass:CMPO_0000023
TOOLS = "pan,wheel_zoom,reset"
phenotypes = figure(title="Fig 1",
tools=TOOLS,
y_axis_type="log",
width=900,
toolbar_location="above")
source = ColumnDataSource(
data=dict(
ph=[ph['FigureCmpoName'] for ph in phalls],
n=[ph['n'] for ph in phalls],
names=[ph['screens'] for ph in phalls],
desc=[ph['desc'] for ph in phalls],
x=[2*x for x in range(len(phall.keys()))],
r=[1*len(ph['screens']) for ph in phalls],
color=[ph['groupColor'] for ph in phalls],
groupName=[ph['group'] for ph in phalls]
))
label_data = {2*i: x for i, x in
enumerate([ph['FigureCmpoName'] for ph in phalls])}
cir = phenotypes.circle('x', 'n', radius='r', source=source, color='color')
hover = HoverTool(
tooltips=[
("Term", "@ph"),
("Description", "@desc"),
("Number of samples", "@n"),
("Screens name", "@names"),
("group", "@groupName")
]
)
phenotypes.add_tools(hover)
phenotypes.xaxis.major_label_orientation = np.pi/4.
phenotypes.xaxis.axis_label_text_font_size = "10pt"
show(phenotypes)
Copyright (C) 2016-2021 University of Dundee. All Rights Reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.