%matplotlib inline
import os
from pprint import pprint
import matplotlib.pyplot as plt
import sciunit
import sciunit.scores
import neuronunit
from neuronunit.models.reduced import ReducedModel
from neuronunit.capabilities import ProducesSpikes
# Assumes imported neuronunit is from source, e.g. pip install -e
path = os.path.join(neuronunit.__path__[0],'models/NeuroML2/LEMS_2007One.xml')
# Instantiate three identical models
models = [ReducedModel(path, name='Izhikevich', backend='jNeuroML') for i in range(2)]
# Change the parameters of the second model
models[1].set_attrs(izhikevich2007Cell={'a': '0.04 per_ms'})
models[1].name = 'Izhikevich_new'
for model in models:
plt.plot(model.get_membrane_potential(),label=model)
plt.legend();
pyNeuroML >>> Reloading data specified in LEMS file: /var/folders/_j/vg2m860n23d_9ty1h2z9_2880000gn/T/tmpw4nxbjl4/Izhikevich.xml (/private/var/folders/_j/vg2m860n23d_9ty1h2z9_2880000gn/T/tmpw4nxbjl4/Izhikevich.xml), base_dir: /var/folders/_j/vg2m860n23d_9ty1h2z9_2880000gn/T/tmp1hr8cang, cwd: /Users/rgerkin/Dropbox/dev/scidash/neuronunit/neuronunit/unit_test pyNeuroML >>> Reloading data specified in LEMS file: /var/folders/_j/vg2m860n23d_9ty1h2z9_2880000gn/T/tmpddw73r6n/Izhikevich.xml (/private/var/folders/_j/vg2m860n23d_9ty1h2z9_2880000gn/T/tmpddw73r6n/Izhikevich.xml), base_dir: /var/folders/_j/vg2m860n23d_9ty1h2z9_2880000gn/T/tmp8bwyo0o8, cwd: /Users/rgerkin/Dropbox/dev/scidash/neuronunit/neuronunit/unit_test
# A new M2M test which will compare the equality of spike counts across models
class MyTest(sciunit.Test):
required_capabilities = (ProducesSpikes,)
score_type = sciunit.scores.ZScore
def generate_prediction(self,model):
count = model.get_spike_count()
return count
test = MyTest(observation={'mean':8, 'std':3, 'url':'http://somewhere.thathasdata.com'})
s = test.judge(models)
# The score matrix
s
MyTest | |
---|---|
Izhikevich | Z = -0.33 |
Izhikevich_new | Z = 0.00 |
x = test.json(string=False)
pprint(x)
{'description': None, 'name': 'MyTest', 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'unpicklable': [], 'verbose': 1}
def check_url(x):
if not x.json(add_props=True,string=False).get('url'):
print("Model has no associated url; please set the url attribute of the model")
check_url(models[0])
# How do you want your GitHub URLs?
lems_url = ("https://github.com/scidash/neuronunit/blob/master/neuronunit/"
"models/NeuroML2/LEMS_2007One.xml")
models[0]._url = lems_url
models[1]._url = lems_url
models[0].url
'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'
check_url(models[0])
# Has fewer attributes because most of them came from the LEMS file so those are assumed
# to still be the values in memory. If we need to extract these then I need to find a
# way to read them from the file; however, there may be no general way to do so for all
# LEMS files, and certainly not for all possible models.
x = models[0].json(string=False)
pprint(x)
{'attrs': {}, 'backend': 'jNeuroML', 'name': 'Izhikevich', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}}
# Has more attributes because some of the original attributes from the LEMS file
# were replaced with new ones in this session
x = models[1].json(string=False)
pprint(x)
{'attrs': {'izhikevich2007Cell': {'a': '0.04 per_ms'}}, 'backend': 'jNeuroML', 'name': 'Izhikevich_new', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}}
# Select one score from the score matrix
score = s['Izhikevich','MyTest']
x = score.json(string=False)
pprint(x)
{'model': {'attrs': {}, 'backend': 'jNeuroML', 'name': 'Izhikevich', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}}, 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'prediction': 7, 'related_data': {}, 'score': -0.3333333333333333, 'test': {'description': None, 'name': 'MyTest', 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'unpicklable': [], 'verbose': 1}, 'unpicklable': []}
# Add additional properties to the JSON output
x = score.json(add_props=True,string=False)
pprint(x)
{'_class': {'name': 'ZScore', 'url': 'http://github.com/scidash/sciunit'}, '_id': 4574572840, 'hash': '370f8d0e9b6a87ae568560b0cedb4cfc567e0e097ff2dce7fe123281', 'model': {'_class': {'name': 'ReducedModel', 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'}, '_id': 4564190320, 'attrs': {}, 'backend': 'jNeuroML', 'capabilities': ['Runnable', 'ReceivesSquareCurrent', 'ProducesActionPotentials', 'ProducesSpikes', 'ProducesMembranePotential'], 'hash': '04433628b02110a59660c84bc268c68fab32cd033f4da5edaf7a18ac', 'name': 'Izhikevich', 'remote_url': 'https://github.com/scidash/neuronunit', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}, 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml', 'version': '52528e5f1d49c5e607a9e567a493b7a634658200*'}, 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'prediction': 7, 'raw': '-0.3333', 'related_data': {}, 'remote_url': 'http://github.com/scidash/sciunit', 'score': -0.3333333333333333, 'score_type': 'ZScore', 'norm_score': 0.7388826803635273, 'summary': "=== Model Izhikevich achieved score Z = -0.33 on test 'MyTest'. " '===', 'test': {'_class': {'name': 'MyTest', 'url': ''}, '_id': 4574574576, 'description': None, 'hash': 'a525227bebc09871767c6e3bf0414556820727f7095437f2c58356ab', 'name': 'MyTest', 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}, 'unpicklable': [], 'url': 'http://github.com/scidash/sciunit', 'version': 'eb5469103f510c12b4897180db7551ccbae7cc0f*'}
# Nicer formatting for the output
x = score.json(add_props=True,string=False)
pprint(x)
{'_class': {'name': 'ZScore', 'url': 'http://github.com/scidash/sciunit'}, '_id': 4574572840, 'hash': '370f8d0e9b6a87ae568560b0cedb4cfc567e0e097ff2dce7fe123281', 'model': {'_class': {'name': 'ReducedModel', 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'}, '_id': 4564190320, 'attrs': {}, 'backend': 'jNeuroML', 'capabilities': ['Runnable', 'ReceivesSquareCurrent', 'ProducesActionPotentials', 'ProducesSpikes', 'ProducesMembranePotential'], 'hash': '04433628b02110a59660c84bc268c68fab32cd033f4da5edaf7a18ac', 'name': 'Izhikevich', 'remote_url': 'https://github.com/scidash/neuronunit', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}, 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml', 'version': '52528e5f1d49c5e607a9e567a493b7a634658200*'}, 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'prediction': 7, 'raw': '-0.3333', 'related_data': {}, 'remote_url': 'http://github.com/scidash/sciunit', 'score': -0.3333333333333333, 'score_type': 'ZScore', 'norm_score': 0.7388826803635273, 'summary': "=== Model Izhikevich achieved score Z = -0.33 on test 'MyTest'. " '===', 'test': {'_class': {'name': 'MyTest', 'url': ''}, '_id': 4574574576, 'description': None, 'hash': 'a525227bebc09871767c6e3bf0414556820727f7095437f2c58356ab', 'name': 'MyTest', 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}, 'unpicklable': [], 'url': 'http://github.com/scidash/sciunit', 'version': 'eb5469103f510c12b4897180db7551ccbae7cc0f*'}
test2 = MyTest(observation={'mean':7, 'std':3.8, 'url':'http://somewhereelse.thathasdata.com'},name='Second Test')
suite = sciunit.TestSuite([test,test2],name='My Suite')
x = suite.json(add_props=True,string=False)
pprint(x)
{'_class': {'name': 'TestSuite', 'url': 'http://github.com/scidash/sciunit'}, '_id': 4646233592, 'hash': '2a162712298f803f30e097fef52c44f42104feede9500e6a76299098', 'hooks': None, 'include_models': [], 'name': 'My Suite', 'remote_url': 'http://github.com/scidash/sciunit', 'skip_models': [], 'tests': [{'_class': {'name': 'MyTest', 'url': ''}, '_id': 4574574576, 'description': None, 'hash': 'a525227bebc09871767c6e3bf0414556820727f7095437f2c58356ab', 'name': 'MyTest', 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}, {'_class': {'name': 'MyTest', 'url': ''}, '_id': 4646233984, 'description': None, 'hash': '1a7b9374f4ee0c3e01680316dacf2ecaed63cb27af2f1e3973d9ad5a', 'name': 'Second Test', 'observation': {'mean': 7, 'std': 3.8, 'url': 'http://somewhereelse.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}], 'unpicklable': [], 'url': 'http://github.com/scidash/sciunit', 'version': 'eb5469103f510c12b4897180db7551ccbae7cc0f*', 'weights': [0.5, 0.5], 'weights_': []}
score_matrix = suite.judge(models)
x = score_matrix.json(add_props=True,string=False)
pprint(x)
{'models': [{'_class': {'name': 'ReducedModel', 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'}, '_id': 4564190320, 'attrs': {}, 'backend': 'jNeuroML', 'capabilities': ['Runnable', 'ReceivesSquareCurrent', 'ProducesActionPotentials', 'ProducesSpikes', 'ProducesMembranePotential'], 'hash': '04433628b02110a59660c84bc268c68fab32cd033f4da5edaf7a18ac', 'name': 'Izhikevich', 'remote_url': 'https://github.com/scidash/neuronunit', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}, 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml', 'version': '52528e5f1d49c5e607a9e567a493b7a634658200*'}, {'_class': {'name': 'ReducedModel', 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'}, '_id': 4301722344, 'attrs': {'izhikevich2007Cell': {'a': '0.04 per_ms'}}, 'backend': 'jNeuroML', 'capabilities': ['Runnable', 'ReceivesSquareCurrent', 'ProducesActionPotentials', 'ProducesSpikes', 'ProducesMembranePotential'], 'hash': '640c67dede0892f857f37332a7dbb9e4fd6bb6f21a48f7b22c059fab', 'name': 'Izhikevich_new', 'remote_url': 'https://github.com/scidash/neuronunit', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}, 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml', 'version': '52528e5f1d49c5e607a9e567a493b7a634658200*'}], 'scores': [[{'_class': {'name': 'ZScore', 'url': 'http://github.com/scidash/sciunit'}, '_id': 4410766840, 'hash': '370f8d0e9b6a87ae568560b0cedb4cfc567e0e097ff2dce7fe123281', 'model': {'_class': {'name': 'ReducedModel', 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'}, '_id': 4564190320, 'attrs': {}, 'backend': 'jNeuroML', 'capabilities': ['Runnable', 'ReceivesSquareCurrent', 'ProducesActionPotentials', 'ProducesSpikes', 'ProducesMembranePotential'], 'hash': '04433628b02110a59660c84bc268c68fab32cd033f4da5edaf7a18ac', 'name': 'Izhikevich', 'remote_url': 'https://github.com/scidash/neuronunit', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}, 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml', 'version': '52528e5f1d49c5e607a9e567a493b7a634658200*'}, 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'prediction': 7, 'raw': '-0.3333', 'related_data': {}, 'remote_url': 'http://github.com/scidash/sciunit', 'score': -0.3333333333333333, 'score_type': 'ZScore', 'norm_score': 0.7388826803635273, 'summary': '=== Model Izhikevich achieved score Z = -0.33 on ' "test 'MyTest'. ===", 'test': {'_class': {'name': 'MyTest', 'url': ''}, '_id': 4574574576, 'description': None, 'hash': 'a525227bebc09871767c6e3bf0414556820727f7095437f2c58356ab', 'name': 'MyTest', 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}, 'unpicklable': [], 'url': 'http://github.com/scidash/sciunit', 'version': 'eb5469103f510c12b4897180db7551ccbae7cc0f*'}, {'_class': {'name': 'ZScore', 'url': 'http://github.com/scidash/sciunit'}, '_id': 4646232304, 'hash': '66ea475bc3bc615f2629b2f42f4d4266f65b1ea572a2a303b1836b1a', 'model': {'_class': {'name': 'ReducedModel', 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'}, '_id': 4564190320, 'attrs': {}, 'backend': 'jNeuroML', 'capabilities': ['Runnable', 'ReceivesSquareCurrent', 'ProducesActionPotentials', 'ProducesSpikes', 'ProducesMembranePotential'], 'hash': '04433628b02110a59660c84bc268c68fab32cd033f4da5edaf7a18ac', 'name': 'Izhikevich', 'remote_url': 'https://github.com/scidash/neuronunit', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}, 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml', 'version': '52528e5f1d49c5e607a9e567a493b7a634658200*'}, 'observation': {'mean': 7, 'std': 3.8, 'url': 'http://somewhereelse.thathasdata.com'}, 'prediction': 7, 'raw': '0', 'related_data': {}, 'remote_url': 'http://github.com/scidash/sciunit', 'score': 0.0, 'score_type': 'ZScore', 'norm_score': 1.0, 'summary': '=== Model Izhikevich achieved score Z = 0.00 on test ' "'Second Test'. ===", 'test': {'_class': {'name': 'MyTest', 'url': ''}, '_id': 4646233984, 'description': None, 'hash': '1a7b9374f4ee0c3e01680316dacf2ecaed63cb27af2f1e3973d9ad5a', 'name': 'Second Test', 'observation': {'mean': 7, 'std': 3.8, 'url': 'http://somewhereelse.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}, 'unpicklable': [], 'url': 'http://github.com/scidash/sciunit', 'version': 'eb5469103f510c12b4897180db7551ccbae7cc0f*'}], [{'_class': {'name': 'ZScore', 'url': 'http://github.com/scidash/sciunit'}, '_id': 4646232584, 'hash': '9baf52a5280631a2372af96162c57a447582445fc293d899f6791855', 'model': {'_class': {'name': 'ReducedModel', 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'}, '_id': 4301722344, 'attrs': {'izhikevich2007Cell': {'a': '0.04 per_ms'}}, 'backend': 'jNeuroML', 'capabilities': ['Runnable', 'ReceivesSquareCurrent', 'ProducesActionPotentials', 'ProducesSpikes', 'ProducesMembranePotential'], 'hash': '640c67dede0892f857f37332a7dbb9e4fd6bb6f21a48f7b22c059fab', 'name': 'Izhikevich_new', 'remote_url': 'https://github.com/scidash/neuronunit', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}, 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml', 'version': '52528e5f1d49c5e607a9e567a493b7a634658200*'}, 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'prediction': 8, 'raw': '0', 'related_data': {}, 'remote_url': 'http://github.com/scidash/sciunit', 'score': 0.0, 'score_type': 'ZScore', 'norm_score': 1.0, 'summary': '=== Model Izhikevich_new achieved score Z = 0.00 on ' "test 'MyTest'. ===", 'test': {'_class': {'name': 'MyTest', 'url': ''}, '_id': 4574574576, 'description': None, 'hash': 'a525227bebc09871767c6e3bf0414556820727f7095437f2c58356ab', 'name': 'MyTest', 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}, 'unpicklable': [], 'url': 'http://github.com/scidash/sciunit', 'version': 'eb5469103f510c12b4897180db7551ccbae7cc0f*'}, {'_class': {'name': 'ZScore', 'url': 'http://github.com/scidash/sciunit'}, '_id': 4646234152, 'hash': '39695c4093b1fe19fd7a13770c0823ddfeee9c2a62f12454b894003b', 'model': {'_class': {'name': 'ReducedModel', 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml'}, '_id': 4301722344, 'attrs': {'izhikevich2007Cell': {'a': '0.04 per_ms'}}, 'backend': 'jNeuroML', 'capabilities': ['Runnable', 'ReceivesSquareCurrent', 'ProducesActionPotentials', 'ProducesSpikes', 'ProducesMembranePotential'], 'hash': '640c67dede0892f857f37332a7dbb9e4fd6bb6f21a48f7b22c059fab', 'name': 'Izhikevich_new', 'remote_url': 'https://github.com/scidash/neuronunit', 'run_params': {'default_java_max_memory': '400M', 'nogui': True, 'v': False}, 'url': 'https://github.com/scidash/neuronunit/blob/master/neuronunit/models/NeuroML2/LEMS_2007One.xml', 'version': '52528e5f1d49c5e607a9e567a493b7a634658200*'}, 'observation': {'mean': 7, 'std': 3.8, 'url': 'http://somewhereelse.thathasdata.com'}, 'prediction': 8, 'raw': '0.2632', 'related_data': {}, 'remote_url': 'http://github.com/scidash/sciunit', 'score': 0.2631578947368421, 'score_type': 'ZScore', 'norm_score': 0.7924288824046, 'summary': '=== Model Izhikevich_new achieved score Z = 0.26 on ' "test 'Second Test'. ===", 'test': {'_class': {'name': 'MyTest', 'url': ''}, '_id': 4646233984, 'description': None, 'hash': '1a7b9374f4ee0c3e01680316dacf2ecaed63cb27af2f1e3973d9ad5a', 'name': 'Second Test', 'observation': {'mean': 7, 'std': 3.8, 'url': 'http://somewhereelse.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}, 'unpicklable': [], 'url': 'http://github.com/scidash/sciunit', 'version': 'eb5469103f510c12b4897180db7551ccbae7cc0f*'}]], 'tests': [{'_class': {'name': 'MyTest', 'url': ''}, '_id': 4574574576, 'description': None, 'hash': 'a525227bebc09871767c6e3bf0414556820727f7095437f2c58356ab', 'name': 'MyTest', 'observation': {'mean': 8, 'std': 3, 'url': 'http://somewhere.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}, {'_class': {'name': 'MyTest', 'url': ''}, '_id': 4646233984, 'description': None, 'hash': '1a7b9374f4ee0c3e01680316dacf2ecaed63cb27af2f1e3973d9ad5a', 'name': 'Second Test', 'observation': {'mean': 7, 'std': 3.8, 'url': 'http://somewhereelse.thathasdata.com'}, 'remote_url': None, 'unpicklable': [], 'url': None, 'verbose': 1, 'version': None}]}