Terry Stewart
import nef
net = nef.Network('Creature')
net.make_input('command_input', [0,0])
net.make('command', neurons=100, dimensions=2)
net.make('motor', neurons=100, dimensions=2)
net.make('position', neurons=1000, dimensions=2, radius=5)
net.make('scared_direction', neurons=100, dimensions=2)
def negative(x):
return -x[0], -x[1]
net.connect('position', 'scared_direction', func=negative)
net.connect('position', 'position')
net.make('plan', neurons=500, dimensions=5)
net.connect('command', 'plan', index_post=[0,1])
net.connect('scared_direction', 'plan', index_post=[2,3])
net.make('scared', neurons=50, dimensions=1)
net.make_input('scared_input', [0])
net.connect('scared_input', 'scared')
net.connect('scared', 'plan', index_post=[4])
def plan_function(x):
c_x, c_y, s_x, s_y, s = x
return s*(s_x)+(1-s)*c_x, s*(s_y)+(1-s)*c_y
net.connect('plan', 'motor', func=plan_function)
def rescale(x):
return x[0]*0.1, x[1]*0.1
net.connect('motor', 'position', func=rescale)
net.connect('command_input', 'command')
net.add_to_nengo()
This sort of system comes up a lot in cognitive models
How can we do this?
In the above example, we did it like this:
This required a 5-dimensional ensemble
What about more complex situations?
One group of neurons to represent the state $s$
One group of neurons for each action's utility $Q(s, a_i)$
What should the output be?
The second option seems easier if we consider that we have to do action execution next...
import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make('Q_A', neurons=50, dimensions=1)
net.make('Q_B', neurons=50, dimensions=1)
net.make('Q_C', neurons=50, dimensions=1)
net.make('Q_D', neurons=50, dimensions=1)
net.connect('s', 'Q_A', transform=[1,0])
net.connect('s', 'Q_B', transform=[-1,0])
net.connect('s', 'Q_C', transform=[0,1])
net.connect('s', 'Q_D', transform=[0,-1])
net.make_input('input', [0,0])
net.connect('input', 's')
net.add_to_nengo()
net.view()
array
capability to help with thisimport nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
net.make_input('input', [0,0])
net.connect('input', 's')
net.add_to_nengo()
net.view()
import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.make('Qall', neurons=200, dimensions=4)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
def maximum(x):
max_x = max(x)
result = [0,0,0,0]
result[x.index(max_x)] = 1
return result
net.make('Action', neurons=200, dimensions=4)
net.connect('Q', 'Qall')
net.connect('Qall', 'Action', func=maximum)
net.make_input('input', [0,0])
net.connect('input', 's')
net.add_to_nengo()
net.view()
import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
net.make_input('input', [0,0])
net.connect('input', 's')
e = 0.1
i = -1
transform = [[e, i, i, i], [i, e, i, i], [i, i, e, i], [i, i, i, e]]
net.connect('Q', 'Q', transform=transform)
net.add_to_nengo()
net.view()
import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
net.make_array('Action', neurons=50, length=4, dimensions=1)
net.connect('Q', 'Action')
net.make_input('input', [0,0])
net.connect('input', 's')
e = 0.5
i = -1
transform = [[e, i, i, i], [i, e, i, i], [i, i, e, i], [i, i, i, e]]
# Let's force the feedback connection to only consider positive values
def positive(x):
if x[0]<0: return [0]
else: return x
net.connect('Action', 'Action', func=positive, transform=transform)
net.add_to_nengo()
net.view()
e
?import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
net.make_array('Action', neurons=50, length=4, dimensions=1)
net.connect('Q', 'Action')
net.make_input('input', [0,0])
net.connect('input', 's')
e = 1
i = -1
transform = [[e, i, i, i], [i, e, i, i], [i, i, e, i], [i, i, i, e]]
def positive(x):
if x[0]<0: return [0]
else: return x
net.connect('Action', 'Action', func=positive, transform=transform)
net.add_to_nengo()
net.view()
e
too much?import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
net.make_array('Action', neurons=50, length=4, dimensions=1)
net.connect('Q', 'Action')
net.make_input('input', [0,0])
net.connect('input', 's')
e = 0.5
i = -1
transform = [[e, i, i, i], [i, e, i, i], [i, i, e, i], [i, i, i, e]]
def positive(x):
if x[0]<0: return [0]
else: return x
net.connect('Action', 'Action', func=positive, transform=transform)
# Apply this function on the output
def select(x):
if x[0]<=0: return [0]
else: return [1]
net.make_array('ActionValue', neurons=50, length=4, dimensions=1)
net.connect('Action', 'ActionValue', func=select)
net.add_to_nengo()
net.view()
e
import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
net.make_array('Action', neurons=50, length=4, dimensions=1)
net.connect('Q', 'Action')
net.make_input('input', [0,0])
net.connect('input', 's')
e = 0.2
i = -1
transform = [[e, i, i, i], [i, e, i, i], [i, i, e, i], [i, i, i, e]]
def positive(x):
if x[0]<0: return [0]
else: return x
net.connect('Action', 'Action', func=positive, transform=transform)
# Apply this function on the output
def select(x):
if x[0]<=0: return [0]
else: return [1]
net.make_array('ActionValue', neurons=50, length=4, dimensions=1)
net.connect('Action', 'ActionValue', func=select)
net.add_to_nengo()
net.view()
Much less memory, but it's still there
And much slower to respond to changes
Note that this speed is dependent on $e$, $i$, and the time constant of the neurotransmitter used
Can be hard to find good values
And this gets harder to balance as the number of actions increases
But this is still a pretty standard approach
They tend to use a "kWTA" (k-Winners Take All) approach in their models
Any other options?
Old terminology:
Then they found:
Activity in the GPi (output)
Leabra approach
Seems to match onto the biology okay
Maybe the weird structure of the basal ganglia is an attempt to do action selection without doing mutual inhibition
Needs to select from a large number of actions
Needs to do so quickly, and without the memory effects
Let's start with a very simple version
Sort of like an "unrolled" version of one step of mutual inhibition
Now let's map that onto the basal ganglia
This turns out to work surprisingly well
But extremely hard to analyze its behaviour
They showed that it qualitatively matches pretty well
So what happens if we convert this into realistic spiking neurons?
Use the same approach where one "neuron" in their model is a pool of neurons in the NEF
The "neuron model" they use was rectified linear
Neurotransmitter time constant are all known
$Q$ values are between 0 and 1
Firing rates max out around 50-100Hz
Encoders are all positive and thresholds are chosen for efficiency
mm=1
mp=1
me=1
mg=1
ws=1
wt=1
wm=1
wg=1
wp=0.9
we=0.3
e=0.2
ep=-0.25
ee=-0.2
eg=-0.2
le=0.2
lg=0.2
D = 5
tau_ampa=0.002
tau_gaba=0.008
N = 50
radius = 1.5
import nef
net = nef.Network('Basal Ganglia')
net.make_input('input', [0]*D)
net.make_array('StrD1',N, D,intercept=(e,1),encoders=[[1]],radius=radius)
net.make_array('StrD2',N, D,intercept=(e,1),encoders=[[1]],radius=radius)
net.make_array('STN',N, D,intercept=(ep,1),encoders=[[1]],radius=radius)
net.make_array('GPi',N, D,intercept=(eg,1),encoders=[[1]],radius=radius)
net.make_array('GPe',N, D,intercept=(ee,1),encoders=[[1]],radius=radius)
net.connect('input','StrD1',weight=ws*(1+lg),pstc=tau_ampa)
net.connect('input','StrD2',weight=ws*(1-le),pstc=tau_ampa)
net.connect('input','STN',weight=wt,pstc=tau_ampa)
def func_str(x):
if x[0]<e: return 0
return mm*(x[0]-e)
net.connect('StrD1','GPi',func=func_str,weight=-wm,pstc=tau_gaba)
net.connect('StrD2','GPe',func=func_str,weight=-wm,pstc=tau_gaba)
def func_stn(x):
if x[0]<ep: return 0
return mp*(x[0]-ep)
tr=[[wp]*D for i in range(D)]
net.connect('STN','GPi',func=func_stn,transform=tr,pstc=tau_ampa)
net.connect('STN','GPe',func=func_stn,transform=tr,pstc=tau_ampa)
def func_gpe(x):
if x[0]<ee: return 0
return me*(x[0]-ee)
net.connect('GPe','GPi',func=func_gpe,weight=-we,pstc=tau_gaba)
net.connect('GPe','STN',func=func_gpe,weight=-wg,pstc=tau_gaba)
net.make_array('Action',N, D,intercept=(0.2,1),encoders=[[1]])
net.make_input('bias', [1]*D)
net.connect('bias', 'Action')
import numeric as np
net.connect('Action', 'Action', (np.eye(D)-1), pstc=tau_gaba)
def func_gpi(x):
if x[0]<eg: return 0
return mg*(x[0]-eg)
net.connect('GPi','Action',func=func_gpi,pstc=tau_gaba,weight=-3)
net.add_to_nengo()
net.view()
Notice that we are also flipping the output from [1, 1, 0, 1] to [0, 0, 1, 0]
Works pretty well
Scales up to many actions
Selects quickly
Gets behavioural match to empirical data, including timing predictions (!)
Dynamic Behaviour of a Spiking Model of Action Selection in the Basal Ganglia
Let's make sure this works with our original system
import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
net.make_input('input', [0,0])
net.connect('input', 's')
D = 4
net.make_array('Action', neurons=50, length=D, dimensions=1, encoders=[[1]], intercept=(0.2,1))
net.make_input('bias', [1]*D)
net.connect('bias', 'Action')
import numeric as np
net.connect('Action', 'Action', (np.eye(D)-1), pstc=0.008)
import nps
nps.basalganglia.make_basal_ganglia(net, 'Q', 'Action', D, same_neurons=False, output_weight=-3)
net.add_to_nengo()
net.view()
import nef
net = nef.Network('Selection')
net.make('s', neurons=200, dimensions=2)
net.make_array('Q', neurons=50, length=4, dimensions=1)
net.connect('s', 'Q', transform=[[1,0],[-1,0],[0,1],[0,-1]])
net.make_input('input', [0,0])
net.connect('input', 's')
D = 4
net.make_array('Action', neurons=50, length=D, dimensions=1, encoders=[[1]], intercept=(0.2,1))
net.make_input('bias', [1]*D)
net.connect('bias', 'Action')
import numeric as np
net.connect('Action', 'Action', (np.eye(D)-1), pstc=0.008)
import nps
nps.basalganglia.make_basal_ganglia(net, 'Q', 'Action', D, same_neurons=False, output_weight=-3)
net.make('motor', neurons=100, dimensions=2)
net.connect('Action.0', 'motor', transform=[1,0])
net.connect('Action.1', 'motor', transform=[-1,0])
net.connect('Action.2', 'motor', transform=[0,1])
net.connect('Action.3', 'motor', transform=[0,-1])
net.add_to_nengo()
net.view()
import nef
net = nef.Network('Creature')
net.make_input('command_input', [0,0])
net.make('command', neurons=100, dimensions=2)
net.make('motor', neurons=100, dimensions=2)
net.make('position', neurons=1000, dimensions=2, radius=5)
net.make('scared_direction', neurons=100, dimensions=2)
def negative(x):
return -x[0], -x[1]
net.connect('position', 'scared_direction', func=negative)
net.connect('position', 'position')
def rescale(x):
return x[0]*0.1, x[1]*0.1
net.connect('motor', 'position', func=rescale)
net.connect('command_input', 'command')
D = 4
net.make_input('Q_input', [0]*D)
net.make_array('Q', neurons=50, length=D)
net.connect('Q_input', 'Q')
net.make_array('Action', neurons=50, length=D, dimensions=1, encoders=[[1]], intercept=(0.2,1))
net.make_input('bias', [1]*D)
net.connect('bias', 'Action')
import numeric as np
net.connect('Action', 'Action', (np.eye(D)-1), pstc=0.008)
import nps
nps.basalganglia.make_basal_ganglia(net, 'Q', 'Action', D, same_neurons=False, output_weight=-3)
net.make('do_command', 300, 3)
net.connect('command', 'do_command', index_post=[0,1])
net.connect('Action.0', 'do_command', index_post=[2])
def command(x):
return x[2]*x[0], x[2]*x[1]
net.connect('do_command', 'motor', func=command)
net.make('do_scared', 300, 3)
net.connect('scared_direction', 'do_scared', index_post=[0,1])
net.connect('Action.1', 'do_scared', index_post=[2])
def command(x):
return x[2]*x[0], x[2]*x[1]
net.connect('do_scared', 'motor', func=command)
net.add_to_nengo()
import nef
net = nef.Network('Creature')
net.make_input('command_input', [0,0])
net.make('command', neurons=100, dimensions=2)
net.make('motor', neurons=100, dimensions=2)
net.make('position', neurons=1000, dimensions=2, radius=5)
net.make('scared_direction', neurons=100, dimensions=2)
def negative(x):
return -x[0], -x[1]
net.connect('position', 'scared_direction', func=negative)
net.connect('position', 'position')
def rescale(x):
return x[0]*0.1, x[1]*0.1
net.connect('motor', 'position', func=rescale)
net.connect('command_input', 'command')
D = 4
net.make_input('Q_input', [0]*D)
net.make_array('Q', neurons=50, length=D)
net.connect('Q_input', 'Q')
net.make_array('Action', neurons=50, length=D, dimensions=1, encoders=[[1]], intercept=(0.2,1))
net.make_input('bias', [1]*D)
net.connect('bias', 'Action')
import numeric as np
net.connect('Action', 'Action', (np.eye(D)-1), pstc=0.008)
import nps
nps.basalganglia.make_basal_ganglia(net, 'Q', 'Action', D, same_neurons=False, output_weight=-3)
net.make('do_command', 200, 2)
net.connect('command', 'do_command')
net.connect('do_command', 'motor')
net.connect('GPi.0', 'do_command', encoders=-10)
net.make('do_scared', 300, 2)
net.connect('scared_direction', 'do_scared')
net.connect('do_scared', 'motor')
net.connect('GPi.1', 'do_scared', encoders=-10)
net.add_to_nengo()
We now have everything we need for a model of one of the primary structures in the mammalian brain
We build systems in cortex that give some input-output functionality
Example
from spa import *
D=16
class Rules: #Define the rules by specifying the start state and the
#desired next state
def A(state='A'): #e.g. If in state A
set(state='B') # then go to state B
def B(state='B'):
set(state='C')
def C(state='C'):
set(state='D')
def D(state='D'):
set(state='E')
def E(state='E'):
set(state='A')
class Sequence(SPA): #Define an SPA model (cortex, basal ganglia, thalamus)
dimensions=16
state=Buffer() #Create a working memory (recurrent network) object:
#i.e. a Buffer
BG=BasalGanglia(Rules()) #Create a basal ganglia with the prespecified
#set of rules
thal=Thalamus(BG) # Create a thalamus for that basal ganglia (so it
# uses the same rules)
seq=Sequence()
from spa import *
D=16
class Rules: #Define the rules by specifying the start state and the
#desired next state
def A(state='A'): #e.g. If in state A
set(state='B') # then go to state B
def B(state='B'):
set(state='C')
def C(state='C'):
set(state='D')
def D(state='D'):
set(state='E')
def E(state='E'):
set(state='A')
class Sequence(SPA): #Define an SPA model (cortex, basal ganglia, thalamus)
dimensions=16
state=Buffer() #Create a working memory (recurrent network) object:
#i.e. a Buffer
BG=BasalGanglia(Rules()) #Create a basal ganglia with the prespecified
#set of rules
thal=Thalamus(BG) # Create a thalamus for that basal ganglia (so it
# uses the same rules)
input=Input(0.1,state='D') #Define an input; set the input to
#state D for 100 ms
seq=Sequence()
from spa import *
D=16
class Rules: #Define the rules by specifying the start state and the
#desired next state
def start(vision='(A+B+C+D+E)*2'):
set(state=vision)
def A(state='A'): #e.g. If in state A
set(state='B') # then go to state B
def B(state='B'):
set(state='C')
def C(state='C'):
set(state='D')
def D(state='D'):
set(state='E')
def E(state='E'):
set(state='A')
class Routing(SPA): #Define an SPA model (cortex, basal ganglia, thalamus)
dimensions=16
state=Buffer() #Create a working memory (recurrent network)
#object: i.e. a Buffer
vision=Buffer(feedback=0) #Create a cortical network object with no
#recurrence (so no memory properties, just
#transient states)
BG=BasalGanglia(Rules) #Create a basal ganglia with the prespecified
#set of rules
thal=Thalamus(BG) # Create a thalamus for that basal ganglia (so it
# uses the same rules)
input=Input(0.1,vision='D')
model=Routing()
What do the actions look like?
State:
goal
: what disk am I trying to move (D0, D1, D2)focus
: what disk am I looking at (D0, D1, D2)goal_peg
: where is the disk I am trying to move (A, B, C)focus_peg
: where is the disk I am looking at (A, B, C)target_peg
: where am I trying to move a disk to (A, B, C)goal_final
: what is the overall final desired location of the disk I'm trying to move (A, B, C)Note: we're not yet modelling all the sensory and memory stuff here, so we manually set things like goal_final
.
Action effects: when an action is selected, it could do the following
focus
goal
goal_peg
move
and move_peg
Is this sufficient to implement the algorithm described above?
focus
=NONE then focus
=D2, goal
=D2, goal_peg
=goal_final
focus
$\cdot$ NONEfocus
=D2 and goal
=D2 and goal_peg
!=target_peg
then focus
=D1focus
$\cdot$ D2 + goal
$\cdot$ D2 - goal_peg
$\cdot$ target_peg
focus
=D2 and goal
=D2 and goal_peg
==target_peg
then focus
=D1, goal
=D1, goal_peg
=goal_final
focus
=D1 and goal
=D1 and goal_peg
!=target_peg
then focus
=D0focus
=D1 and goal
=D1 and goal_peg
==target_peg
then focus
=D0, goal
=D0, goal_peg
=goal_final
focus
=D0 and goal_peg
==target_peg
then focus
=NONEfocus
=D0 and goal
=D0 and goal_peg
!=target_peg
then focus
=NONE, move
=D0, move_peg
=target_peg
focus
!=goal
and focus_peg
==goal_peg
and target_peg!=focus_peg
then goal
=focus
, goal_peg
=A+B+C-target_peg
-focus_peg
focus
!=goal
and focus_peg
!=goal_peg
and target_peg==focus_peg
then goal
=focus
, goal_peg
=A+B+C-target_peg
-goal_peg
focus
=D0 and goal
!=D0 and target_peg
!=focus_peg
and target_peg
!=goal_peg
and focus_peg
!=goal_peg
then move
=goal
, move_peg
=target_peg
focus
=D1 and goal
!=D1 and target_peg
!=focus_peg
and target_peg
!=goal_peg
and focus_peg
!=goal_peg
then focus
=D0Do science
Timing: