Quick reply, both #1 and #3 works in the meantime, going to try #2 along with removing of pooling. I do agree striding or dilation seem like more efficient approaches to âpoolingâ.
Thanks very much. Plenty to work on nowâŚAlso, the new example is pretty neat so great timing with that.
Hey,
So been playing with my model and its grew arms and legs now.
but i cant seem to solve a simple keras to nengo converter issue, dunno if im just missing the obv
but i am getting a
File "/home/paul/.local/lib/python3.6/site-packages/nengo_dl/converter.py", line 401, in __getitem__ return self.dict[self._get_key(key)] KeyError: <Reference wrapping <tf.Tensor 'input_1:0' shape=(None, 7999, 1) dtype=float32>>
which I donât quite understand⌠i have just removed the if do training else use preloaded weights sections and just call a pretrained keras model as model and put it into the converter.
What else would I need to change in the run_network func ??
Hi Paul,
Can you attach your newest code that exhibits the problem? What do you mean by âcall a pretrained keras model as modelâ? One possibility is that when you call nengo_converter.inputs[inp], the input inp that youâre using isnât actually a part of the Keras model that youâve passed to the converter. Is that the line in your script thatâs causing the problem?
So essentially Iâm trying to do the training outside of nengo now just to see and call the model in the converter
looking at my code slimmed down perhaps itâs odd there is a model = followed by another.
train_images = train_images.reshape((train_images.shape[0], 1, -1))
train_histograms = train_histograms.reshape((train_histograms.shape[0], 1, -1))
test_images = test_images.reshape((test_images.shape[0], 1, -1))
test_histograms = test_histograms.reshape((test_histograms.shape[0], 1, -1))
test_images = np.reshape(test_images, (test_images.shape[0], 1, 64, 64))
train_images = np.reshape(train_images, (train_images.shape[0], 1, 64, 64))
inp = Input(shape=(7999, 1))
conv1 = Conv1D(64, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(inp)
conv1 = Conv1D(64, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(conv1)
conv2 = Conv1D(128, 3, activation='relu', padding='valid', kernel_initializer='he_normal', strides=2)(conv1)
conv2 = Conv1D(128, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(conv2)
conv3 = Conv1D(256, 3, activation='relu', padding='valid', kernel_initializer='he_normal', strides=2)(conv2)
conv3 = Conv1D(256, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(conv3)
conv4 = Conv1D(512, 3, activation='relu', padding='valid', kernel_initializer='he_normal', strides=2)(conv3)
conv4 = Conv1D(512, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(conv4)
conv5 = Conv1D(512, 3, activation='relu', padding='valid', kernel_initializer='he_normal', strides=2)(conv4)
conv5 = Conv1D(512, 242, activation='relu', padding='valid', kernel_initializer='he_normal')(conv5)
shape = tensorflow.keras.layers.Reshape((16, 16, 512))(conv5)
tconv1 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(shape)
tconv1 = UpSampling2D(size=(2, 2))(tconv1)
tconv1 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv1)
tconv2 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv1)
tconv2 = UpSampling2D(size=(2, 2))(tconv2)
tconv2 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv2)
tconv3 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv2)
tconv3 = UpSampling2D(size=(2, 2))(tconv3)
tconv3 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv3)
tconv4 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv3)
out = Conv2D(1, 3, padding='same', kernel_initializer='he_normal')(tconv4) # try without relu here?? activation='relu',
model = tensorflow.keras.Model(inputs=inp, outputs=out)
model = tensorflow.keras.models.load_model('upsample.h5')
converter = nengo_dl.Converter(model, scale_firing_rates=1.0001)
def run_network(activation, params_file="Keras_SNN_LIDAR_params2", n_steps=30,
scale_firing_rates=1.0001, synapse=None, n_test=10):
# convert the keras model to a nengo network
nengo_converter = nengo_dl.Converter(
model,
swap_activations={tensorflow.nn.relu: activation},
scale_firing_rates=scale_firing_rates,
synapse=synapse,
)
# # turn off the optimizer completely, as to not change the number of tensors
# with converter.net:
# nengo_dl.configure_settings(simplifications=[])
# get input/output objects
nengo_input = nengo_converter.inputs[inp]
nengo_output = nengo_converter.outputs[out]
# add a probe to the first convolutional layer to record activity
with nengo_converter.net:
conv1_probe = nengo.Probe(nengo_converter.layers[conv1])
# repeat inputs for some number of timesteps ##### Could edit this to do sequence data with LMUs
tiled_test_histograms = np.tile(test_histograms[:n_test], (1, n_steps, 1))
# set some options to speed up simulation
with nengo_converter.net:
nengo_dl.configure_settings(stateful=False)
# build network, load in trained weights, run inference on test histograms
with nengo_dl.Simulator(
nengo_converter.net, minibatch_size=10,
progress_bar=True) as nengo_sim:
# nengo_sim.load_params(params_file)
data = nengo_sim.predict({nengo_input: tiled_test_histograms})
# compute accuracy on test data, using the output of network on
# last timestep
predictions = data[nengo_output][:, -1]
mse = ((predictions - test_images[:n_test, 0, :]) ** 2).mean()When you call tensorflow.keras.models.load_model, itâs loading the whole model, including all the tensors inside it. So the layers youâve defined earlier (including inp) arenât actually part of that new model; it all gets overwritten by the second model = ....
So I would get rid of the whole model definition before the second model = (or at least comment it out for now). Then you still need a way to get those input and output tensors. You should be able to do inp = model.inputs[0] and out = model.outputs[0] to get handles to those input and output tensors.
Ahh okay, I get my mistake. Similarly, I take it model.layers[x] is how I refer to any of the Conv layers for probes then.
Yes, exactly.
An alternative is to define your model as usual, and then use model.save_weights and model.load_weights to save and load the weights. Iâm not sure if that will work with your existing weights (i.e. if you can just load the weights from a checkpoint thatâs been created with save); I feel like it should (I think the .h5 file is all the weights), but Iâve never tried. But if you retrain the model and save the weights with save_weights, that should definitely work. The downside is youâve got to always make sure that your model definition matches the saved weights, since none of the information about your model architecture/definition is saved.
Yeah i already tried that, seem to work well.
When i run the SReLU, i dont get spike output from the probe, i havenât been able to either way i tried, with all the models shown above. i just seems to show the same output as the relu version.
i also still get a few of the 0) Resource exhausted: OOM when allocating tensor with shape[10,120,511680] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc [[{{node TensorGraph/transpose_1}}]] issue, but I think I need to come up with ways of resetting everything after each simulation
plt.subplot(1, 3, 2)
sample_neurons = np.linspace(
0,
data[conv1_probe].shape[-1],
1000,
endpoint=False,
dtype=np.int32,
)
scaled_data = data[conv1_probe][ii, :, sample_neurons].T * scale_firing_rates
if isinstance(activation, nengo.SpikingRectifiedLinear):
scaled_data *= 0.001
rates = np.sum(scaled_data, axis=0) / (n_steps * nengo_sim.dt)
plt.ylabel('Number of spikes')
else:
rates = scaled_data
plt.ylabel('Firing rates (Hz)')
plt.xlabel('Timestep')
plt.title(
"Neural activities (conv1 mean=%dHz max=%dHz)" % (
rates.mean(), rates.max())
)
plt.plot(scaled_data)
plt.title("Output Predictions of Image Over Time")What it looks like from your plot is that your output is just your biases, i.e. no spikes are making it through your network. Try using scale_firing_rates to scale things up.
Yeah the same thing seems to happen even at these scaled versions of 2 and 20
and when i try the regularizor i get
File "/home/paul/PycharmProjects/LIDAR/keras-snn.py", line 612, in <module> conv1_p: np.ones((train_images.shape[0], 1, conv1_p.size_in))
File "/home/paul/.local/lib/python3.6/site-packages/numpy/core/numeric.py", line 207, in ones a = empty(shape, dtype, order)
MemoryError: Unable to allocate 43.5 GiB for an array with shape (11402, 1, 511680) and data type float64
I seen for the recent INRC meeting that yâall made a spiking unet for segmentation ?
So i believe in the Nengo way!
can get similar failure to spike and only biases in the example by adding a good few conv layers.
What does it mean by the output just being bias, is that similar to overfitting from the model?
No. The problem is that when you train an ANN, it doesnât care about what the scale of the outputs are if youâre training with ReLUs. If the range of the outputs for a layer is [0, 1] or if itâs [0, 100] doesnât make a difference to the network, because the subsequent layer can just e.g. scale down its weights by a factor of 100 in the case where the range is [0, 100]. Furthermore, the ANN could have different ranges for different layers.
So if your ANN happens to choose a lower range for a layer (e.g. [0, 3]) then when you switch to spiking neurons, the neurons in that layer are going to fire between 0 and 3 spikes a second; not very many.
The first thing I would try is higher firing rates scaling. Go up to 100, 1000, or even higher, just to see if you can get some output.
The more principled way to do it is to actually measure the outputs of different layers, to see how quickly neurons are firing. In the example notebook, we do this by looking at the number of non-zero timesteps (i.e. the percent of timesteps that the neuron has a non-zero output), and turning that into a firing rate.
For U-Net and other larger networks that weâve made, we always use the regularization method, because it provides more exact control over the firing rates, and works for LIF neurons (the firing rate scaling only works for ReLUs because it depends on the fact that theyâre scale-invariant).
Looking more at your memory error, I see why youâre getting it now. The firing rate regularization in that example has an inefficient way of passing the firing rate targets; since you have the same target for all neurons, thereâs no need to pass a giant array where every element is the same value. Take a look at how I do firing rate regularization in the reworked CIFAR-10 example here.
Amazing, Thanks for all the help.
My goal is to convert to LIF next, but i need to get the regularizer working anyway, i will muster on with that from the loihi example, i mean thats where i want to port the model in the end also, but ways to go!.
I had tried scaling upto 1000 without any difference, but iâll try a few different things.
Cheers
Hi Paul!
Not sure if you are still having this issue of no spikes in the network, but I was running into the same problem trying to convert a u-net to spiking. I finally realized that the converter wasnât actually converting the activation functions to SpikingRectictifiedLinear() (i.e. for ensemble in nengo_converter.net.ensembles: print(ensemble, ensemble.neuron_type) was showing all RectifiedLinear() outputs. To fix it, I had to switch over from using from keras.layers.Conv2D to tf.keras.layers.Conv2D. I also needed to switch from activation='relu' to activation='tf.nn.relu'. Small changes, but I think it had to do with the converter not recognizing the activation function.
Hello @msanch35! You are correct, if you use activation="relu" and then swap it with following statement:
swap_activations={tf.nn.relu: nengo.SpikingRectifiedLinear()}
it wonât swap it. Rather you should use the following:
swap_activations={tf.keras.activations.relu: nengo.SpikingRectifiedLinear()}
while using âreluâ in non-spiking network. This is adapted from Conversion of sequential model.
Thanks so much to both @msanch35 and @zerone !!
That was exactly my problem, now if I scale the firing rate to like 10k I get actual spiking activity.
I had taken a while off the project until I had more time to implement a few things to try and get it working, but thanks to the magic of the forum it solved itself! (or well someone else solved it for me) 
now I might actually get to the LIF neuron stage
Hi @Eric,
I am using NengoDL version 3.3.0
I am having the same issue with:
nengo.exceptions.SimulationError: Number of saved parameters in ./my-model (5) != number of variables in the model (4)
My network is a custom network with 3 hidden layers 50 neurons each,
Iâve tried the suggested solutions #1 and #3 and all donât work in my network architecture.
I am trying to avoid rewriting code as suggested in approach #2
Do you have any additional suggestion for my case?
Best Regards,
Alex
Hi @alex.hex,
Are you sure that your model is the same architecture that you used to save my-model? The other likely cause of this error is that your model changed between when you created my-model and the current version of your script, so that the number of parameter tensors is actually different.
Otherwise, I canât think of any reasons why you should get this error. If you can reduce the problem to a specific reproducible test case, you can post it as a bug report on NengoDL issues.
Hi Eric,
I donât know why reason is when I convert Darknet 19 to SNN, I have an error " SimulationError: Number of saved parameters in keras_to_snn_params (41) != number of variables in the model (44)" if synapse =0.05 or non zeros and no error for synapse=None even I turn off optimizer " ```
with converter.net:
nengo_dl.configure_settings(simplifications=[])
inp=tf.keras.layers.Input(shape=(28,28,1))
conv0 = tf.keras.layers.Conv2D(filters=8,kernel_size=3,padding="same", activation=tf.nn.relu)(inp)
pool0 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv0)
conv1 = tf.keras.layers.Conv2D(filters=16,kernel_size=3,padding="same", activation=tf.nn.relu)(pool0)
pool1 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv1)
conv2 = tf.keras.layers.Conv2D(filters=32,kernel_size=3,padding="same", activation=tf.nn.relu)(pool1)
conv3 = tf.keras.layers.Conv2D(filters=16,kernel_size=1,padding="same", activation=tf.nn.relu)(conv2)
conv4 = tf.keras.layers.Conv2D(filters=32,kernel_size=3,padding="same", activation=tf.nn.relu)(conv3)
pool2 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv4)
conv5 = tf.keras.layers.Conv2D(filters=64,kernel_size=3,padding="same", activation=tf.nn.relu)(pool2)
conv6 = tf.keras.layers.Conv2D(filters=32,kernel_size=1,padding="same", activation=tf.nn.relu)(conv5)
conv7 = tf.keras.layers.Conv2D(filters=64,kernel_size=3,padding="same", activation=tf.nn.relu)(conv6)
pool3 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv7)
conv8 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(pool3)
conv9 = tf.keras.layers.Conv2D(filters=64,kernel_size=1,padding="same", activation=tf.nn.relu)(conv8)
conv10 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(conv9)
conv11 = tf.keras.layers.Conv2D(filters=64,kernel_size=1,padding="same", activation=tf.nn.relu)(conv10)
conv12 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(conv11)
# pool4 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2,padding='same')(conv12)
conv13 = tf.keras.layers.Conv2D(filters=256,kernel_size=3,padding="same", activation=tf.nn.relu)(conv12)
conv14 = tf.keras.layers.Conv2D(filters=128,kernel_size=1,padding="same", activation=tf.nn.relu)(conv13)
conv15 = tf.keras.layers.Conv2D(filters=356,kernel_size=3,padding="same", activation=tf.nn.relu)(conv14)
conv16 = tf.keras.layers.Conv2D(filters=128,kernel_size=1,padding="same", activation=tf.nn.relu)(conv15)
conv17 = tf.keras.layers.Conv2D(filters=256,kernel_size=3,padding="same", activation=tf.nn.relu)(conv16)
conv18 = tf.keras.layers.Conv2D(filters=250,kernel_size=1,padding="same", activation=tf.nn.relu)(conv16)
pool5=tf.keras.layers.GlobalAveragePooling2D()(conv18)
flatten = tf.keras.layers.Flatten()(pool5)
dense = tf.keras.layers.Dense(units=10)(flatten)
model = tf.keras.Model(inputs = inp, outputs = dense)
model.summary()
converter = nengo_dl.Converter(model)
do_training = True
if do_training:
with converter.net:
nengo_dl.configure_settings(simplifications=[])
with nengo_dl.Simulator(converter.net, minibatch_size=200) as sim:
# run training
sim.compile(
optimizer=tf.optimizers.Adam(0.001),
loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.metrics.sparse_categorical_accuracy],
)
sim.fit(
{converter.inputs[inp]: train_images},
{converter.outputs[dense]: train_labels},
validation_data=(
{converter.inputs[inp]: test_images},
{converter.outputs[dense]: test_labels},
),
epochs=2,
)
# save the parameters to file
sim.save_params("./keras_to_snn_params")
else:
# download pretrained weights
urlretrieve(
"https://drive.google.com/uc?export=download&"
"id=1lBkR968AQo__t8sMMeDYGTQpBJZIs2_T",
"keras_to_snn_params.npz",
)
def run_network(
activation,
params_file="keras_to_snn_params",
n_steps=120,
scale_firing_rates=1,
synapse=None,
n_test=400,
):
# convert the keras model to a nengo network
nengo_converter = nengo_dl.Converter(
model,
swap_activations={tf.nn.relu: activation},
scale_firing_rates=scale_firing_rates,
synapse=synapse,
)
# get input/output objects
nengo_input = nengo_converter.inputs[inp]
nengo_output = nengo_converter.outputs[dense]
with nengo_converter.net:
nengo_dl.configure_settings(simplifications=[])
# add a probe to the first convolutional layer to record activity.
# we'll only record from a subset of neurons, to save memory.
sample_neurons = np.linspace(
0,
np.prod(conv0.shape[1:]),
1000,
endpoint=False,
dtype=np.int32,
)
with nengo_converter.net:
conv0_probe = nengo.Probe(nengo_converter.layers[conv0][sample_neurons])
# repeat inputs for some number of timesteps
tiled_test_images = np.tile(test_images[:n_test], (1, n_steps, 1))
# set some options to speed up simulation
with nengo_converter.net:
nengo_dl.configure_settings(stateful=False)
# build network, load in trained weights, run inference on test images
with nengo_dl.Simulator(
nengo_converter.net, minibatch_size=10, progress_bar=False
) as nengo_sim:
params = list(nengo_sim.keras_model.weights)
print(len(params))
nengo_sim.load_params(params_file)
data = nengo_sim.predict({nengo_input: tiled_test_images})
# compute accuracy on test data, using output of network on
# last timestep
predictions = np.argmax(data[nengo_output][:, -1], axis=-1)
accuracy = (predictions == test_labels[:n_test, 0, 0]).mean()
print(f"Test accuracy: {100 * accuracy:.2f}%")
# plot the results
for ii in range(3):
plt.figure(figsize=(12, 4))
plt.subplot(1, 3, 1)
plt.title("Input image")
plt.imshow(test_images[ii, 0].reshape((28, 28)), cmap="gray")
plt.axis("off")
plt.subplot(1, 3, 2)
scaled_data = data[conv0_probe][ii] * scale_firing_rates
if isinstance(activation, nengo.SpikingRectifiedLinear):
scaled_data *= 0.001
rates = np.sum(scaled_data, axis=0) / (n_steps * nengo_sim.dt)
plt.ylabel("Number of spikes")
else:
rates = scaled_data
plt.ylabel("Firing rates (Hz)")
plt.xlabel("Timestep")
plt.title(
f"Neural activities (conv0 mean={rates.mean():.1f} Hz, "
f"max={rates.max():.1f} Hz)"
)
plt.plot(scaled_data)
plt.subplot(1, 3, 3)
plt.title("Output predictions")
plt.plot(tf.nn.softmax(data[nengo_output][ii]))
plt.legend([str(j) for j in range(10)], loc="upper left")
plt.xlabel("Timestep")
plt.ylabel("Probability")
plt.tight_layout()
print("Loaded pretrained weights")
for s in [0.1, 0.02, 0.07]:
print(f"Synapse={s:.3f}")
run_network(
activation=nengo.SpikingRectifiedLinear(),
n_steps=10,
synapse=s,
)
plt.show()