Nengo DL converter

Quick reply, both #1 and #3 works in the meantime, going to try #2 along with removing of pooling. I do agree striding or dilation seem like more efficient approaches to “pooling”.

Thanks very much. Plenty to work on now…Also, the new example is pretty neat so great timing with that.
:nerd_face: :nerd_face:


So been playing with my model and its grew arms and legs now.
but i cant seem to solve a simple keras to nengo converter issue, dunno if im just missing the obv

but i am getting a
File "/home/paul/.local/lib/python3.6/site-packages/nengo_dl/", line 401, in __getitem__ return self.dict[self._get_key(key)] KeyError: <Reference wrapping <tf.Tensor 'input_1:0' shape=(None, 7999, 1) dtype=float32>>

which I don’t quite understand… i have just removed the if do training else use preloaded weights sections and just call a pretrained keras model as model and put it into the converter.

What else would I need to change in the run_network func ??

Hi Paul,

Can you attach your newest code that exhibits the problem? What do you mean by “call a pretrained keras model as model”? One possibility is that when you call nengo_converter.inputs[inp], the input inp that you’re using isn’t actually a part of the Keras model that you’ve passed to the converter. Is that the line in your script that’s causing the problem?

So essentially I’m trying to do the training outside of nengo now just to see and call the model in the converter

looking at my code slimmed down perhaps it’s odd there is a model = followed by another.

train_images = train_images.reshape((train_images.shape[0], 1, -1))
train_histograms = train_histograms.reshape((train_histograms.shape[0], 1, -1))
test_images = test_images.reshape((test_images.shape[0], 1, -1))
test_histograms = test_histograms.reshape((test_histograms.shape[0], 1, -1))

test_images = np.reshape(test_images, (test_images.shape[0], 1, 64, 64))
train_images = np.reshape(train_images, (train_images.shape[0], 1, 64, 64))

inp = Input(shape=(7999, 1))

conv1 = Conv1D(64, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(inp)
conv1 = Conv1D(64, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(conv1)
conv2 = Conv1D(128, 3, activation='relu', padding='valid', kernel_initializer='he_normal', strides=2)(conv1)
conv2 = Conv1D(128, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(conv2)

conv3 = Conv1D(256, 3, activation='relu', padding='valid', kernel_initializer='he_normal', strides=2)(conv2)
conv3 = Conv1D(256, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(conv3)
conv4 = Conv1D(512, 3, activation='relu', padding='valid', kernel_initializer='he_normal', strides=2)(conv3)
conv4 = Conv1D(512, 3, activation='relu', padding='valid', kernel_initializer='he_normal')(conv4)

conv5 = Conv1D(512, 3, activation='relu', padding='valid', kernel_initializer='he_normal', strides=2)(conv4)
conv5 = Conv1D(512, 242, activation='relu', padding='valid', kernel_initializer='he_normal')(conv5)

shape = tensorflow.keras.layers.Reshape((16, 16, 512))(conv5)

tconv1 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(shape)
tconv1 = UpSampling2D(size=(2, 2))(tconv1)
tconv1 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv1)

tconv2 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv1)
tconv2 = UpSampling2D(size=(2, 2))(tconv2)
tconv2 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv2)

tconv3 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv2)
tconv3 = UpSampling2D(size=(2, 2))(tconv3)
tconv3 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv3)

tconv4 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(tconv3)

out = Conv2D(1, 3, padding='same', kernel_initializer='he_normal')(tconv4)  # try without relu here?? activation='relu',

model = tensorflow.keras.Model(inputs=inp, outputs=out)

model = tensorflow.keras.models.load_model('upsample.h5')

converter = nengo_dl.Converter(model, scale_firing_rates=1.0001)

def run_network(activation, params_file="Keras_SNN_LIDAR_params2", n_steps=30,
                scale_firing_rates=1.0001, synapse=None, n_test=10):
    # convert the keras model to a nengo network
    nengo_converter = nengo_dl.Converter(
        swap_activations={tensorflow.nn.relu: activation},
    # # turn off the optimizer completely, as to not change the number of tensors
    # with
    #     nengo_dl.configure_settings(simplifications=[])

    # get input/output objects
    nengo_input = nengo_converter.inputs[inp]
    nengo_output = nengo_converter.outputs[out]

    # add a probe to the first convolutional layer to record activity
        conv1_probe = nengo.Probe(nengo_converter.layers[conv1])

    # repeat inputs for some number of timesteps ##### Could edit this to do sequence data with LMUs
    tiled_test_histograms = np.tile(test_histograms[:n_test], (1, n_steps, 1))

    # set some options to speed up simulation

    # build network, load in trained weights, run inference on test histograms
    with nengo_dl.Simulator(
  , minibatch_size=10,
            progress_bar=True) as nengo_sim:
        # nengo_sim.load_params(params_file)
        data = nengo_sim.predict({nengo_input: tiled_test_histograms})

    # compute accuracy on test data, using the output of network on
    # last timestep

    predictions = data[nengo_output][:, -1]
    mse = ((predictions - test_images[:n_test, 0, :]) ** 2).mean()

When you call tensorflow.keras.models.load_model, it’s loading the whole model, including all the tensors inside it. So the layers you’ve defined earlier (including inp) aren’t actually part of that new model; it all gets overwritten by the second model = ....

So I would get rid of the whole model definition before the second model = (or at least comment it out for now). Then you still need a way to get those input and output tensors. You should be able to do inp = model.inputs[0] and out = model.outputs[0] to get handles to those input and output tensors.

Ahh okay, I get my mistake. Similarly, I take it model.layers[x] is how I refer to any of the Conv layers for probes then.

Yes, exactly.

An alternative is to define your model as usual, and then use model.save_weights and model.load_weights to save and load the weights. I’m not sure if that will work with your existing weights (i.e. if you can just load the weights from a checkpoint that’s been created with save); I feel like it should (I think the .h5 file is all the weights), but I’ve never tried. But if you retrain the model and save the weights with save_weights, that should definitely work. The downside is you’ve got to always make sure that your model definition matches the saved weights, since none of the information about your model architecture/definition is saved.

Yeah i already tried that, seem to work well.

When i run the SReLU, i dont get spike output from the probe, i haven’t been able to either way i tried, with all the models shown above. i just seems to show the same output as the relu version.

i also still get a few of the 0) Resource exhausted: OOM when allocating tensor with shape[10,120,511680] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc [[{{node TensorGraph/transpose_1}}]] issue, but I think I need to come up with ways of resetting everything after each simulation

plt.subplot(1, 3, 2)
        sample_neurons = np.linspace(
        scaled_data = data[conv1_probe][ii, :, sample_neurons].T * scale_firing_rates
        if isinstance(activation, nengo.SpikingRectifiedLinear):
            scaled_data *= 0.001
            rates = np.sum(scaled_data, axis=0) / (n_steps * nengo_sim.dt)
            plt.ylabel('Number of spikes')
            rates = scaled_data
            plt.ylabel('Firing rates (Hz)')
            "Neural activities (conv1 mean=%dHz max=%dHz)" % (
                rates.mean(), rates.max())
        plt.title("Output Predictions of Image Over Time")

What it looks like from your plot is that your output is just your biases, i.e. no spikes are making it through your network. Try using scale_firing_rates to scale things up.

Yeah the same thing seems to happen even at these scaled versions of 2 and 20

and when i try the regularizor i get

File "/home/paul/PycharmProjects/LIDAR/", line 612, in <module> conv1_p: np.ones((train_images.shape[0], 1, conv1_p.size_in))
File "/home/paul/.local/lib/python3.6/site-packages/numpy/core/", line 207, in ones a = empty(shape, dtype, order)

MemoryError: Unable to allocate 43.5 GiB for an array with shape (11402, 1, 511680) and data type float64

I seen for the recent INRC meeting that y’all made a spiking unet for segmentation ?
So i believe in the Nengo way!

can get similar failure to spike and only biases in the example by adding a good few conv layers.

What does it mean by the output just being bias, is that similar to overfitting from the model?

No. The problem is that when you train an ANN, it doesn’t care about what the scale of the outputs are if you’re training with ReLUs. If the range of the outputs for a layer is [0, 1] or if it’s [0, 100] doesn’t make a difference to the network, because the subsequent layer can just e.g. scale down its weights by a factor of 100 in the case where the range is [0, 100]. Furthermore, the ANN could have different ranges for different layers.

So if your ANN happens to choose a lower range for a layer (e.g. [0, 3]) then when you switch to spiking neurons, the neurons in that layer are going to fire between 0 and 3 spikes a second; not very many.

The first thing I would try is higher firing rates scaling. Go up to 100, 1000, or even higher, just to see if you can get some output.

The more principled way to do it is to actually measure the outputs of different layers, to see how quickly neurons are firing. In the example notebook, we do this by looking at the number of non-zero timesteps (i.e. the percent of timesteps that the neuron has a non-zero output), and turning that into a firing rate.

For U-Net and other larger networks that we’ve made, we always use the regularization method, because it provides more exact control over the firing rates, and works for LIF neurons (the firing rate scaling only works for ReLUs because it depends on the fact that they’re scale-invariant).

Looking more at your memory error, I see why you’re getting it now. The firing rate regularization in that example has an inefficient way of passing the firing rate targets; since you have the same target for all neurons, there’s no need to pass a giant array where every element is the same value. Take a look at how I do firing rate regularization in the reworked CIFAR-10 example here.

Amazing, Thanks for all the help.

My goal is to convert to LIF next, but i need to get the regularizer working anyway, i will muster on with that from the loihi example, i mean thats where i want to port the model in the end also, but ways to go!.

I had tried scaling upto 1000 without any difference, but i’ll try a few different things.


Hi Paul!

Not sure if you are still having this issue of no spikes in the network, but I was running into the same problem trying to convert a u-net to spiking. I finally realized that the converter wasn’t actually converting the activation functions to SpikingRectictifiedLinear() (i.e. for ensemble in print(ensemble, ensemble.neuron_type) was showing all RectifiedLinear() outputs. To fix it, I had to switch over from using from keras.layers.Conv2D to tf.keras.layers.Conv2D. I also needed to switch from activation='relu' to activation='tf.nn.relu'. Small changes, but I think it had to do with the converter not recognizing the activation function.

1 Like

Hello @msanch35! You are correct, if you use activation="relu" and then swap it with following statement:

swap_activations={tf.nn.relu: nengo.SpikingRectifiedLinear()}

it won’t swap it. Rather you should use the following:

swap_activations={tf.keras.activations.relu: nengo.SpikingRectifiedLinear()}

while using “relu” in non-spiking network. This is adapted from Conversion of sequential model.

1 Like

Thanks so much to both @msanch35 and @zerone !!
That was exactly my problem, now if I scale the firing rate to like 10k I get actual spiking activity.

I had taken a while off the project until I had more time to implement a few things to try and get it working, but thanks to the magic of the forum it solved itself! (or well someone else solved it for me) :smiley:

now I might actually get to the LIF neuron stage


Hi @Eric,

I am using NengoDL version 3.3.0

I am having the same issue with:
nengo.exceptions.SimulationError: Number of saved parameters in ./my-model (5) != number of variables in the model (4)

My network is a custom network with 3 hidden layers 50 neurons each,
I’ve tried the suggested solutions #1 and #3 and all don’t work in my network architecture.
I am trying to avoid rewriting code as suggested in approach #2

Do you have any additional suggestion for my case?

Best Regards,

Hi @alex.hex,

Are you sure that your model is the same architecture that you used to save my-model? The other likely cause of this error is that your model changed between when you created my-model and the current version of your script, so that the number of parameter tensors is actually different.

Otherwise, I can’t think of any reasons why you should get this error. If you can reduce the problem to a specific reproducible test case, you can post it as a bug report on NengoDL issues.

Hi Eric,
I don’t know why reason is when I convert Darknet 19 to SNN, I have an error " SimulationError: Number of saved parameters in keras_to_snn_params (41) != number of variables in the model (44)" if synapse =0.05 or non zeros and no error for synapse=None even I turn off optimizer " ```


conv0 = tf.keras.layers.Conv2D(filters=8,kernel_size=3,padding="same", activation=tf.nn.relu)(inp)

pool0 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv0)

conv1 = tf.keras.layers.Conv2D(filters=16,kernel_size=3,padding="same", activation=tf.nn.relu)(pool0)

pool1 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv1)

conv2 = tf.keras.layers.Conv2D(filters=32,kernel_size=3,padding="same", activation=tf.nn.relu)(pool1)

conv3 = tf.keras.layers.Conv2D(filters=16,kernel_size=1,padding="same", activation=tf.nn.relu)(conv2)

conv4 = tf.keras.layers.Conv2D(filters=32,kernel_size=3,padding="same", activation=tf.nn.relu)(conv3)

pool2 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv4)

conv5 = tf.keras.layers.Conv2D(filters=64,kernel_size=3,padding="same", activation=tf.nn.relu)(pool2)

conv6 = tf.keras.layers.Conv2D(filters=32,kernel_size=1,padding="same", activation=tf.nn.relu)(conv5)

conv7 = tf.keras.layers.Conv2D(filters=64,kernel_size=3,padding="same", activation=tf.nn.relu)(conv6)

pool3 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv7)

conv8 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(pool3)

conv9 = tf.keras.layers.Conv2D(filters=64,kernel_size=1,padding="same", activation=tf.nn.relu)(conv8)

conv10 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(conv9)

conv11 = tf.keras.layers.Conv2D(filters=64,kernel_size=1,padding="same", activation=tf.nn.relu)(conv10)

conv12 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(conv11)

# pool4 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2,padding='same')(conv12)

conv13 = tf.keras.layers.Conv2D(filters=256,kernel_size=3,padding="same", activation=tf.nn.relu)(conv12)

conv14 = tf.keras.layers.Conv2D(filters=128,kernel_size=1,padding="same", activation=tf.nn.relu)(conv13)

conv15 = tf.keras.layers.Conv2D(filters=356,kernel_size=3,padding="same", activation=tf.nn.relu)(conv14)

conv16 = tf.keras.layers.Conv2D(filters=128,kernel_size=1,padding="same", activation=tf.nn.relu)(conv15)

conv17 = tf.keras.layers.Conv2D(filters=256,kernel_size=3,padding="same", activation=tf.nn.relu)(conv16)

conv18 = tf.keras.layers.Conv2D(filters=250,kernel_size=1,padding="same", activation=tf.nn.relu)(conv16)


flatten = tf.keras.layers.Flatten()(pool5)

dense = tf.keras.layers.Dense(units=10)(flatten)

model = tf.keras.Model(inputs = inp, outputs = dense)


converter = nengo_dl.Converter(model)

do_training = True
if do_training:
    with nengo_dl.Simulator(, minibatch_size=200) as sim:
        # run training
            {converter.inputs[inp]: train_images},
            {converter.outputs[dense]: train_labels},
                {converter.inputs[inp]: test_images},
                {converter.outputs[dense]: test_labels},

        # save the parameters to file
    # download pretrained weights

def run_network(
    # convert the keras model to a nengo network
    nengo_converter = nengo_dl.Converter(
        swap_activations={tf.nn.relu: activation},

    # get input/output objects
    nengo_input = nengo_converter.inputs[inp]
    nengo_output = nengo_converter.outputs[dense]

    # add a probe to the first convolutional layer to record activity.
    # we'll only record from a subset of neurons, to save memory.
    sample_neurons = np.linspace(
        conv0_probe = nengo.Probe(nengo_converter.layers[conv0][sample_neurons])
    # repeat inputs for some number of timesteps
    tiled_test_images = np.tile(test_images[:n_test], (1, n_steps, 1))

    # set some options to speed up simulation

    # build network, load in trained weights, run inference on test images
    with nengo_dl.Simulator(, minibatch_size=10, progress_bar=False
    ) as nengo_sim:
        params = list(nengo_sim.keras_model.weights)
        data = nengo_sim.predict({nengo_input: tiled_test_images})

    # compute accuracy on test data, using output of network on
    # last timestep
    predictions = np.argmax(data[nengo_output][:, -1], axis=-1)
    accuracy = (predictions == test_labels[:n_test, 0, 0]).mean()
    print(f"Test accuracy: {100 * accuracy:.2f}%")

    # plot the results
    for ii in range(3):
        plt.figure(figsize=(12, 4))

        plt.subplot(1, 3, 1)
        plt.title("Input image")
        plt.imshow(test_images[ii, 0].reshape((28, 28)), cmap="gray")

        plt.subplot(1, 3, 2)
        scaled_data = data[conv0_probe][ii] * scale_firing_rates
        if isinstance(activation, nengo.SpikingRectifiedLinear):
            scaled_data *= 0.001
            rates = np.sum(scaled_data, axis=0) / (n_steps * nengo_sim.dt)
            plt.ylabel("Number of spikes")
            rates = scaled_data
            plt.ylabel("Firing rates (Hz)")
            f"Neural activities (conv0 mean={rates.mean():.1f} Hz, "
            f"max={rates.max():.1f} Hz)"

        plt.subplot(1, 3, 3)
        plt.title("Output predictions")
        plt.legend([str(j) for j in range(10)], loc="upper left")

    print("Loaded pretrained weights")

for s in [0.1, 0.02, 0.07]:

This has been addressed here. :smiley: