Convert Darknet 19 to SNN

General Discussion
1

I don’t know why reason is when I convert Darknet 19 to SNN, I have an error

**SimulationError: Number of saved parameters in keras_to_snn_params (41) != number of variables in the model (44)**

if synapse =0.05 or non zeros and no error for synapse=None even I turn off optimizer "

with converter.net:
    nengo_dl.configure_settings(simplifications=[])


inp=tf.keras.layers.Input(shape=(28,28,1))

conv0 = tf.keras.layers.Conv2D(filters=8,kernel_size=3,padding="same", activation=tf.nn.relu)(inp)

pool0 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv0)

conv1 = tf.keras.layers.Conv2D(filters=16,kernel_size=3,padding="same", activation=tf.nn.relu)(pool0)

pool1 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv1)

conv2 = tf.keras.layers.Conv2D(filters=32,kernel_size=3,padding="same", activation=tf.nn.relu)(pool1)

conv3 = tf.keras.layers.Conv2D(filters=16,kernel_size=1,padding="same", activation=tf.nn.relu)(conv2)

conv4 = tf.keras.layers.Conv2D(filters=32,kernel_size=3,padding="same", activation=tf.nn.relu)(conv3)

pool2 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv4)

conv5 = tf.keras.layers.Conv2D(filters=64,kernel_size=3,padding="same", activation=tf.nn.relu)(pool2)

conv6 = tf.keras.layers.Conv2D(filters=32,kernel_size=1,padding="same", activation=tf.nn.relu)(conv5)

conv7 = tf.keras.layers.Conv2D(filters=64,kernel_size=3,padding="same", activation=tf.nn.relu)(conv6)

pool3 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv7)

conv8 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(pool3)

conv9 = tf.keras.layers.Conv2D(filters=64,kernel_size=1,padding="same", activation=tf.nn.relu)(conv8)

conv10 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(conv9)

conv11 = tf.keras.layers.Conv2D(filters=64,kernel_size=1,padding="same", activation=tf.nn.relu)(conv10)

conv12 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(conv11)

# pool4 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2,padding='same')(conv12)

conv13 = tf.keras.layers.Conv2D(filters=256,kernel_size=3,padding="same", activation=tf.nn.relu)(conv12)

conv14 = tf.keras.layers.Conv2D(filters=128,kernel_size=1,padding="same", activation=tf.nn.relu)(conv13)

conv15 = tf.keras.layers.Conv2D(filters=356,kernel_size=3,padding="same", activation=tf.nn.relu)(conv14)

conv16 = tf.keras.layers.Conv2D(filters=128,kernel_size=1,padding="same", activation=tf.nn.relu)(conv15)

conv17 = tf.keras.layers.Conv2D(filters=256,kernel_size=3,padding="same", activation=tf.nn.relu)(conv16)

conv18 = tf.keras.layers.Conv2D(filters=250,kernel_size=1,padding="same", activation=tf.nn.relu)(conv16)

pool5=tf.keras.layers.GlobalAveragePooling2D()(conv18)

flatten = tf.keras.layers.Flatten()(pool5)

dense = tf.keras.layers.Dense(units=10)(flatten)

model = tf.keras.Model(inputs = inp, outputs = dense)

model.summary()

converter = nengo_dl.Converter(model)

do_training = True
if do_training:
    with converter.net:
        nengo_dl.configure_settings(simplifications=[])
    with nengo_dl.Simulator(converter.net, minibatch_size=200) as sim:
        # run training
        sim.compile(
            optimizer=tf.optimizers.Adam(0.001),
            loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
            metrics=[tf.metrics.sparse_categorical_accuracy],
        )
        sim.fit(
            {converter.inputs[inp]: train_images},
            {converter.outputs[dense]: train_labels},
            validation_data=(
                {converter.inputs[inp]: test_images},
                {converter.outputs[dense]: test_labels},
            ),
            epochs=2,
        )
       

        # save the parameters to file
        sim.save_params("./keras_to_snn_params")
else:
    # download pretrained weights
    urlretrieve(
        "https://drive.google.com/uc?export=download&"
        "id=1lBkR968AQo__t8sMMeDYGTQpBJZIs2_T",
        "keras_to_snn_params.npz",
    )

def run_network(
    activation,
    params_file="keras_to_snn_params",
    n_steps=120,
    scale_firing_rates=1,
    synapse=None,
    n_test=400,
):
    # convert the keras model to a nengo network
    nengo_converter = nengo_dl.Converter(
        model,
        swap_activations={tf.nn.relu: activation},
        scale_firing_rates=scale_firing_rates,
        synapse=synapse,
    )

    # get input/output objects
    nengo_input = nengo_converter.inputs[inp]
    nengo_output = nengo_converter.outputs[dense]
    with nengo_converter.net:
        nengo_dl.configure_settings(simplifications=[])
    

    # add a probe to the first convolutional layer to record activity.
    # we'll only record from a subset of neurons, to save memory.
    sample_neurons = np.linspace(
        0,
        np.prod(conv0.shape[1:]),
        1000,
        endpoint=False,
        dtype=np.int32,
    )
    with nengo_converter.net:
        conv0_probe = nengo.Probe(nengo_converter.layers[conv0][sample_neurons])
        
   
    # repeat inputs for some number of timesteps
    tiled_test_images = np.tile(test_images[:n_test], (1, n_steps, 1))

    # set some options to speed up simulation
    with nengo_converter.net:
        nengo_dl.configure_settings(stateful=False)

    # build network, load in trained weights, run inference on test images
    with nengo_dl.Simulator(
        nengo_converter.net, minibatch_size=10, progress_bar=False
    ) as nengo_sim:
        params = list(nengo_sim.keras_model.weights)
        print(len(params))
        nengo_sim.load_params(params_file)
        data = nengo_sim.predict({nengo_input: tiled_test_images})

    # compute accuracy on test data, using output of network on
    # last timestep
    predictions = np.argmax(data[nengo_output][:, -1], axis=-1)
    accuracy = (predictions == test_labels[:n_test, 0, 0]).mean()
    print(f"Test accuracy: {100 * accuracy:.2f}%")

    # plot the results
    for ii in range(3):
        plt.figure(figsize=(12, 4))

        plt.subplot(1, 3, 1)
        plt.title("Input image")
        plt.imshow(test_images[ii, 0].reshape((28, 28)), cmap="gray")
        plt.axis("off")

        plt.subplot(1, 3, 2)
        scaled_data = data[conv0_probe][ii] * scale_firing_rates
        if isinstance(activation, nengo.SpikingRectifiedLinear):
            scaled_data *= 0.001
            rates = np.sum(scaled_data, axis=0) / (n_steps * nengo_sim.dt)
            plt.ylabel("Number of spikes")
        else:
            rates = scaled_data
            plt.ylabel("Firing rates (Hz)")
        plt.xlabel("Timestep")
        plt.title(
            f"Neural activities (conv0 mean={rates.mean():.1f} Hz, "
            f"max={rates.max():.1f} Hz)"
        )
        plt.plot(scaled_data)

        plt.subplot(1, 3, 3)
        plt.title("Output predictions")
        plt.plot(tf.nn.softmax(data[nengo_output][ii]))
        plt.legend([str(j) for j in range(10)], loc="upper left")
        plt.xlabel("Timestep")
        plt.ylabel("Probability")

        plt.tight_layout()
    print("Loaded pretrained weights")

for s in [0.1, 0.02, 0.07]:
    print(f"Synapse={s:.3f}")
    run_network(
        activation=nengo.SpikingRectifiedLinear(),
        n_steps=10,
        synapse=s,
    )
    plt.show()
2

Hi @ntquyen2, and welcome to the Nengo forums!

With your code in particular, it looks like the NengoDL graph optimizer has come up with two different solutions (when the synapses are included vs when they are not). To get around this issue, you’ll want to disable the graph optimizer entirely.

To disable the graph optimizer, configure the NengoDL settings to use planner=nengo_dl.graph_optimizer.noop_planner instead of simplications=[].

As a quick explanation, simplications=[] still applies the operator merging, but disables simplifications on the transformation functions. As comparision, planner=noop_planner disables the graph optimizer entirely.

1 Like
3

Thanks @xchoo for yours answer, I changed planner=nengo_dl.graph_optimizer.noop_plannerinstead ofsimplications=[] but I had error ValueError: Cannot assign to variable TensorGraph/base_params/trainable_float32_3_3_32_16:0 due to variable shape (3, 3, 32, 16) and value shape (3, 3, 16, 32) are incompatible and I know the reason for it.

do_training = True
if do_training:
    # with converter.net:
    #     nengo_dl.configure_settings(simplifications=[])

    with converter.net:
          nengo_dl.configure_settings(planner=noop_planner)

and

def run_network(
    activation,
    params_file="keras_to_snn_params",
    n_steps=120,
    scale_firing_rates=5,
    synapse=None,
    n_test=400,
):
    # convert the keras model to a nengo network
    nengo_converter = nengo_dl.Converter(
        model,
        swap_activations={tf.nn.relu: activation},
        scale_firing_rates=scale_firing_rates,
        synapse=synapse,
    )

    # get input/output objects
    nengo_input = nengo_converter.inputs[inp]
    nengo_output = nengo_converter.outputs[dense]
    with nengo_converter.net:
        nengo_dl.configure_settings(planner=noop_planner)
4

I am unable to reproduce this error using the code you posted in your first post. If you can, can you please include your entire script in your reply so that I can run it?

Here is the script I used to test your code, and it seems to run fine on my machine:
test_tf_convert_synapse.py (8.1 KB)
Perhaps you can compare the code to what you have and see if you can spot the difference causing your error. :slight_smile:

5

I’m really grateful for your help @xchoo . I fixed my code and I still want to say " Thank you so much <3"

6

Hi @xchoo, I want to ask you why reason is when I convert Darknet 19 to SNN, I have UserWarning: average_pooling2d_4.padding has value same != valid, which is not supported. Falling back to TensorNode.
% (error_msg + ". " if error_msg else “”) if I don’t comment pool4 in my above code although I still run fine my code. Moreover, I convert small model such as Alexnet to SNN, I don’t have to turn off optimizer whereas Darknet have to it. May be Darknet is a large model??? Can you help me, please?

7

The NengoDL converter attempts to convert all of the TensorFlow objects into Nengo-native objects (e.g., tf.keras.layers.dense gets converted into a nengo.Ensemble). This warning is just to let you know that the specific configuration of the average pooling layer (i.e., with padding=same) doesn’t have a default Nengo object mapping, and so, NengoDL is using a generic TensorNode to implement that specific layer.

If you don’t need the Nengo specific objects, then having the TensorNode should be fine for your use. Otherwise, you’ll need to look at implementing a Nengo specific translation for the tf.keras.layers.AveragePooling2D(..., padding="same") layer. You can read up on how to do this here! :smiley:

For smaller models, the NengoDL optimizer tends to be unable to find graph optimizations to make (because the model is already so small). Larger models, however, enable the graph optimizers to make more potential optimizations, which is probably why the Darknet network has two different optimization paths (one with the synapses and one without them). For the most part, however, the graph optimizer is there to reduce the overall runtime of the network, and disabling them for your specific use case (where you are loading saved weights) shouldn’t be too much of a performance impact.

1 Like
8

Thanks so much for your helpful response :blush:. I understand more clearly about when convert CNN to SNN on Nengo :slight_smile:

9

Hi @xchoo, I relize the accuracy that increases significantly when I raise synapse, but if synapse reaches a certain threshold, the accuracy will be low. I think the reason for this may be the output less depends the information input. Alright?

10

The synapse value controls how much filtering (smoothing) is applied to each spike within the network. Raising the synapse value has the effect of smoothing out spike “noise” making the spike train more like a rate-based value (although, it’ll still be very spikey).

However, the because the filtering is performed by a low-pass filter (the default synapse is an exponential synapse, which low-pass filters the spike train), with larger synapse values the system response time decreases. The rise-time of a low-pass filter is roughly 1/3 $\times$ the synapse time constant, and the rise-time adds a delay to the propagation of signals through the network. So, larger synapse values lead to larger delays, and eventually, the signal takes so long to propagate through the network, and depending on how your network computes accuracy, the accuracy will go down if not enough time is given for the information to propagate through the entire network.

11

That means if the time is sufficient to propagate through the entire network, the accuracy will not go down. So I need to increase timestep (means n_steps), don’t I?

12

Hello @ntquyen2, yes. That’s what @xchoo is implying. You can have more details here: Synaptic smoothing, especially the fourth paragraph.

1 Like
13

That is correct yes. By increasing n_steps, you increase the time between each stimulus input, which in turn gives enough time for information to propagate through each layer of the network. Note, however, that in addition to increasing the synapse value, you’ll also want to play around with increasing the scale_firing_rates value (see the Nengo example that was linked by @zerone).

1 Like
14

So when does n_steps increase sufficiently to achieve the best accuracy for synapse? Example if n_steps=500, synapse = 0.02- → accuracy =7.5%, synapse =0.05 → accuracy=12.5%, synapse=0.1 → accuracy =8.75% but n_steps=1000,synapse =0.02 → accuracy =69.75% ,sysnapse =0.05 → accuracy=12.75%, synapse=0.1 → accuracy=81.25%

15

There is unfortunately no hard and fast rule regarding this. It’s very much application and network dependent, and you’ll need to experiment with various values (as well as values for scale_firing_rates) to find what gives you the best results. I suggest going through this Nengo tutorial to get a better idea of these parameters.

16

And I want to ask you about what is the equation of default synapse (exponential synapse)?

17

You can find information on all of the synapse models that come default with Nengo here: Nengo frontend API — Nengo 3.2.0.dev0 docs

The default synapse is nengo.Lowpass.

1 Like
18

Excellent :)) And how to plot the information input to propagate through the entire network when I change synapse?

19

If you want to plot the information as it propagates through the network, you’ll need to probe the outputs of each layer (search your original code for nengo.Probe for an example for how to do this), and then you’ll need to use matplotlib to plot the outputs.

20

conv0_probe = nengo.Probe(nengo_converter.layers[conv0][sample_neurons])
Is scaled_data = data[conv0_probe][ii] * scale_firing_rates ? I only change conv0_probe for convolution layer I want to show, don’t I?