Convert Darknet 19 to SNN

I don’t know why reason is when I convert Darknet 19 to SNN, I have an error

**SimulationError: Number of saved parameters in keras_to_snn_params (41) != number of variables in the model (44)**

if synapse =0.05 or non zeros and no error for synapse=None even I turn off optimizer "

with converter.net:
    nengo_dl.configure_settings(simplifications=[])


inp=tf.keras.layers.Input(shape=(28,28,1))

conv0 = tf.keras.layers.Conv2D(filters=8,kernel_size=3,padding="same", activation=tf.nn.relu)(inp)

pool0 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv0)

conv1 = tf.keras.layers.Conv2D(filters=16,kernel_size=3,padding="same", activation=tf.nn.relu)(pool0)

pool1 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv1)

conv2 = tf.keras.layers.Conv2D(filters=32,kernel_size=3,padding="same", activation=tf.nn.relu)(pool1)

conv3 = tf.keras.layers.Conv2D(filters=16,kernel_size=1,padding="same", activation=tf.nn.relu)(conv2)

conv4 = tf.keras.layers.Conv2D(filters=32,kernel_size=3,padding="same", activation=tf.nn.relu)(conv3)

pool2 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv4)

conv5 = tf.keras.layers.Conv2D(filters=64,kernel_size=3,padding="same", activation=tf.nn.relu)(pool2)

conv6 = tf.keras.layers.Conv2D(filters=32,kernel_size=1,padding="same", activation=tf.nn.relu)(conv5)

conv7 = tf.keras.layers.Conv2D(filters=64,kernel_size=3,padding="same", activation=tf.nn.relu)(conv6)

pool3 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2)(conv7)

conv8 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(pool3)

conv9 = tf.keras.layers.Conv2D(filters=64,kernel_size=1,padding="same", activation=tf.nn.relu)(conv8)

conv10 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(conv9)

conv11 = tf.keras.layers.Conv2D(filters=64,kernel_size=1,padding="same", activation=tf.nn.relu)(conv10)

conv12 = tf.keras.layers.Conv2D(filters=128,kernel_size=3,padding="same", activation=tf.nn.relu)(conv11)

# pool4 = tf.keras.layers.AveragePooling2D(pool_size=2,strides=2,padding='same')(conv12)

conv13 = tf.keras.layers.Conv2D(filters=256,kernel_size=3,padding="same", activation=tf.nn.relu)(conv12)

conv14 = tf.keras.layers.Conv2D(filters=128,kernel_size=1,padding="same", activation=tf.nn.relu)(conv13)

conv15 = tf.keras.layers.Conv2D(filters=356,kernel_size=3,padding="same", activation=tf.nn.relu)(conv14)

conv16 = tf.keras.layers.Conv2D(filters=128,kernel_size=1,padding="same", activation=tf.nn.relu)(conv15)

conv17 = tf.keras.layers.Conv2D(filters=256,kernel_size=3,padding="same", activation=tf.nn.relu)(conv16)

conv18 = tf.keras.layers.Conv2D(filters=250,kernel_size=1,padding="same", activation=tf.nn.relu)(conv16)

pool5=tf.keras.layers.GlobalAveragePooling2D()(conv18)

flatten = tf.keras.layers.Flatten()(pool5)

dense = tf.keras.layers.Dense(units=10)(flatten)

model = tf.keras.Model(inputs = inp, outputs = dense)

model.summary()

converter = nengo_dl.Converter(model)

do_training = True
if do_training:
    with converter.net:
        nengo_dl.configure_settings(simplifications=[])
    with nengo_dl.Simulator(converter.net, minibatch_size=200) as sim:
        # run training
        sim.compile(
            optimizer=tf.optimizers.Adam(0.001),
            loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
            metrics=[tf.metrics.sparse_categorical_accuracy],
        )
        sim.fit(
            {converter.inputs[inp]: train_images},
            {converter.outputs[dense]: train_labels},
            validation_data=(
                {converter.inputs[inp]: test_images},
                {converter.outputs[dense]: test_labels},
            ),
            epochs=2,
        )
       

        # save the parameters to file
        sim.save_params("./keras_to_snn_params")
else:
    # download pretrained weights
    urlretrieve(
        "https://drive.google.com/uc?export=download&"
        "id=1lBkR968AQo__t8sMMeDYGTQpBJZIs2_T",
        "keras_to_snn_params.npz",
    )

def run_network(
    activation,
    params_file="keras_to_snn_params",
    n_steps=120,
    scale_firing_rates=1,
    synapse=None,
    n_test=400,
):
    # convert the keras model to a nengo network
    nengo_converter = nengo_dl.Converter(
        model,
        swap_activations={tf.nn.relu: activation},
        scale_firing_rates=scale_firing_rates,
        synapse=synapse,
    )

    # get input/output objects
    nengo_input = nengo_converter.inputs[inp]
    nengo_output = nengo_converter.outputs[dense]
    with nengo_converter.net:
        nengo_dl.configure_settings(simplifications=[])
    

    # add a probe to the first convolutional layer to record activity.
    # we'll only record from a subset of neurons, to save memory.
    sample_neurons = np.linspace(
        0,
        np.prod(conv0.shape[1:]),
        1000,
        endpoint=False,
        dtype=np.int32,
    )
    with nengo_converter.net:
        conv0_probe = nengo.Probe(nengo_converter.layers[conv0][sample_neurons])
        
   
    # repeat inputs for some number of timesteps
    tiled_test_images = np.tile(test_images[:n_test], (1, n_steps, 1))

    # set some options to speed up simulation
    with nengo_converter.net:
        nengo_dl.configure_settings(stateful=False)

    # build network, load in trained weights, run inference on test images
    with nengo_dl.Simulator(
        nengo_converter.net, minibatch_size=10, progress_bar=False
    ) as nengo_sim:
        params = list(nengo_sim.keras_model.weights)
        print(len(params))
        nengo_sim.load_params(params_file)
        data = nengo_sim.predict({nengo_input: tiled_test_images})

    # compute accuracy on test data, using output of network on
    # last timestep
    predictions = np.argmax(data[nengo_output][:, -1], axis=-1)
    accuracy = (predictions == test_labels[:n_test, 0, 0]).mean()
    print(f"Test accuracy: {100 * accuracy:.2f}%")

    # plot the results
    for ii in range(3):
        plt.figure(figsize=(12, 4))

        plt.subplot(1, 3, 1)
        plt.title("Input image")
        plt.imshow(test_images[ii, 0].reshape((28, 28)), cmap="gray")
        plt.axis("off")

        plt.subplot(1, 3, 2)
        scaled_data = data[conv0_probe][ii] * scale_firing_rates
        if isinstance(activation, nengo.SpikingRectifiedLinear):
            scaled_data *= 0.001
            rates = np.sum(scaled_data, axis=0) / (n_steps * nengo_sim.dt)
            plt.ylabel("Number of spikes")
        else:
            rates = scaled_data
            plt.ylabel("Firing rates (Hz)")
        plt.xlabel("Timestep")
        plt.title(
            f"Neural activities (conv0 mean={rates.mean():.1f} Hz, "
            f"max={rates.max():.1f} Hz)"
        )
        plt.plot(scaled_data)

        plt.subplot(1, 3, 3)
        plt.title("Output predictions")
        plt.plot(tf.nn.softmax(data[nengo_output][ii]))
        plt.legend([str(j) for j in range(10)], loc="upper left")
        plt.xlabel("Timestep")
        plt.ylabel("Probability")

        plt.tight_layout()
    print("Loaded pretrained weights")

for s in [0.1, 0.02, 0.07]:
    print(f"Synapse={s:.3f}")
    run_network(
        activation=nengo.SpikingRectifiedLinear(),
        n_steps=10,
        synapse=s,
    )
    plt.show()

Hi @ntquyen2, and welcome to the Nengo forums!

With your code in particular, it looks like the NengoDL graph optimizer has come up with two different solutions (when the synapses are included vs when they are not). To get around this issue, you’ll want to disable the graph optimizer entirely.

To disable the graph optimizer, configure the NengoDL settings to use planner=nengo_dl.graph_optimizer.noop_planner instead of simplications=[].

As a quick explanation, simplications=[] still applies the operator merging, but disables simplifications on the transformation functions. As comparision, planner=noop_planner disables the graph optimizer entirely.

1 Like

Thanks @xchoo for yours answer, I changed planner=nengo_dl.graph_optimizer.noop_plannerinstead ofsimplications=[] but I had error ValueError: Cannot assign to variable TensorGraph/base_params/trainable_float32_3_3_32_16:0 due to variable shape (3, 3, 32, 16) and value shape (3, 3, 16, 32) are incompatible and I know the reason for it.

do_training = True
if do_training:
    # with converter.net:
    #     nengo_dl.configure_settings(simplifications=[])

    with converter.net:
          nengo_dl.configure_settings(planner=noop_planner)

and

def run_network(
    activation,
    params_file="keras_to_snn_params",
    n_steps=120,
    scale_firing_rates=5,
    synapse=None,
    n_test=400,
):
    # convert the keras model to a nengo network
    nengo_converter = nengo_dl.Converter(
        model,
        swap_activations={tf.nn.relu: activation},
        scale_firing_rates=scale_firing_rates,
        synapse=synapse,
    )

    # get input/output objects
    nengo_input = nengo_converter.inputs[inp]
    nengo_output = nengo_converter.outputs[dense]
    with nengo_converter.net:
        nengo_dl.configure_settings(planner=noop_planner)

I am unable to reproduce this error using the code you posted in your first post. If you can, can you please include your entire script in your reply so that I can run it?

Here is the script I used to test your code, and it seems to run fine on my machine:
test_tf_convert_synapse.py (8.1 KB)
Perhaps you can compare the code to what you have and see if you can spot the difference causing your error. :slight_smile:

I’m really grateful for your help @xchoo . I fixed my code and I still want to say " Thank you so much <3"

Hi @xchoo, I want to ask you why reason is when I convert Darknet 19 to SNN, I have UserWarning: average_pooling2d_4.padding has value same != valid, which is not supported. Falling back to TensorNode.
% (error_msg + ". " if error_msg else “”) if I don’t comment pool4 in my above code although I still run fine my code. Moreover, I convert small model such as Alexnet to SNN, I don’t have to turn off optimizer whereas Darknet have to it. May be Darknet is a large model??? Can you help me, please?

The NengoDL converter attempts to convert all of the TensorFlow objects into Nengo-native objects (e.g., tf.keras.layers.dense gets converted into a nengo.Ensemble). This warning is just to let you know that the specific configuration of the average pooling layer (i.e., with padding=same) doesn’t have a default Nengo object mapping, and so, NengoDL is using a generic TensorNode to implement that specific layer.

If you don’t need the Nengo specific objects, then having the TensorNode should be fine for your use. Otherwise, you’ll need to look at implementing a Nengo specific translation for the tf.keras.layers.AveragePooling2D(..., padding="same") layer. You can read up on how to do this here! :smiley:

For smaller models, the NengoDL optimizer tends to be unable to find graph optimizations to make (because the model is already so small). Larger models, however, enable the graph optimizers to make more potential optimizations, which is probably why the Darknet network has two different optimization paths (one with the synapses and one without them). For the most part, however, the graph optimizer is there to reduce the overall runtime of the network, and disabling them for your specific use case (where you are loading saved weights) shouldn’t be too much of a performance impact.

1 Like

Thanks so much for your helpful response :blush:. I understand more clearly about when convert CNN to SNN on Nengo :slight_smile:

Hi @xchoo, I relize the accuracy that increases significantly when I raise synapse, but if synapse reaches a certain threshold, the accuracy will be low. I think the reason for this may be the output less depends the information input. Alright?