Number of saved parameters error

General Discussion Frequently Asked Questions
1

Hi!

I converted a trainable network from tensorflow to Nengo, including a Batchnormalization layer, trained it and saved the parameters using save_params(). However when I tried to convert my tensorflow network with inference_only=True I got the params Error, I quickly found it was probably the Batchnormalization causing it.

My question now is, is it either supported to load the parameters from such a trainable network to a non-trainable which uses the native Nengo layers for BatchNormalization, or is it possible to convert a TensorNode wrapping a Batchnormalization layer to it’s Nengo native layer after conversion and training?

Cheers for all help or pointers!
Always reluctant to post, but couldn’t find anything yet in the forums regarding this.

2

Hello! can you post a minimal example script showing this?

3

Of course, sorry for not doing so in my initial post.

inputLayer = Input(shape=(size,1))

conv0 = Conv1D(512, kernel_size= 2, strides=2, activation=relu, use_bias=False,)(inputLayer)

batchLayer = BatchNormalization()(conv0)

conv1 = Conv1D(128, kernel_size= 2, strides=2, activation=relu, use_bias=False,)(batchLayer)

conv2 = Conv1D(4, kernel_size= 2, strides=2, activation=relu, use_bias=False,)(conv1)

flatLayer = Flatten()(conv2)

dense0 = Dense(8, kernel_regularizer=regularizers.L1(0.001), activation=relu, use_bias=False,)(flatLayer)

outputLayer = Dense(2, activation=softmax, use_bias=False,)(dense0)

model = Model(inputs=inputLayer, outputs=outputLayer)

converter = Converter(model)
nengo_output = converter.outputs[converter.model.output]

with Simulator(converter.net, seed=seed, minibatch_size=currNoBatch,) as sim:
	sim.compile(
		optimizer=optimizers.Adam(0.001),
		loss=losses.SparseCategoricalCrossentropy(from_logits=True),
		metrics=[metrics.sparse_categorical_crossentropy],
	)
	sim.fit(
		{converter.inputs[converter.model.input]: x_train},
		{converter.outputs[converter.model.output]: y_train},
		validation_data=(
			{converter.inputs[converter.model.input]: x_val},
			{converter.outputs[converter.model.output]: y_val},
		),
		epochs=50,
		verbose="auto",
	)

	# save the parameters to file
	sim.save_params("model")
	
converter = Converter(
	model,
	inference_only = True,
	swap_activations={tensorflow.nn.relu: nengo.loihi.neurons.LoihiSpikingRectifiedLinear},
	scale_firing_rates=100,
	synapse=0.02,
)

with Simulator(converter.net, seed=seed, minibatch_size=currNoBatch,) as sim:
	sim.compile(
		optimizer=optimizers.Adam(0.001),
		loss=losses.SparseCategoricalCrossentropy(from_logits=True),
		metrics=[metrics.sparse_categorical_crossentropy],
	)
	
	# Load previously trained model
	sim.load_params("model")
	sim.fit(
		{converter.inputs[converter.model.input]: x_train},
		{converter.outputs[converter.model.output]: y_train},
		validation_data=(
			{converter.inputs[converter.model.input]: x_val},
			{converter.outputs[converter.model.output]: y_val},
		),
		epochs=50,
		verbose="auto",
	)

I tried to be as concise as possible. Ideally the second sim.load_params would load the params despite having only_inference=True, or some way of converting the layer retrospectively and use a compatible network to first load the parameters.

this is the error I’m receiving SimulationError: Number of saved parameters in model (15) != number of variables in the model (7)

4

sorry, can you post the full script with all the imports?

5

Sorry for the late reply!

Of course, thanks for being so nice about my slip-ups.

import nengo_loihi
from numpy import shape as npshape, reshape as npreshape, tile as nptile, concatenate as npconcatenate, random as nprandom, argmax as npargmax, mean as npmean
from tensorflow import optimizers, losses, metrics, get_logger, test, random as tfrandom
from tensorflow.nn import softmax, relu
from tensorflow.keras import Model, regularizers
from tensorflow.keras.layers import Input, Conv1D, BatchNormalization, Dense, Flatten
from tensorflow.keras.callbacks import EarlyStopping
get_logger().setLevel('INFO')
from sklearn.model_selection import train_test_split
import mat73

from nengo_dl import Simulator, Converter


seed = 42
noBatch = 8
train_activations = {relu: nengo_loihi.neurons.LoihiSpikingRectifiedLinear()}

nprandom.seed(seed)
tfrandom.set_seed(seed)

matFile = mat73.loadmat(f"data.mat")

x_train, x_val, y_train, y_val = train_test_split(matFile['train'], matFile['labels'], test_size=0.2, random_state=seed)


#create the model
size = 2048

callback = EarlyStopping(monitor='accuracy', patience=3, min_delta=0.01, mode='auto')

inputLayer = Input(shape=(size,1))

conv0 = Conv1D(512, kernel_size= 2, strides=2, activation=relu, use_bias=False,)(inputLayer)

batchLayer = BatchNormalization()(conv0)

conv1 = Conv1D(128, kernel_size= 2, strides=2, activation=relu, use_bias=False,)(batchLayer)

conv2 = Conv1D(4, kernel_size= 2, strides=2, activation=relu, use_bias=False,)(conv1)

flatLayer = Flatten()(conv2)

dense0 = Dense(8, kernel_regularizer=regularizers.L1(0.001), activation=relu, use_bias=False,)(flatLayer)

outputLayer = Dense(2, activation=softmax, use_bias=False,)(dense0)

model = Model(inputs=inputLayer, outputs=outputLayer)

converter = Converter(model)
nengo_output = converter.outputs[converter.model.output]

with Simulator(converter.net, seed=seed, minibatch_size=noBatch,) as sim:
	sim.compile(
		optimizer=optimizers.Adam(0.001),
		loss=losses.SparseCategoricalCrossentropy(from_logits=True),
		metrics=[metrics.sparse_categorical_crossentropy],
	)
	sim.fit(
		{converter.inputs[converter.model.input]: x_train},
		{converter.outputs[converter.model.output]: y_train},
		validation_data=(
			{converter.inputs[converter.model.input]: x_val},
			{converter.outputs[converter.model.output]: y_val},
		),
		epochs=50,
		verbose="auto",
	)

	# save the parameters to file
	sim.save_params("model")

converter = Converter(
	model,
	inference_only = True,
	swap_activations={relu: nengo_loihi.neurons.LoihiSpikingRectifiedLinear()},
	scale_firing_rates=100,
	synapse=0.02,
)

with Simulator(converter.net, seed=seed, minibatch_size=noBatch,) as sim:
	sim.compile(
		optimizer=optimizers.Adam(0.001),
		loss=losses.SparseCategoricalCrossentropy(from_logits=True),
		metrics=[metrics.sparse_categorical_crossentropy],
	)

	# Load previously trained model
	sim.load_params("model")
	sim.fit(
		{converter.inputs[converter.model.input]: x_train},
		{converter.outputs[converter.model.output]: y_train},
		validation_data=(
			{converter.inputs[converter.model.input]: x_val},
			{converter.outputs[converter.model.output]: y_val},
		),
		epochs=50,
		verbose="auto",
	)

Except for exchanging the file loading with a more generic version I tested this and got:
nengo.exceptions.SimulationError: Number of saved parameters in model (9) != number of variables in the model (6)