Some problems on the depthwise_conv2D into nengo


#1

Excuse me, I want to use depthwise_conv2d with nengo:

with nengo.Network() as net:

net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
neuron_type = nengo.LIF(amplitude=0.01)

max_norm_reg=max_norm_regularizer(threshold=1.0,axes=3)
Weight_depthwise=tf.get_variable('Weight_depthwise',[Chans,1,F1,D],
                                                      initializer=tf.glorot_uniform_initializer(),regularizer=max_norm_reg)

nengo_dl.configure_settings(trainable=True)

    
inp = nengo.Node([0] * Chans * Sample*1)


    
x = nengo_dl.tensor_layer(
        inp, tf.layers.conv2d, shape_in=(Chans, Sample,1), filters=4,
        kernel_size=(1,kernLength),strides=(1,1),padding='same')

  
   
x=nengo_dl.tensor_layer(x,tf.nn.depthwise_conv2d,shape_in=(Chans,1,Sample,4),filter=Weight_depthwise,strides=(1,1,1,1),padding='valid')

x = nengo_dl.tensor_layer(x, neuron_type)

x = nengo_dl.tensor_layer(
                x, tf.layers.average_pooling2d, shape_in=(F1*D,1 ,Sample),
                pool_size=(1,4), strides=(1,4))
    
x=nengo_dl.tensor_layer(x,tf.layers.separable_conv2d,shape_in=(F1*D,1,Sample//4),filters=F2,kernel_size=(1,16),strides=(1,1),padding='same')

x = nengo_dl.tensor_layer(x, neuron_type)

x = nengo_dl.tensor_layer(x, tf.layers.average_pooling2d, shape_in=(F2,1 ,Sample//32),pool_size=(1,8), strides=(1,8))
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=2)
out_p = nengo.Probe(x)
out_p_filt = nengo.Probe(x, synapse=0.1)

But it cause error as fellows:
ValidationError: TensorNode.tensor_func: Calling TensorNode function with arguments (<tf.Tensor ‘Const:0’ shape=() dtype=float32>, <tf.Tensor ‘zeros:0’ shape=(1, 65536) dtype=float32>) produced an error: Tensor(“Weight_depthwise:0”, shape=(64, 1, 4, 2), dtype=float32_ref) must be from the same graph as Tensor(“Reshape:0”, shape=(1, 64, 1, 256, 4), dtype=float32).


#2

The problem is that the weights are being created outside the NengoDL scope:

Weight_depthwise=tf.get_variable('Weight_depthwise', [Chans,1,F1,D],
                                 initializer=tf.glorot_uniform_initializer(),
                                 regularizer=max_norm_reg)

The weights need to be created within the same graph in which the rest of the network is built. The simplest way to do this is to build it into your layer function.

def depthwise_conv2d(x, **kwargs):
    Weight_depthwise=tf.get_variable('Weight_depthwise', [Chans,1,F1,D],
                                     initializer=tf.glorot_uniform_initializer(),
                                     regularizer=max_norm_reg)
    return tf.nn.depthwise_conv2d(x, filter=Weight_depthwise, **kwargs)

and then use that new function (which combines the weight creation and application) in the tensor_layer. I.e. in this line

x=nengo_dl.tensor_layer(x,tf.nn.depthwise_conv2d,shape_in=(Chans,1,Sample,4),filter=Weight_depthwise,strides=(1,1,1,1),padding='valid')

replace tf.nn.depthwise_conv2d with our new depthwise_conv2d function.

Note that this will create a new kernel every time the function is called. This will work OK as long as you are only building the model with unroll_simulation=1. However, the more general and robust way to solve this problem is to use the TensorNode pre_build functionality (see https://www.nengo.ai/nengo-dl/tensor-node.html). The idea is that we create a callable class for our tensorflow layer, and do the weight initialization inside the pre_build function so that it will be properly built into the graph. Something like

class DepthwiseConv2D:
    def pre_build(shape_in, shape_out):
        self.Weight_depthwise=tf.get_variable('Weight_depthwise',[Chans,1,F1,D],
                                              initializer=tf.glorot_uniform_initializer(),regularizer=max_norm_reg))
    def __call__(self, t, x):
        x = tf.reshape(x, (Chans, 1, Sample, 4))
        return tf.nn.depthwise_conv2d(x, filter=self.Weight_depthwise, strides=(1, 1, 1, 1), padding="valid")

and then replace

x=nengo_dl.tensor_layer(x,tf.nn.depthwise_conv2d,shape_in=(Chans,1,Sample,4),filter=Weight_depthwise,strides=(1,1,1,1),padding='valid')

with our new TensorNode

y = nengo_dl.TensorNode(DepthwiseConv2d())
nengo.Connection(x, y)
x = y

#3

Sorry, it also has errors as fellow:

def depthwise_conv2d(x, **kwargs):

Weight_depthwise=tf.get_variable(‘Weight_depthwise’, [Chans,1,F1,D],
initializer=tf.glorot_uniform_initializer(),
regularizer=max_norm_reg)
return tf.nn.depthwise_conv2d(x, filter=Weight_depthwise, **kwargs)

with nengo.Network as net:

        x=nengo_dl.tensor_layer(x,depthwise_conv2d,shape_in=(Chans,1,Sample,4),strides=(1,1,1,1),padding='valid')

#4

What is the error?


#6

def depthwise_conv(x,**kwargs):
max_norm_reg = max_norm_regularizer(threshold=1.0, axes=3)
Weight_depthwise = tf.get_variable(‘Weight_depthwise’, [Chans, 1, F1, D], # [filter_height, filter_width, in_channels, channel_multiplier].
initializer=tf.glorot_uniform_initializer(), regularizer=max_norm_regularizer)
return tf.nn.depthwise_conv2d(x,filter=Weight_depthwise,**kwargs)
with nengo.Network() as net:

net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
neuron_type = nengo.LIF(amplitude=0.01)

nengo_dl.configure_settings(trainable=True)

inp = nengo.Node([0] * Chans * Sample*1)
 
x = nengo_dl.tensor_layer(
    inp, tf.layers.conv2d, shape_in=(Chans, Sample, 1), filters=4,
    kernel_size=(1,kernLength),strides=(1,1),padding='same')
x=  nengo_dl.tensor_layer(x,depthwise_conv,shape_in=(Chans,1,Sample,4),strides=(1,1,1,1),padding='valid')   
x = nengo_dl.tensor_layer(x, neuron_type)

The error as fellows:
nengo.exceptions.ValidationError: TensorNode.tensor_func: Calling TensorNode function with arguments (<tf.Tensor ‘Const:0’ shape=() dtype=float32>, <tf.Tensor ‘zeros:0’ shape=(1, 248000) dtype=float32>) produced an error:
‘function’ object has no attribute ‘name’


#7

It looks like there is a problem with your regularizer function (TensorFlow expects it to return a Tensor, but instead it is returning something else).