First off, I want thank you for your prompt response.
INFO:DRV: SLURM is being run in background
INFO:DRV: Connecting to 10.212.98.108:41839
INFO:DRV: Host server up…Done 0.24s
INFO:DRV: Encoding axons/synapses…Done 39.34s
INFO:DRV: Compiling Embedded snips…Done 0.67s
INFO:DRV: Compiling MPDS Registers…Done 0.29ms
INFO:HST: Args chip=0 cpu=0 /homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/driver/compilers/…/…/…/temp/1638411285.3001146/launcher_chip0_lmt0.bin --chips=2 --remote-relay=1
INFO:HST: Args chip=1 cpu=0 /homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/driver/compilers/…/…/…/temp/1638411285.3001146/launcher_chip1_lmt0.bin --chips=2 --remote-relay=1
INFO:HST: Nx…
INFO:DRV: Booting up…Done 1.18s
INFO:DRV: Encoding probes…Done 0.87ms
running loihi sim
INFO:DRV: Transferring probes…Done 0.01s
INFO:DRV: Configuring registers…Done 6.13s
INFO:DRV: Transferring spikes…Done 95.33s
INFO:HST: srun: Force Terminated job 1320729
INFO:HST: srun: Job step aborted: Waiting up to 32 seconds for job step to finish.
INFO:HST: slurmstepd: error: *** STEP 1320729.0 ON ncl-ext-ghrd-04 CANCELLED AT 2021-12-01T20:14:05 DUE TO TIME LIMIT ***
INFO:HST: srun: error: ncl-ext-ghrd-04: task 0: Terminated
INFO:DRV: Executing…Error 6953.38s
INFO:DRV: Executor: 400 timesteps…Error 7054.86s
Traceback (most recent call last):
File “/homes/mjurado3/workspace/neuromorphics/loihi_conversion.py”, line 72, in test_predict_timesteps
loihi_sim.run(batch_size * pres_time)
File “/homes/mjurado3/nengo-loihi/nengo_loihi/simulator.py”, line 330, in run
self.run_steps(steps)
File “/homes/mjurado3/nengo-loihi/nengo_loihi/simulator.py”, line 343, in run_steps
self._runner.run_steps(steps)
File “/homes/mjurado3/nengo-loihi/nengo_loihi/simulator.py”, line 497, in loihi_precomputed_host_pre_and_host
self.loihi.run_steps(steps, blocking=True)
File “/homes/mjurado3/nengo-loihi/nengo_loihi/hardware/interface.py”, line 253, in run_steps
d_get(self.nxsdk_board, b"cnVu")(steps, **{d(b"YVN5bmM="): not blocking})
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/arch/base/nxboard.py”, line 283, in run
self._run(
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/arch/base/nxboard.py”, line 257, in _run
self.executor.start(numSteps, aSync)
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/driver/executor.py”, line 84, in start
self.finish()
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/driver/executor.py”, line 121, in finish
self._wait()
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/driver/executor.py”, line 128, in _wait
self._executor_service.waitExecution(empty)
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/grpc/_channel.py”, line 946, in call
return _end_unary_response_blocking(state, call, False, None)
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/grpc/_channel.py”, line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = “Socket closed”
debug_error_string = “{“created”:”@1638418445.822006232",“description”:“Error received from peer ipv4:10.212.98.108:41839”,“file”:“src/core/lib/surface/call.cc”,“file_line”:1069,“grpc_message”:“Socket closed”,“grpc_status”:14}"
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/homes/mjurado3/workspace/neuromorphics/loihi_conversion.py”, line 256, in
scores, layer_output = test_predict_timesteps(net, pres_time, nengo_outputs, test_images, test_labels, batch_size=batch_size)
File “/homes/mjurado3/workspace/neuromorphics/loihi_conversion.py”, line 93, in test_predict_timesteps
conv_data.append(loihi_sim.data[nengo_outputs[-2]][j:j+timesteps].astype(np.int))
File “/homes/mjurado3/nengo-loihi/nengo_loihi/simulator.py”, line 217, in exit
sim.exit(exc_type, exc_value, traceback)
File “/homes/mjurado3/nengo-loihi/nengo_loihi/hardware/interface.py”, line 129, in exit
self.close()
File “/homes/mjurado3/nengo-loihi/nengo_loihi/hardware/interface.py”, line 161, in close
d_func(self.nxsdk_board, b"ZGlzY29ubmVjdA==")
File “/homes/mjurado3/nengo-loihi/nengo_loihi/nxsdk_obfuscation.py”, line 77, in d_func
return func(**kwargs)
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/arch/base/nxboard.py”, line 345, in disconnect
self.executor.stop()
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/driver/executor.py”, line 97, in stop
self._notifyListeners(ExecutionEventEnum.ON_STOP)
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/driver/executor.py”, line 159, in _notifyListeners
listener.onStop()
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/nxsdk/driver/listeners/lakemont_orchestrator.py”, line 41, in onStop
self.stopNxDriver(empty)
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/grpc/_channel.py”, line 946, in call
return _end_unary_response_blocking(state, call, False, None)
File “/homes/mjurado3/miniconda3/envs/loihi_vishal/lib/python3.9/site-packages/grpc/_channel.py”, line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = “failed to connect to all addresses”
debug_error_string = “{“created”:”@1638418445.846656905",“description”:“Failed to pick subchannel”,“file”:“src/core/ext/filters/client_channel/client_channel.cc”,“file_line”:3158,“referenced_errors”:[{“created”:"@1638418445.846655545",“description”:“failed to connect to all addresses”,“file”:“src/core/lib/transport/error_utils.cc”,“file_line”:147,“grpc_status”:14}]}"