It seems that the problem was not those things... TrueCommand reports it offline again after just a few hours.
The last thing I can see in the logs (middleware, since messages has nothing for a few hours before):
Code:
[2020/10/23 12:15:46] (WARNING) application.call_method():173 - Exception while calling sharing.webdav.query(*[])
concurrent.futures.process._RemoteTraceback:
"""
Traceback (most recent call last):
File "/usr/local/lib/python3.8/concurrent/futures/process.py", line 239, in _process_worker
r = call_item.fn(*call_item.args, **call_item.kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/worker.py", line 91, in main_worker
res = MIDDLEWARE._run(*call_args)
File "/usr/local/lib/python3.8/site-packages/middlewared/worker.py", line 45, in _run
return self._call(name, serviceobj, methodobj, args, job=job)
File "/usr/local/lib/python3.8/site-packages/middlewared/worker.py", line 33, in _call
with Client('ws+unix:///var/run/middlewared-internal.sock', py_exceptions=True) as c:
File "/usr/local/lib/python3.8/site-packages/middlewared/client/client.py", line 281, in __init__
self._ws.connect()
File "/usr/local/lib/python3.8/site-packages/middlewared/client/client.py", line 124, in connect
rv = super(WSClient, self).connect()
File "/usr/local/lib/python3.8/site-packages/ws4py/client/__init__.py", line 223, in connect
bytes = self.sock.recv(128)
socket.timeout: timed out
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 137, in call_method
result = await self.middleware._call(message['method'], serviceobj, methodobj, params, app=self,
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/service.py", line 442, in query
result = await self.middleware.call(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1233, in call
return await self._call(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/datastore/read.py", line 163, in query
result = await self._queryset_serialize(
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/datastore/read.py", line 207, in _queryset_serialize
extend_context_value = await self.middleware.call(extend_context, extra_options)
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1233, in call
return await self._call(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/service.py", line 577, in sharing_task_extend_context
'locked_datasets': await self.middleware.call('zfs.dataset.locked_datasets'),
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1233, in call
return await self._call(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1199, in _call
return await self._call_worker(name, *prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1205, in _call_worker
return await self.run_in_proc(main_worker, name, args, job)
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1132, in run_in_proc
return await self.run_in_executor(self.__procpool, method, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1106, in run_in_executor
return await loop.run_in_executor(pool, functools.partial(method, *args, **kwargs))
socket.timeout: timed out
[2020/10/23 12:15:46] (DEBUG) application.__crash_reporting():181 - [Crash Reporting] is disabled using sentinel file.
[2020/10/23 12:15:46] (DEBUG) application.__crash_reporting():181 - [Crash Reporting] is disabled using sentinel file.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:46] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:47] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:47] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:47] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:47] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:47] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:48] (DEBUG) urllib3.connectionpool._new_conn():955 - Starting new HTTPS connection (1): update-master.ixsystems.com:443
[2020/10/23 12:15:50] (DEBUG) urllib3.connectionpool._make_request():428 - https://update-master.ixsystems.com:443 "GET /TrueNAS/trains_redir.json HTTP/1.1" 200 233
[2020/10/23 12:15:50] (DEBUG) freenasOS.Configuration.TryGetNetworkFile():606 - TryGetNetworkFile(['https://update.ixsystems.com/TrueNAS/trains.txt', 'https://update-master.ixsystems.com/TrueNAS/trains.txt'])
[2020/10/23 12:15:50] (DEBUG) urllib3.connectionpool._new_conn():955 - Starting new HTTPS connection (1): update.ixsystems.com:443
[2020/10/23 12:15:50] (WARNING) asyncio.write():901 - socket.send() raised exception.
[2020/10/23 12:15:51] (DEBUG) urllib3.connectionpool._make_request():428 - https://update.ixsystems.com:443 "GET /TrueNAS/trains.txt HTTP/1.1" 301 169
[2020/10/23 12:15:51] (DEBUG) urllib3.connectionpool._new_conn():955 - Starting new HTTPS connection (1): update.freenas.org:443
[2020/10/23 12:15:52] (DEBUG) urllib3.connectionpool._make_request():428 - https://update.freenas.org:443 "GET /TrueNAS/trains.txt HTTP/1.1" 200 377
[2020/10/23 12:15:52] (DEBUG) freenasOS.Configuration.TryGetNetworkFile():745 - TryGetNetworkFile(['https://update.ixsystems.com/TrueNAS/trains.txt', 'https://update-master.ixsystems.com/TrueNAS/trains.txt']): Read 377 bytes total
[2020/10/23 12:15:52] (DEBUG) freenasOS.Configuration.TryGetNetworkFile():606 - TryGetNetworkFile(['https://update-master.ixsystems.com/TrueNAS/TrueNAS-12.0-STABLE/LATEST'])
[2020/10/23 12:15:52] (DEBUG) urllib3.connectionpool._new_conn():955 - Starting new HTTPS connection (1): update-master.ixsystems.com:443
[2020/10/23 12:15:52] (DEBUG) urllib3.connectionpool._make_request():428 - https://update-master.ixsystems.com:443 "GET /TrueNAS/TrueNAS-12.0-STABLE/LATEST HTTP/1.1" 200 1622
[2020/10/23 12:15:52] (DEBUG) freenasOS.Configuration.TryGetNetworkFile():745 - TryGetNetworkFile(['https://update-master.ixsystems.com/TrueNAS/TrueNAS-12.0-STABLE/LATEST']): Read 1622 bytes total
[2020/10/23 12:15:52] (DEBUG) freenasOS.Configuration.TryGetNetworkFile():606 - TryGetNetworkFile(['https://update-master.ixsystems.com/updates/ix_crl.pem'])
[2020/10/23 12:15:52] (DEBUG) urllib3.connectionpool._new_conn():955 - Starting new HTTPS connection (1): update-master.ixsystems.com:443
[2020/10/23 12:15:53] (DEBUG) urllib3.connectionpool._make_request():428 - https://update-master.ixsystems.com:443 "GET /updates/ix_crl.pem HTTP/1.1" 200 1028
[2020/10/23 12:15:53] (DEBUG) freenasOS.Configuration.CheckFreeSpace():77 - CheckFreeSpace(path=/tmp/tmprrfcp133.pem, pool=None, required=1028)
[2020/10/23 12:15:53] (DEBUG) freenasOS.Configuration.TryGetNetworkFile():745 - TryGetNetworkFile(['https://update-master.ixsystems.com/updates/ix_crl.pem']): Read 1028 bytes total
[2020/10/23 12:15:53] (DEBUG) freenasOS.Configuration.TryGetNetworkFile():606 - TryGetNetworkFile(['https://update.ixsystems.com/TrueNAS/Validators/ValidateUpdate-mg3ef525.txt', 'https://update-master.ixsystems.com/TrueNAS/Validators/ValidateUpdate-mg3ef525.txt'])
[2020/10/23 12:15:53] (DEBUG) urllib3.connectionpool._new_conn():955 - Starting new HTTPS connection (1): update.ixsystems.com:443
[2020/10/23 12:15:54] (DEBUG) urllib3.connectionpool._make_request():428 - https://update.ixsystems.com:443 "GET /TrueNAS/Validators/ValidateUpdate-mg3ef525.txt HTTP/1.1" 301 169
[2020/10/23 12:15:54] (DEBUG) urllib3.connectionpool._new_conn():955 - Starting new HTTPS connection (1): update.freenas.org:443
[2020/10/23 12:15:54] (DEBUG) urllib3.connectionpool._make_request():428 - https://update.freenas.org:443 "GET /TrueNAS/Validators/ValidateUpdate-mg3ef525.txt HTTP/1.1" 200 185
[2020/10/23 12:15:54] (DEBUG) freenasOS.Configuration.CheckFreeSpace():77 - CheckFreeSpace(path=/tmp/tmpxf13q4zj, pool=None, required=185)
[2020/10/23 12:15:54] (DEBUG) freenasOS.Configuration.TryGetNetworkFile():745 - TryGetNetworkFile(['https://update.ixsystems.com/TrueNAS/Validators/ValidateUpdate-mg3ef525.txt', 'https://update-master.ixsystems.com/TrueNAS/Validators/ValidateUpdate-mg3ef525.txt']): Read 185 bytes total
[2020/10/23 12:17:25] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/system.py", line 549, in info
product_version = (await(await Popen(
File "/usr/local/lib/python3.8/asyncio/subprocess.py", line 236, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1630, in subprocess_exec
transport = await self._make_subprocess_transport(
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 197, in _make_subprocess_transport
transp = _UnixSubprocessTransport(self, protocol, args, shell,
File "/usr/local/lib/python3.8/asyncio/base_subprocess.py", line 36, in __init__
self._start(args=args, shell=shell, stdin=stdin, stdout=stdout,
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 789, in _start
self._proc = subprocess.Popen(
File "/usr/local/lib/python3.8/subprocess.py", line 854, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/local/lib/python3.8/subprocess.py", line 1664, in _execute_child
os.close(errpipe_read)
[2020/10/23 12:17:51] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/system.py", line 554, in info
manufacturer = (await(await Popen(
File "/usr/local/lib/python3.8/asyncio/subprocess.py", line 236, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1630, in subprocess_exec
transport = await self._make_subprocess_transport(
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 197, in _make_subprocess_transport
transp = _UnixSubprocessTransport(self, protocol, args, shell,
File "/usr/local/lib/python3.8/asyncio/base_subprocess.py", line 36, in __init__
self._start(args=args, shell=shell, stdin=stdin, stdout=stdout,
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 789, in _start
self._proc = subprocess.Popen(
File "/usr/local/lib/python3.8/subprocess.py", line 854, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/local/lib/python3.8/subprocess.py", line 1637, in _execute_child
self.pid = _posixsubprocess.fork_exec(
[2020/10/23 12:22:28] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/service_/services/base.py", line 72, in get_state
return await self._get_state_freebsd()
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/service_/services/base_freebsd.py", line 27, in _get_state_freebsd
proc = await run(*cmd, check=False, encoding="utf-8")
File "/usr/local/lib/python3.8/site-packages/middlewared/utils/__init__.py", line 53, in run
proc = await asyncio.create_subprocess_exec(*args, **kwargs)
File "/usr/local/lib/python3.8/asyncio/subprocess.py", line 236, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1630, in subprocess_exec
transport = await self._make_subprocess_transport(
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 202, in _make_subprocess_transport
watcher.add_child_handler(transp.get_pid(),
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 1299, in add_child_handler
thread.start()
File "/usr/local/lib/python3.8/threading.py", line 857, in start
self._started.wait()
File "/usr/local/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
File "/usr/local/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
[2020/10/23 12:24:51] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/service_/services/base.py", line 72, in get_state
return await self._get_state_freebsd()
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/service_/services/base_freebsd.py", line 27, in _get_state_freebsd
proc = await run(*cmd, check=False, encoding="utf-8")
File "/usr/local/lib/python3.8/site-packages/middlewared/utils/__init__.py", line 53, in run
proc = await asyncio.create_subprocess_exec(*args, **kwargs)
File "/usr/local/lib/python3.8/asyncio/subprocess.py", line 236, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1630, in subprocess_exec
transport = await self._make_subprocess_transport(
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 202, in _make_subprocess_transport
watcher.add_child_handler(transp.get_pid(),
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 1299, in add_child_handler
thread.start()
File "/usr/local/lib/python3.8/threading.py", line 857, in start
self._started.wait()
File "/usr/local/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
File "/usr/local/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
Code:
[2020/10/23 12:03:14] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/system.py", line 544, in info
product = (await(await Popen(
File "/usr/local/lib/python3.8/asyncio/subprocess.py", line 236, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1630, in subprocess_exec
transport = await self._make_subprocess_transport(
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 197, in _make_subprocess_transport
transp = _UnixSubprocessTransport(self, protocol, args, shell,
File "/usr/local/lib/python3.8/asyncio/base_subprocess.py", line 36, in __init__
self._start(args=args, shell=shell, stdin=stdin, stdout=stdout,
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 789, in _start
self._proc = subprocess.Popen(
File "/usr/local/lib/python3.8/subprocess.py", line 854, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/local/lib/python3.8/subprocess.py", line 1658, in _execute_child
part = os.read(errpipe_read, 50000)
[2020/10/23 12:06:59] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/network.py", line 2233, in query
resolvconf = (await run('resolvconf', '-l')).stdout.decode()
File "/usr/local/lib/python3.8/site-packages/middlewared/utils/__init__.py", line 53, in run
proc = await asyncio.create_subprocess_exec(*args, **kwargs)
File "/usr/local/lib/python3.8/asyncio/subprocess.py", line 236, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1630, in subprocess_exec
transport = await self._make_subprocess_transport(
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 197, in _make_subprocess_transport
transp = _UnixSubprocessTransport(self, protocol, args, shell,
File "/usr/local/lib/python3.8/asyncio/base_subprocess.py", line 36, in __init__
self._start(args=args, shell=shell, stdin=stdin, stdout=stdout,
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 789, in _start
self._proc = subprocess.Popen(
File "/usr/local/lib/python3.8/subprocess.py", line 854, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/local/lib/python3.8/subprocess.py", line 1658, in _execute_child
part = os.read(errpipe_read, 50000)
[2020/10/23 12:09:06] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/system.py", line 567, in info
ecc_memory = bool(RE_ECC_MEMORY.findall((await run(['dmidecode', '-t', '16'])).stdout.decode()))
File "/usr/local/lib/python3.8/site-packages/middlewared/utils/__init__.py", line 53, in run
proc = await asyncio.create_subprocess_exec(*args, **kwargs)
File "/usr/local/lib/python3.8/asyncio/subprocess.py", line 236, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1630, in subprocess_exec
transport = await self._make_subprocess_transport(
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 197, in _make_subprocess_transport
transp = _UnixSubprocessTransport(self, protocol, args, shell,
File "/usr/local/lib/python3.8/asyncio/base_subprocess.py", line 36, in __init__
self._start(args=args, shell=shell, stdin=stdin, stdout=stdout,
File "/usr/local/lib/python3.8/asyncio/unix_events.py", line 789, in _start
self._proc = subprocess.Popen(
File "/usr/local/lib/python3.8/subprocess.py", line 854, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/local/lib/python3.8/subprocess.py", line 1658, in _execute_child
part = os.read(errpipe_read, 50000)
[2020/10/23 12:09:41] (WARNING) application.call_method():173 - Exception while calling sharing.webdav.query(*[])
concurrent.futures.process._RemoteTraceback:
"""
Traceback (most recent call last):
File "/usr/local/lib/python3.8/concurrent/futures/process.py", line 239, in _process_worker
r = call_item.fn(*call_item.args, **call_item.kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/worker.py", line 91, in main_worker
res = MIDDLEWARE._run(*call_args)
File "/usr/local/lib/python3.8/site-packages/middlewared/worker.py", line 45, in _run
return self._call(name, serviceobj, methodobj, args, job=job)
File "/usr/local/lib/python3.8/site-packages/middlewared/worker.py", line 33, in _call
with Client('ws+unix:///var/run/middlewared-internal.sock', py_exceptions=True) as c:
File "/usr/local/lib/python3.8/site-packages/middlewared/client/client.py", line 281, in __init__
self._ws.connect()
File "/usr/local/lib/python3.8/site-packages/middlewared/client/client.py", line 124, in connect
rv = super(WSClient, self).connect()
File "/usr/local/lib/python3.8/site-packages/ws4py/client/__init__.py", line 223, in connect
bytes = self.sock.recv(128)
socket.timeout: timed out
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 137, in call_method
result = await self.middleware._call(message['method'], serviceobj, methodobj, params, app=self,
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/service.py", line 442, in query
result = await self.middleware.call(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1233, in call
return await self._call(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/schema.py", line 973, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/datastore/read.py", line 163, in query
result = await self._queryset_serialize(
File "/usr/local/lib/python3.8/site-packages/middlewared/plugins/datastore/read.py", line 207, in _queryset_serialize
extend_context_value = await self.middleware.call(extend_context, extra_options)
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1233, in call
return await self._call(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1191, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/service.py", line 577, in sharing_task_extend_context
'locked_datasets': await self.middleware.call('zfs.dataset.locked_datasets'),
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1233, in call
return await self._call(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1199, in _call
return await self._call_worker(name, *prepared_call.args)
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1205, in _call_worker
return await self.run_in_proc(main_worker, name, args, job)
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1132, in run_in_proc
return await self.run_in_executor(self.__procpool, method, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1106, in run_in_executor
return await loop.run_in_executor(pool, functools.partial(method, *args, **kwargs))
socket.timeout: timed out
[2020/10/23 12:09:41] (DEBUG) application.__crash_reporting():181 - [Crash Reporting] is disabled using sentinel file.
[2020/10/23 12:13:59] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/bin/middlewared", line 10, in <module>
sys.exit(main())
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1577, in main
Middleware(
File "/usr/local/lib/python3.8/site-packages/middlewared/main.py", line 1421, in run
self.loop.run_forever()
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 570, in run_forever
self._run_once()
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1859, in _run_once
handle._run()
File "/usr/local/lib/python3.8/asyncio/events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.8/site-packages/aiohttp/web_ws.py", line 281, in send_str
await self._writer.send(data, binary=False, compress=compress)
File "/usr/local/lib/python3.8/site-packages/aiohttp/http_websocket.py", line 649, in send
await self._send_frame(message, WSMsgType.TEXT, compress)
File "/usr/local/lib/python3.8/site-packages/aiohttp/http_websocket.py", line 620, in _send_frame
self.transport.write(header + message)
File "/usr/local/lib/python3.8/asyncio/selector_events.py", line 908, in write
n = self._sock.send(data)
[2020/10/23 12:15:45] (WARNING) middlewared._loop_monitor_thread():1404 - Task seems blocked:
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 570, in run_forever
self._run_once()
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1859, in _run_once
handle._run()
File "/usr/local/lib/python3.8/asyncio/events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.8/site-packages/aiohttp/web_ws.py", line 281, in send_str
await self._writer.send(data, binary=False, compress=compress)
File "/usr/local/lib/python3.8/site-packages/aiohttp/http_websocket.py", line 649, in send
await self._send_frame(message, WSMsgType.TEXT, compress)
File "/usr/local/lib/python3.8/site-packages/aiohttp/http_websocket.py", line 618, in _send_frame
self.transport.write(message)
File "/usr/local/lib/python3.8/asyncio/selector_events.py", line 921, in write
self._loop._add_writer(self._sock_fd, self._write_ready)
File "/usr/local/lib/python3.8/asyncio/selector_events.py", line 301, in _add_writer
self._selector.modify(fd, mask | selectors.EVENT_WRITE,
File "/usr/local/lib/python3.8/selectors.py", line 261, in modify
key = self.register(fileobj, events, data)
File "/usr/local/lib/python3.8/selectors.py", line 526, in register
self._selector.control([kev], 0, 0)
The GUI is currently working (without a refresh), but is noticeably slow to respond.
EDIT: I noticed that the TrueCommand GUI wasn't responding, so I redeployed that container and it came back... not sure if it's a two-way thing.