Hey all,
Been struggling for the past few days with a fun new error.
I have previously installed a handful of custom docker images with no issues, that is until this past week.
When attempting to start any application i get a 404 error.
[EFAULT] Unable to delete namespace: (404) Reason: Not Found HTTP response headers: HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"ix-nginx\" not found","reason":"NotFound","details":{"name":"ix-nginx","kind":"namespaces"},"code":404}
Things ive tried/verified;
Network Settings (set other DNS to primary than pi-hole)
DNS settings (10.0.0.160 is pihole)
Can install apps from official charts and TrueCharts
can ping and NS lookup as many ips and domains as i want
Have restarted server, router, switch and modem
running date in shell displays correct date and time
verified date/time in server bios are correct
Have checked pi-hole and no queries are being blocked
disabled pi-hole blocking
have set application advanced settings to proper interface and gateway
As i stated above this happens with all custom docker images not just the one listed in the error
ive been trying to resolve this and honestly have no idea what to do at this point so any help would be appreciated
Been struggling for the past few days with a fun new error.
I have previously installed a handful of custom docker images with no issues, that is until this past week.
When attempting to start any application i get a 404 error.
[EFAULT] Unable to delete namespace: (404) Reason: Not Found HTTP response headers: HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"ix-nginx\" not found","reason":"NotFound","details":{"name":"ix-nginx","kind":"namespaces"},"code":404}
Code:
Error: concurrent.futures.process._RemoteTraceback:
Full error
"""
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/middlewared/plugins/zfs.py", line 916, in do_create
pool.create(
File "libzfs.pyx", line 411, in libzfs.ZFS.__exit__
File "/usr/lib/python3/dist-packages/middlewared/plugins/zfs.py", line 915, in do_create
pool = zfs.get(data['name'].split('/')[0])
File "libzfs.pyx", line 1102, in libzfs.ZFS.get
libzfs.ZFSException: Pool not found
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.9/concurrent/futures/process.py", line 243, in _process_worker
r = call_item.fn(*call_item.args, **call_item.kwargs)
File "/usr/lib/python3/dist-packages/middlewared/worker.py", line 114, in main_worker
res = MIDDLEWARE._run(*call_args)
File "/usr/lib/python3/dist-packages/middlewared/worker.py", line 45, in _run
return self._call(name, serviceobj, methodobj, args, job=job)
File "/usr/lib/python3/dist-packages/middlewared/worker.py", line 39, in _call
return methodobj(*params)
File "/usr/lib/python3/dist-packages/middlewared/worker.py", line 39, in _call
return methodobj(*params)
File "/usr/lib/python3/dist-packages/middlewared/schema.py", line 1146, in nf
res = f(*args, **kwargs)
File "/usr/lib/python3/dist-packages/middlewared/schema.py", line 1276, in nf
return func(*args, **kwargs)
File "/usr/lib/python3/dist-packages/middlewared/plugins/zfs.py", line 922, in do_create
raise CallError(f'Failed to create dataset: {e}')
middlewared.service_exception.CallError: [EFAULT] Failed to create dataset: Pool not found
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/middlewared/plugins/chart_releases_linux/chart_release.py", line 444, in do_create
await self.perform_actions(context)
File "/usr/lib/python3/dist-packages/middlewared/plugins/chart_releases_linux/chart_release.py", line 350, in perform_actions
await self.middleware.call(f'chart.release.{action["method"]}', *action['args'])
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1345, in call
return await self._call(
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1294, in _call
return await methodobj(*prepared_call.args)
File "/usr/lib/python3/dist-packages/middlewared/plugins/chart_releases_linux/ix_volumes.py", line 24, in update_volumes_for_release
await self.middleware.call(
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1345, in call
return await self._call(
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1294, in _call
return await methodobj(*prepared_call.args)
File "/usr/lib/python3/dist-packages/middlewared/service.py", line 920, in create
rv = await self.middleware._call(
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1302, in _call
return await self._call_worker(name, *prepared_call.args)
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1308, in _call_worker
return await self.run_in_proc(main_worker, name, args, job)
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1223, in run_in_proc
return await self.run_in_executor(self.__procpool, method, *args, **kwargs)
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1206, in run_in_executor
return await loop.run_in_executor(pool, functools.partial(method, *args, **kwargs))
middlewared.service_exception.CallError: [EFAULT] Failed to create dataset: Pool not found
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/middlewared/plugins/kubernetes_linux/namespace.py", line 61, in do_delete
await context['core_api'].delete_namespace(namespace)
File "/usr/lib/python3/dist-packages/kubernetes_asyncio/client/api_client.py", line 193, in __call_api
raise e
File "/usr/lib/python3/dist-packages/kubernetes_asyncio/client/api_client.py", line 186, in __call_api
response_data = await self.request(
File "/usr/lib/python3/dist-packages/kubernetes_asyncio/client/rest.py", line 220, in DELETE
return (await self.request("DELETE", url,
File "/usr/lib/python3/dist-packages/kubernetes_asyncio/client/rest.py", line 187, in request
raise ApiException(http_resp=r)
kubernetes_asyncio.client.exceptions.ApiException: (404)
Reason: Not Found
HTTP response headers: <CIMultiDictProxy('Audit-Id': 'ad85d143-5867-4adb-9057-6f7dd4598436', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': '3b50da09-fbfb-4c9d-9519-eef6c56c7d47', 'X-Kubernetes-Pf-Prioritylevel-Uid': '8e9b3540-0bbd-4e2e-8fcd-6919c9dd75fa', 'Date': 'Wed, 23 Nov 2022 01:36:35 GMT', 'Content-Length': '196')>
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"ix-nginx\" not found","reason":"NotFound","details":{"name":"ix-nginx","kind":"namespaces"},"code":404}
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/middlewared/job.py", line 411, in run
await self.future
File "/usr/lib/python3/dist-packages/middlewared/job.py", line 446, in __run_body
rv = await self.method(*([self] + args))
File "/usr/lib/python3/dist-packages/middlewared/schema.py", line 1140, in nf
res = await f(*args, **kwargs)
File "/usr/lib/python3/dist-packages/middlewared/schema.py", line 1272, in nf
return await func(*args, **kwargs)
File "/usr/lib/python3/dist-packages/middlewared/plugins/chart_releases_linux/chart_release.py", line 493, in do_create
await self.post_remove_tasks(data['release_name'])
File "/usr/lib/python3/dist-packages/middlewared/plugins/chart_releases_linux/chart_release.py", line 589, in post_remove_tasks
await self.middleware.call('k8s.namespace.delete', get_namespace(release_name))
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1345, in call
return await self._call(
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1294, in _call
return await methodobj(*prepared_call.args)
File "/usr/lib/python3/dist-packages/middlewared/service.py", line 942, in delete
rv = await self.middleware._call(
File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1294, in _call
return await methodobj(*prepared_call.args)
File "/usr/lib/python3/dist-packages/middlewared/schema.py", line 1140, in nf
res = await f(*args, **kwargs)
File "/usr/lib/python3/dist-packages/middlewared/schema.py", line 1272, in nf
return await func(*args, **kwargs)
File "/usr/lib/python3/dist-packages/middlewared/plugins/kubernetes_linux/namespace.py", line 63, in do_delete
raise CallError(f'Unable to delete namespace: {e}')
middlewared.service_exception.CallError: [EFAULT] Unable to delete namespace: (404)
Reason: Not Found
HTTP response headers: <CIMultiDictProxy('Audit-Id': 'ad85d143-5867-4adb-9057-6f7dd4598436', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': '3b50da09-fbfb-4c9d-9519-eef6c56c7d47', 'X-Kubernetes-Pf-Prioritylevel-Uid': '8e9b3540-0bbd-4e2e-8fcd-6919c9dd75fa', 'Date': 'Wed, 23 Nov 2022 01:36:35 GMT', 'Content-Length': '196')>
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"ix-nginx\" not found","reason":"NotFound","details":{"name":"ix-nginx","kind":"namespaces"},"code":404}Things ive tried/verified;
Network Settings (set other DNS to primary than pi-hole)
DNS settings (10.0.0.160 is pihole)
Can install apps from official charts and TrueCharts
can ping and NS lookup as many ips and domains as i want
Have restarted server, router, switch and modem
running date in shell displays correct date and time
verified date/time in server bios are correct
Have checked pi-hole and no queries are being blocked
disabled pi-hole blocking
have set application advanced settings to proper interface and gateway
As i stated above this happens with all custom docker images not just the one listed in the error
ive been trying to resolve this and honestly have no idea what to do at this point so any help would be appreciated
