
-------------
1) K3s logs
-------------

Feb 05 18:52:39 truenas k3s[30501]: I0205 18:52:39.059074   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:52:39 truenas k3s[30501]: I0205 18:52:39.059109   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:52:39 truenas k3s[30501]: E0205 18:52:39.060863   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:52:39 truenas k3s[30501]: I0205 18:52:39.133580   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="3e5d09fe3918bc55e3f605120b68ff215b5016ed4f4202dcac51a9632d27a605"
Feb 05 18:52:40 truenas k3s[30501]: I0205 18:52:40.727126   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:52:40 truenas k3s[30501]: E0205 18:52:40.727871   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:52:41 truenas k3s[30501]: I0205 18:52:41.658667   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:52:41 truenas k3s[30501]: I0205 18:52:41.658716   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:52:41 truenas k3s[30501]: I0205 18:52:41.658740   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:52:41 truenas k3s[30501]: I0205 18:52:41.658760   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:52:41 truenas k3s[30501]: I0205 18:52:41.658780   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:52:41 truenas k3s[30501]: E0205 18:52:41.660405   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:52:43 truenas k3s[30501]: I0205 18:52:43.881548   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:52:43 truenas k3s[30501]: I0205 18:52:43.881578   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:52:43 truenas k3s[30501]: I0205 18:52:43.881589   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:52:43 truenas k3s[30501]: E0205 18:52:43.882319   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:52:45 truenas k3s[30501]: E0205 18:52:45.014175   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:52:45 truenas k3s[30501]: I0205 18:52:45.666991   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:52:45 truenas k3s[30501]: E0205 18:52:45.667952   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:52:46 truenas k3s[30501]: I0205 18:52:46.945610   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:52:46 truenas k3s[30501]: E0205 18:52:46.946008   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:52:48 truenas k3s[30501]: I0205 18:52:48.154750   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:52:48 truenas k3s[30501]: I0205 18:52:48.154809   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:52:48 truenas k3s[30501]: E0205 18:52:48.155981   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:52:49 truenas k3s[30501]: I0205 18:52:49.139134   30501 scope.go:115] "RemoveContainer" containerID="5d0f9183814958c635fd27248f6ce2f2afd9ed940147259b2cc96442f94e0871"
Feb 05 18:52:49 truenas k3s[30501]: E0205 18:52:49.140105   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:52:49 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:52:49.161+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:52:50 truenas k3s[30501]: I0205 18:52:50.323027   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:52:50 truenas k3s[30501]: I0205 18:52:50.323075   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:52:50 truenas k3s[30501]: I0205 18:52:50.323100   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:52:50 truenas k3s[30501]: E0205 18:52:50.324571   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:52:51 truenas k3s[30501]: I0205 18:52:51.491035   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:52:51 truenas k3s[30501]: E0205 18:52:51.491615   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:52:52 truenas k3s[30501]: I0205 18:52:52.594959   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:52:52 truenas k3s[30501]: I0205 18:52:52.595009   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:52:52 truenas k3s[30501]: I0205 18:52:52.595032   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:52:52 truenas k3s[30501]: I0205 18:52:52.595051   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:52:52 truenas k3s[30501]: E0205 18:52:52.596587   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:52:53 truenas k3s[30501]: I0205 18:52:53.526825   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:52:53 truenas k3s[30501]: I0205 18:52:53.526876   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:52:53 truenas k3s[30501]: I0205 18:52:53.526899   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:52:53 truenas k3s[30501]: I0205 18:52:53.526920   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:52:53 truenas k3s[30501]: I0205 18:52:53.526942   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:52:53 truenas k3s[30501]: E0205 18:52:53.528526   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:52:54 truenas k3s[30501]: I0205 18:52:54.875107   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:52:54 truenas k3s[30501]: I0205 18:52:54.875154   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:52:54 truenas k3s[30501]: I0205 18:52:54.875177   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:52:54 truenas k3s[30501]: E0205 18:52:54.876554   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:52:58 truenas k3s[30501]: I0205 18:52:58.962991   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:52:58 truenas k3s[30501]: E0205 18:52:58.963586   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:53:00 truenas k3s[30501]: I0205 18:53:00.126915   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:53:00 truenas k3s[30501]: E0205 18:53:00.127734   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:53:01 truenas k3s[30501]: I0205 18:53:01.127010   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:53:01 truenas k3s[30501]: I0205 18:53:01.127061   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:53:01 truenas k3s[30501]: I0205 18:53:01.127084   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:53:01 truenas k3s[30501]: E0205 18:53:01.128442   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:53:02 truenas k3s[30501]: I0205 18:53:02.347073   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:53:02 truenas k3s[30501]: I0205 18:53:02.347122   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:53:02 truenas k3s[30501]: I0205 18:53:02.347145   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:53:02 truenas k3s[30501]: E0205 18:53:02.349582   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:53:03 truenas k3s[30501]: I0205 18:53:03.754778   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:53:03 truenas k3s[30501]: I0205 18:53:03.754826   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:53:03 truenas k3s[30501]: E0205 18:53:03.755953   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:53:04 truenas k3s[30501]: I0205 18:53:04.819011   30501 scope.go:115] "RemoveContainer" containerID="5d0f9183814958c635fd27248f6ce2f2afd9ed940147259b2cc96442f94e0871"
Feb 05 18:53:04 truenas k3s[30501]: E0205 18:53:04.819984   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:53:06 truenas k3s[30501]: I0205 18:53:06.963140   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:53:06 truenas k3s[30501]: E0205 18:53:06.963834   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:53:08 truenas k3s[30501]: I0205 18:53:08.406770   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:53:08 truenas k3s[30501]: I0205 18:53:08.406830   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:53:08 truenas k3s[30501]: I0205 18:53:08.406868   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:53:08 truenas k3s[30501]: I0205 18:53:08.406899   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:53:08 truenas k3s[30501]: I0205 18:53:08.406930   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:53:08 truenas k3s[30501]: E0205 18:53:08.409226   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:53:09 truenas k3s[30501]: I0205 18:53:09.019100   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:53:09 truenas k3s[30501]: I0205 18:53:09.019149   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:53:09 truenas k3s[30501]: I0205 18:53:09.019172   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:53:09 truenas k3s[30501]: I0205 18:53:09.019193   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:53:09 truenas k3s[30501]: E0205 18:53:09.020747   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:53:10 truenas k3s[30501]: I0205 18:53:10.155044   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:53:10 truenas k3s[30501]: I0205 18:53:10.155101   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:53:10 truenas k3s[30501]: I0205 18:53:10.155136   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:53:10 truenas k3s[30501]: E0205 18:53:10.156903   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:53:11 truenas k3s[30501]: I0205 18:53:11.783006   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:53:11 truenas k3s[30501]: E0205 18:53:11.783822   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:53:12 truenas k3s[30501]: I0205 18:53:12.935038   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:53:12 truenas k3s[30501]: E0205 18:53:12.935627   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:53:15 truenas k3s[30501]: I0205 18:53:15.779067   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:53:15 truenas k3s[30501]: I0205 18:53:15.779120   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:53:15 truenas k3s[30501]: I0205 18:53:15.779143   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:53:15 truenas k3s[30501]: E0205 18:53:15.780401   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:53:16 truenas k3s[30501]: I0205 18:53:16.946682   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:53:16 truenas k3s[30501]: I0205 18:53:16.946732   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:53:16 truenas k3s[30501]: E0205 18:53:16.947895   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:53:17 truenas k3s[30501]: I0205 18:53:17.830065   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:53:17 truenas k3s[30501]: I0205 18:53:17.830113   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:53:17 truenas k3s[30501]: I0205 18:53:17.830135   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:53:17 truenas k3s[30501]: E0205 18:53:17.831622   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:53:21 truenas k3s[30501]: I0205 18:53:21.287275   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:53:21 truenas k3s[30501]: I0205 18:53:21.287326   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:53:21 truenas k3s[30501]: I0205 18:53:21.287375   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:53:21 truenas k3s[30501]: I0205 18:53:21.287411   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:53:21 truenas k3s[30501]: E0205 18:53:21.288917   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:53:22 truenas k3s[30501]: I0205 18:53:22.723075   30501 scope.go:115] "RemoveContainer" containerID="5d0f9183814958c635fd27248f6ce2f2afd9ed940147259b2cc96442f94e0871"
Feb 05 18:53:22 truenas k3s[30501]: E0205 18:53:22.724049   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:53:23 truenas k3s[30501]: I0205 18:53:23.862927   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:53:23 truenas k3s[30501]: E0205 18:53:23.863560   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:53:24 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:53:24.720+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:53:25 truenas k3s[30501]: I0205 18:53:25.022846   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:53:25 truenas k3s[30501]: I0205 18:53:25.022914   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:53:25 truenas k3s[30501]: I0205 18:53:25.022952   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:53:25 truenas k3s[30501]: I0205 18:53:25.022992   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:53:25 truenas k3s[30501]: I0205 18:53:25.023028   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:53:25 truenas k3s[30501]: E0205 18:53:25.024716   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:53:25 truenas k3s[30501]: I0205 18:53:25.919032   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:53:25 truenas k3s[30501]: I0205 18:53:25.919088   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:53:25 truenas k3s[30501]: I0205 18:53:25.919123   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:53:25 truenas k3s[30501]: E0205 18:53:25.921166   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:53:26 truenas k3s[30501]: I0205 18:53:26.770950   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:53:26 truenas k3s[30501]: E0205 18:53:26.771766   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:53:27 truenas k3s[30501]: I0205 18:53:27.898445   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:53:27 truenas k3s[30501]: E0205 18:53:27.899072   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:53:29 truenas k3s[30501]: I0205 18:53:29.802448   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:53:29 truenas k3s[30501]: I0205 18:53:29.802476   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:53:29 truenas k3s[30501]: E0205 18:53:29.803508   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:53:30 truenas k3s[30501]: I0205 18:53:30.823015   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:53:30 truenas k3s[30501]: I0205 18:53:30.823064   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:53:30 truenas k3s[30501]: I0205 18:53:30.823086   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:53:30 truenas k3s[30501]: E0205 18:53:30.824433   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:53:31 truenas k3s[30501]: I0205 18:53:31.915034   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:53:31 truenas k3s[30501]: I0205 18:53:31.915085   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:53:31 truenas k3s[30501]: I0205 18:53:31.915107   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:53:31 truenas k3s[30501]: E0205 18:53:31.916610   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:53:33 truenas k3s[30501]: I0205 18:53:33.082940   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:53:33 truenas k3s[30501]: I0205 18:53:33.082990   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:53:33 truenas k3s[30501]: I0205 18:53:33.083013   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:53:33 truenas k3s[30501]: I0205 18:53:33.083035   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:53:33 truenas k3s[30501]: E0205 18:53:33.084595   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:53:34 truenas k3s[30501]: I0205 18:53:34.835054   30501 scope.go:115] "RemoveContainer" containerID="5d0f9183814958c635fd27248f6ce2f2afd9ed940147259b2cc96442f94e0871"
Feb 05 18:53:38 truenas k3s[30501]: I0205 18:53:38.342650   30501 scope.go:115] "RemoveContainer" containerID="5d0f9183814958c635fd27248f6ce2f2afd9ed940147259b2cc96442f94e0871"
Feb 05 18:53:39 truenas k3s[30501]: I0205 18:53:39.349920   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="cb5803a7f5f02e81882e99bb587093dc0771546206a7e099f0a15f84397ed87f"
Feb 05 18:53:39 truenas k3s[30501]: I0205 18:53:39.906960   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:53:39 truenas k3s[30501]: E0205 18:53:39.907555   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:53:41 truenas k3s[30501]: I0205 18:53:41.078951   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:53:41 truenas k3s[30501]: I0205 18:53:41.079001   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:53:41 truenas k3s[30501]: I0205 18:53:41.079024   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:53:41 truenas k3s[30501]: E0205 18:53:41.080434   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:53:42 truenas k3s[30501]: I0205 18:53:42.503018   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:53:42 truenas k3s[30501]: E0205 18:53:42.503650   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:53:43 truenas k3s[30501]: I0205 18:53:43.414791   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:53:43 truenas k3s[30501]: I0205 18:53:43.414841   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:53:43 truenas k3s[30501]: I0205 18:53:43.414864   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:53:43 truenas k3s[30501]: I0205 18:53:43.414887   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:53:43 truenas k3s[30501]: I0205 18:53:43.414908   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:53:43 truenas k3s[30501]: E0205 18:53:43.416523   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:53:44 truenas k3s[30501]: I0205 18:53:44.895129   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:53:44 truenas k3s[30501]: E0205 18:53:44.895957   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:53:46 truenas k3s[30501]: I0205 18:53:46.038951   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:53:46 truenas k3s[30501]: I0205 18:53:46.038999   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:53:46 truenas k3s[30501]: I0205 18:53:46.039022   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:53:46 truenas k3s[30501]: E0205 18:53:46.040390   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:53:47 truenas k3s[30501]: I0205 18:53:47.150671   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:53:47 truenas k3s[30501]: I0205 18:53:47.150729   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:53:47 truenas k3s[30501]: E0205 18:53:47.151907   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:53:49 truenas k3s[30501]: I0205 18:53:49.150915   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:53:49 truenas k3s[30501]: I0205 18:53:49.150965   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:53:49 truenas k3s[30501]: I0205 18:53:49.150987   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:53:49 truenas k3s[30501]: E0205 18:53:49.152442   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:53:50 truenas k3s[30501]: I0205 18:53:50.283022   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:53:50 truenas k3s[30501]: I0205 18:53:50.283069   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:53:50 truenas k3s[30501]: I0205 18:53:50.283091   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:53:50 truenas k3s[30501]: I0205 18:53:50.283112   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:53:50 truenas k3s[30501]: E0205 18:53:50.284665   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:53:51 truenas k3s[30501]: I0205 18:53:51.242877   30501 scope.go:115] "RemoveContainer" containerID="8ea1bd4156946a73147cbaf2eaa00cd8353912ab1a7ed65cdcb9c2b93bb6b43f"
Feb 05 18:53:51 truenas k3s[30501]: I0205 18:53:51.247397   30501 scope.go:115] "RemoveContainer" containerID="779d85a21cfd6dc19873de5195293af34601f96ec09862b5b6f1311a60274048"
Feb 05 18:53:52 truenas k3s[30501]: E0205 18:53:52.607437   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus-install-01-permissions\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus-install-01-permissions pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:53:54 truenas k3s[30501]: I0205 18:53:54.110934   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:53:54 truenas k3s[30501]: E0205 18:53:54.111520   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:53:55 truenas k3s[30501]: I0205 18:53:55.323094   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:53:55 truenas k3s[30501]: I0205 18:53:55.323150   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:53:55 truenas k3s[30501]: I0205 18:53:55.323173   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:53:55 truenas k3s[30501]: E0205 18:53:55.324593   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:53:57 truenas k3s[30501]: I0205 18:53:57.905657   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:53:57 truenas k3s[30501]: I0205 18:53:57.905705   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:53:57 truenas k3s[30501]: I0205 18:53:57.905730   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:53:57 truenas k3s[30501]: I0205 18:53:57.905750   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:53:57 truenas k3s[30501]: I0205 18:53:57.905771   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:53:57 truenas k3s[30501]: E0205 18:53:57.907320   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:53:59 truenas k3s[30501]: I0205 18:53:59.295180   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:53:59 truenas k3s[30501]: E0205 18:53:59.296016   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:54:01 truenas k3s[30501]: I0205 18:54:01.879007   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:54:01 truenas k3s[30501]: I0205 18:54:01.879066   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:54:01 truenas k3s[30501]: I0205 18:54:01.879101   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:54:01 truenas k3s[30501]: E0205 18:54:01.880649   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:54:02 truenas k3s[30501]: I0205 18:54:02.942802   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:54:02 truenas k3s[30501]: I0205 18:54:02.942867   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:54:02 truenas k3s[30501]: E0205 18:54:02.944100   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:54:03 truenas k3s[30501]: I0205 18:54:03.887128   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:54:03 truenas k3s[30501]: E0205 18:54:03.887824   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:54:06 truenas k3s[30501]: I0205 18:54:06.130971   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:54:06 truenas k3s[30501]: I0205 18:54:06.131022   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:54:06 truenas k3s[30501]: I0205 18:54:06.131045   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:54:06 truenas k3s[30501]: E0205 18:54:06.132470   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:54:07 truenas k3s[30501]: I0205 18:54:07.227114   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:54:07 truenas k3s[30501]: I0205 18:54:07.227161   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:54:07 truenas k3s[30501]: I0205 18:54:07.227185   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:54:07 truenas k3s[30501]: I0205 18:54:07.227206   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:54:07 truenas k3s[30501]: E0205 18:54:07.228750   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:54:07 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:54:07.303+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:54:10 truenas k3s[30501]: I0205 18:54:10.151081   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:54:10 truenas k3s[30501]: E0205 18:54:10.151669   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:54:11 truenas k3s[30501]: I0205 18:54:11.382964   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:54:11 truenas k3s[30501]: I0205 18:54:11.383035   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:54:11 truenas k3s[30501]: I0205 18:54:11.383075   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:54:11 truenas k3s[30501]: I0205 18:54:11.383113   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:54:11 truenas k3s[30501]: I0205 18:54:11.383144   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:54:11 truenas k3s[30501]: E0205 18:54:11.384811   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:54:13 truenas k3s[30501]: I0205 18:54:13.574226   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:54:13 truenas k3s[30501]: I0205 18:54:13.574288   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:54:13 truenas k3s[30501]: I0205 18:54:13.574303   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:54:13 truenas k3s[30501]: E0205 18:54:13.575195   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:54:16 truenas k3s[30501]: I0205 18:54:16.543003   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:54:16 truenas k3s[30501]: I0205 18:54:16.543055   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:54:16 truenas k3s[30501]: I0205 18:54:16.543079   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:54:16 truenas k3s[30501]: E0205 18:54:16.544896   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:54:18 truenas k3s[30501]: I0205 18:54:18.082998   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:54:18 truenas k3s[30501]: E0205 18:54:18.083837   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:54:20 truenas k3s[30501]: I0205 18:54:20.150842   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:54:20 truenas k3s[30501]: I0205 18:54:20.150869   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:54:20 truenas k3s[30501]: E0205 18:54:20.151424   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:54:21 truenas k3s[30501]: I0205 18:54:21.189411   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:54:21 truenas k3s[30501]: I0205 18:54:21.189435   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:54:21 truenas k3s[30501]: I0205 18:54:21.189443   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:54:21 truenas k3s[30501]: E0205 18:54:21.190033   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:54:22 truenas k3s[30501]: I0205 18:54:22.842556   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:54:22 truenas k3s[30501]: E0205 18:54:22.842922   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:54:24 truenas k3s[30501]: I0205 18:54:24.191539   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:54:24 truenas k3s[30501]: I0205 18:54:24.191604   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:54:24 truenas k3s[30501]: I0205 18:54:24.191638   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:54:24 truenas k3s[30501]: I0205 18:54:24.191673   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:54:24 truenas k3s[30501]: E0205 18:54:24.193244   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:54:25 truenas k3s[30501]: I0205 18:54:25.528618   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:54:25 truenas k3s[30501]: E0205 18:54:25.529545   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:54:26 truenas k3s[30501]: I0205 18:54:26.658965   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:54:26 truenas k3s[30501]: E0205 18:54:26.659568   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:54:27 truenas k3s[30501]: I0205 18:54:27.602694   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:54:27 truenas k3s[30501]: I0205 18:54:27.602743   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:54:27 truenas k3s[30501]: I0205 18:54:27.602765   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:54:27 truenas k3s[30501]: I0205 18:54:27.602786   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:54:27 truenas k3s[30501]: I0205 18:54:27.602805   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:54:27 truenas k3s[30501]: E0205 18:54:27.604407   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:54:30 truenas k3s[30501]: I0205 18:54:30.010930   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:54:30 truenas k3s[30501]: I0205 18:54:30.010979   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:54:30 truenas k3s[30501]: I0205 18:54:30.011001   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:54:30 truenas k3s[30501]: E0205 18:54:30.012395   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:54:31 truenas k3s[30501]: I0205 18:54:31.203061   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:54:31 truenas k3s[30501]: I0205 18:54:31.203111   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:54:31 truenas k3s[30501]: I0205 18:54:31.203134   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:54:31 truenas k3s[30501]: E0205 18:54:31.204413   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:54:32 truenas k3s[30501]: I0205 18:54:32.119040   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:54:32 truenas k3s[30501]: E0205 18:54:32.119858   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:54:34 truenas k3s[30501]: I0205 18:54:34.778741   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:54:34 truenas k3s[30501]: I0205 18:54:34.778800   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:54:34 truenas k3s[30501]: E0205 18:54:34.780303   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:54:35 truenas k3s[30501]: I0205 18:54:35.770960   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:54:35 truenas k3s[30501]: E0205 18:54:35.771547   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:54:36 truenas k3s[30501]: I0205 18:54:36.907056   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:54:36 truenas k3s[30501]: I0205 18:54:36.907108   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:54:36 truenas k3s[30501]: I0205 18:54:36.907130   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:54:36 truenas k3s[30501]: E0205 18:54:36.908569   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:54:38 truenas k3s[30501]: I0205 18:54:38.767076   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:54:38 truenas k3s[30501]: I0205 18:54:38.767123   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:54:38 truenas k3s[30501]: I0205 18:54:38.767148   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:54:38 truenas k3s[30501]: I0205 18:54:38.767167   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:54:38 truenas k3s[30501]: E0205 18:54:38.768809   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:54:39 truenas k3s[30501]: I0205 18:54:39.894738   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:54:39 truenas k3s[30501]: I0205 18:54:39.894785   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:54:39 truenas k3s[30501]: I0205 18:54:39.894806   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:54:39 truenas k3s[30501]: I0205 18:54:39.894827   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:54:39 truenas k3s[30501]: I0205 18:54:39.894847   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:54:39 truenas k3s[30501]: E0205 18:54:39.896448   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:54:40 truenas k3s[30501]: I0205 18:54:40.839113   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:54:40 truenas k3s[30501]: E0205 18:54:40.840527   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:54:41 truenas k3s[30501]: I0205 18:54:41.978924   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:54:42 truenas k3s[30501]: E0205 18:54:41.979498   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:54:44 truenas k3s[30501]: I0205 18:54:44.066928   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:54:44 truenas k3s[30501]: I0205 18:54:44.066974   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:54:44 truenas k3s[30501]: I0205 18:54:44.066995   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:54:44 truenas k3s[30501]: E0205 18:54:44.068378   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:54:45 truenas k3s[30501]: I0205 18:54:45.011056   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:54:45 truenas k3s[30501]: I0205 18:54:45.011113   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:54:45 truenas k3s[30501]: I0205 18:54:45.011149   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:54:45 truenas k3s[30501]: E0205 18:54:45.013219   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:54:46 truenas k3s[30501]: I0205 18:54:46.198680   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:54:46 truenas k3s[30501]: I0205 18:54:46.198728   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:54:46 truenas k3s[30501]: E0205 18:54:46.199872   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:54:47 truenas k3s[30501]: I0205 18:54:47.807095   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:54:47 truenas k3s[30501]: E0205 18:54:47.807931   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:54:49 truenas k3s[30501]: I0205 18:54:49.835237   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:54:49 truenas k3s[30501]: E0205 18:54:49.835897   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:54:51 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:54:51.224+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:54:52 truenas k3s[30501]: I0205 18:54:52.058760   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:54:52 truenas k3s[30501]: I0205 18:54:52.058809   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:54:52 truenas k3s[30501]: I0205 18:54:52.058831   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:54:52 truenas k3s[30501]: I0205 18:54:52.058852   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:54:52 truenas k3s[30501]: I0205 18:54:52.058872   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:54:53 truenas k3s[30501]: I0205 18:54:53.142726   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:54:53 truenas k3s[30501]: I0205 18:54:53.142753   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:54:53 truenas k3s[30501]: I0205 18:54:53.142762   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:54:53 truenas k3s[30501]: I0205 18:54:53.142770   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:54:53 truenas k3s[30501]: E0205 18:54:53.144165   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:54:54 truenas k3s[30501]: E0205 18:54:54.539042   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:54:55 truenas k3s[30501]: I0205 18:54:55.126972   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:54:55 truenas k3s[30501]: I0205 18:54:55.127022   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:54:55 truenas k3s[30501]: I0205 18:54:55.127045   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:54:55 truenas k3s[30501]: E0205 18:54:55.128469   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:54:56 truenas k3s[30501]: I0205 18:54:56.375159   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:54:56 truenas k3s[30501]: E0205 18:54:56.376185   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:54:57 truenas k3s[30501]: I0205 18:54:57.302989   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:54:57 truenas k3s[30501]: E0205 18:54:57.303559   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:54:59 truenas k3s[30501]: I0205 18:54:59.067005   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:54:59 truenas k3s[30501]: I0205 18:54:59.067054   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:54:59 truenas k3s[30501]: I0205 18:54:59.067082   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:54:59 truenas k3s[30501]: E0205 18:54:59.068345   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:55:00 truenas k3s[30501]: I0205 18:55:00.238955   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:55:00 truenas k3s[30501]: I0205 18:55:00.239023   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:55:00 truenas k3s[30501]: E0205 18:55:00.240252   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:55:00 truenas k3s[30501]: I0205 18:55:00.978983   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:55:00 truenas k3s[30501]: I0205 18:55:00.979039   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:55:00 truenas k3s[30501]: I0205 18:55:00.979073   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:55:00 truenas k3s[30501]: E0205 18:55:00.980781   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:55:02 truenas k3s[30501]: I0205 18:55:02.147034   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:55:02 truenas k3s[30501]: E0205 18:55:02.148203   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:55:03 truenas k3s[30501]: I0205 18:55:03.722983   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:55:03 truenas k3s[30501]: E0205 18:55:03.723563   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:55:05 truenas k3s[30501]: I0205 18:55:05.747075   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:55:05 truenas k3s[30501]: I0205 18:55:05.747124   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:55:05 truenas k3s[30501]: I0205 18:55:05.747147   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:55:05 truenas k3s[30501]: I0205 18:55:05.747167   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:55:05 truenas k3s[30501]: E0205 18:55:05.748708   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:55:08 truenas k3s[30501]: I0205 18:55:08.754996   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:55:08 truenas k3s[30501]: I0205 18:55:08.755046   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:55:08 truenas k3s[30501]: I0205 18:55:08.755068   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:55:08 truenas k3s[30501]: E0205 18:55:08.756489   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:55:10 truenas k3s[30501]: I0205 18:55:10.339069   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:55:10 truenas k3s[30501]: E0205 18:55:10.340044   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:55:11 truenas k3s[30501]: I0205 18:55:11.787038   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:55:11 truenas k3s[30501]: I0205 18:55:11.787088   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:55:11 truenas k3s[30501]: I0205 18:55:11.787113   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:55:11 truenas k3s[30501]: E0205 18:55:11.788370   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:55:12 truenas k3s[30501]: I0205 18:55:12.955014   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:55:16 truenas k3s[30501]: I0205 18:55:16.303028   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:55:16 truenas k3s[30501]: E0205 18:55:16.303940   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:55:17 truenas k3s[30501]: I0205 18:55:17.726673   30501 scope.go:115] "RemoveContainer" containerID="62e416d6f52b244418c2844728f9d2149173d80033983f70e94c86e9317b6f63"
Feb 05 18:55:18 truenas k3s[30501]: I0205 18:55:18.351106   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:55:18 truenas k3s[30501]: I0205 18:55:18.351153   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:55:18 truenas k3s[30501]: I0205 18:55:18.351175   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:55:18 truenas k3s[30501]: E0205 18:55:18.352572   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:55:19 truenas k3s[30501]: I0205 18:55:19.714905   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:55:19 truenas k3s[30501]: I0205 18:55:19.714956   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:55:19 truenas k3s[30501]: E0205 18:55:19.716468   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:55:20 truenas k3s[30501]: I0205 18:55:20.734520   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:55:20 truenas k3s[30501]: E0205 18:55:20.735134   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:55:22 truenas k3s[30501]: I0205 18:55:22.471456   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:55:22 truenas k3s[30501]: I0205 18:55:22.471507   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:55:22 truenas k3s[30501]: I0205 18:55:22.471530   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:55:22 truenas k3s[30501]: I0205 18:55:22.471549   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:55:22 truenas k3s[30501]: E0205 18:55:22.473207   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:55:22 truenas k3s[30501]: I0205 18:55:22.744076   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="821df86ff0f166cd2c36b814bc92442201a5edb5bf021e4a2d40e3d56f5670b9"
Feb 05 18:55:23 truenas k3s[30501]: I0205 18:55:23.823325   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:55:23 truenas k3s[30501]: E0205 18:55:23.824538   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:55:25 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:55:25.535+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:55:25 truenas k3s[30501]: I0205 18:55:25.681818   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:55:25 truenas k3s[30501]: I0205 18:55:25.681843   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:55:25 truenas k3s[30501]: I0205 18:55:25.681851   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:55:25 truenas k3s[30501]: E0205 18:55:25.682436   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:55:27 truenas k3s[30501]: I0205 18:55:27.762834   30501 scope.go:115] "RemoveContainer" containerID="8a6c3dff5c298f7661b5b153bc91bded65b5dae798dfc904050de0e8000db521"
Feb 05 18:55:30 truenas k3s[30501]: I0205 18:55:30.103719   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:55:30 truenas k3s[30501]: I0205 18:55:30.103774   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:55:30 truenas k3s[30501]: I0205 18:55:30.103796   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:55:30 truenas k3s[30501]: E0205 18:55:30.105050   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:55:33 truenas k3s[30501]: I0205 18:55:33.542077   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:55:33 truenas k3s[30501]: I0205 18:55:33.796522   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319"
Feb 05 18:55:34 truenas k3s[30501]: E0205 18:55:34.906555   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:55:36 truenas k3s[30501]: I0205 18:55:36.783202   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:55:36 truenas k3s[30501]: I0205 18:55:36.783256   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:55:36 truenas k3s[30501]: I0205 18:55:36.783279   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:55:36 truenas k3s[30501]: E0205 18:55:36.784704   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:55:38 truenas k3s[30501]: I0205 18:55:38.889301   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:55:38 truenas k3s[30501]: E0205 18:55:38.889551   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:55:41 truenas k3s[30501]: I0205 18:55:41.394890   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:55:41 truenas k3s[30501]: I0205 18:55:41.394939   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:55:41 truenas k3s[30501]: I0205 18:55:41.394963   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:55:41 truenas k3s[30501]: I0205 18:55:41.394984   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.399912   30501 remote_runtime.go:302] "CreateContainer in sandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found" podSandboxID="a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319"
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.400052   30501 kuberuntime_manager.go:872] container &Container{Name:csi-resizer,Image:registry.k8s.io/sig-storage/csi-resizer:v1.8.0,Command:[],Args:[--v=5 --csi-address=$(ADDRESS) --leader-election],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{EnvVar{Name:ADDRESS,Value:/var/lib/csi/sockets/pluginproxy/csi.sock,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:socket-dir,ReadOnly:false,MountPath:/var/lib/csi/sockets/pluginproxy/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-snm6j,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd): CreateContainerError: failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.404055   30501 remote_runtime.go:302] "CreateContainer in sandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found" podSandboxID="a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319"
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.404196   30501 kuberuntime_manager.go:872] container &Container{Name:csi-snapshotter,Image:registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2,Command:[],Args:[--csi-address=$(ADDRESS) --leader-election],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{EnvVar{Name:ADDRESS,Value:/var/lib/csi/sockets/pluginproxy/csi.sock,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:socket-dir,ReadOnly:false,MountPath:/var/lib/csi/sockets/pluginproxy/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-snm6j,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd): CreateContainerError: failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.407756   30501 remote_runtime.go:302] "CreateContainer in sandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found" podSandboxID="a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319"
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.407870   30501 kuberuntime_manager.go:872] container &Container{Name:snapshot-controller,Image:registry.k8s.io/sig-storage/snapshot-controller:v6.2.2,Command:[],Args:[--v=5 --leader-election=true],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-snm6j,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd): CreateContainerError: failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.411854   30501 remote_runtime.go:302] "CreateContainer in sandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found" podSandboxID="a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319"
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.411997   30501 kuberuntime_manager.go:872] container &Container{Name:csi-provisioner,Image:registry.k8s.io/sig-storage/csi-provisioner:v3.5.0,Command:[],Args:[--csi-address=$(ADDRESS) --v=5 --feature-gates=Topology=true --strict-topology --leader-election --extra-create-metadata=true --enable-capacity=true --default-fstype=ext4],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{EnvVar{Name:ADDRESS,Value:/var/lib/csi/sockets/pluginproxy/csi.sock,ValueFrom:nil,},EnvVar{Name:NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:socket-dir,ReadOnly:false,MountPath:/var/lib/csi/sockets/pluginproxy/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-snm6j,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd): CreateContainerError: failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found
Feb 05 18:55:41 truenas k3s[30501]: E0205 18:55:41.412156   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CreateContainerError: \"failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found\", failed to \"StartContainer\" for \"csi-snapshotter\" with CreateContainerError: \"failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found\", failed to \"StartContainer\" for \"snapshot-controller\" with CreateContainerError: \"failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found\", failed to \"StartContainer\" for \"csi-provisioner\" with CreateContainerError: \"failed to get sandbox container task: no running task found: task a52dd4b29007196a60db5f9f0c46e7fca698c039f5c244729ec2279bf15fb319 not found: not found\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:55:42 truenas k3s[30501]: I0205 18:55:42.323169   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:55:42 truenas k3s[30501]: I0205 18:55:42.323219   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:55:42 truenas k3s[30501]: I0205 18:55:42.323242   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:55:42 truenas k3s[30501]: I0205 18:55:42.323265   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:55:42 truenas k3s[30501]: E0205 18:55:42.324824   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:55:43 truenas k3s[30501]: I0205 18:55:43.847108   30501 scope.go:115] "RemoveContainer" containerID="652e2baddc80c77bb46bdc0965b26ef590b9d60bd6c8dbf931f4f0ecc5757cf2"
Feb 05 18:55:44 truenas k3s[30501]: I0205 18:55:44.842779   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:55:44 truenas k3s[30501]: I0205 18:55:44.842829   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:55:44 truenas k3s[30501]: E0205 18:55:44.844003   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:55:45 truenas k3s[30501]: I0205 18:55:45.423020   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:55:45 truenas k3s[30501]: E0205 18:55:45.423897   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:55:46 truenas k3s[30501]: I0205 18:55:46.511034   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:55:46 truenas k3s[30501]: I0205 18:55:46.511083   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:55:46 truenas k3s[30501]: I0205 18:55:46.511106   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:55:46 truenas k3s[30501]: E0205 18:55:46.512531   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:55:47 truenas k3s[30501]: I0205 18:55:47.618951   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:55:47 truenas k3s[30501]: E0205 18:55:47.619924   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:55:48 truenas k3s[30501]: I0205 18:55:48.869822   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="1876c63718e2c2a4e9b3166f5dade884f410fa99308eb7fe7fe0dc037fe14d0b"
Feb 05 18:55:50 truenas k3s[30501]: I0205 18:55:50.402987   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:55:50 truenas k3s[30501]: I0205 18:55:50.403036   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:55:50 truenas k3s[30501]: I0205 18:55:50.403058   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:55:50 truenas k3s[30501]: E0205 18:55:50.405057   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:55:52 truenas k3s[30501]: I0205 18:55:52.215018   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:55:52 truenas k3s[30501]: E0205 18:55:52.215613   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:55:55 truenas k3s[30501]: I0205 18:55:55.206519   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:55:55 truenas k3s[30501]: I0205 18:55:55.206569   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:55:55 truenas k3s[30501]: I0205 18:55:55.206591   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:55:55 truenas k3s[30501]: E0205 18:55:55.207969   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:55:56 truenas k3s[30501]: I0205 18:55:56.507048   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:55:56 truenas k3s[30501]: E0205 18:55:56.507619   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:55:59 truenas k3s[30501]: I0205 18:55:59.875096   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:55:59 truenas k3s[30501]: I0205 18:55:59.875145   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:55:59 truenas k3s[30501]: I0205 18:55:59.875168   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:55:59 truenas k3s[30501]: I0205 18:55:59.875189   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:56:02 truenas k3s[30501]: I0205 18:56:02.973400   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:56:02 truenas k3s[30501]: I0205 18:56:02.973423   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:56:02 truenas k3s[30501]: I0205 18:56:02.973432   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:56:05 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:56:05.172+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:56:07 truenas k3s[30501]: E0205 18:56:07.423847   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:56:10 truenas k3s[30501]: I0205 18:56:10.766720   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:56:10 truenas k3s[30501]: I0205 18:56:10.766769   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:56:10 truenas k3s[30501]: E0205 18:56:10.767914   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:56:12 truenas k3s[30501]: I0205 18:56:12.571056   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:56:12 truenas k3s[30501]: E0205 18:56:12.572039   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:56:13 truenas k3s[30501]: W0205 18:56:13.781487   30501 container.go:488] Failed to get RecentStats("/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod39953722_45d6_4f0c_b0bc_2bf8b3ec0cdd.slice/cri-containerd-c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb.scope") while determining the next housekeeping: unable to find data in memory cache
Feb 05 18:56:14 truenas k3s[30501]: I0205 18:56:14.058229   30501 scope.go:115] "RemoveContainer" containerID="fccb8782873bd159e4005d71d52e2e6d15404cf4001bbec2c31321c81f00e4cb"
Feb 05 18:56:14 truenas k3s[30501]: I0205 18:56:14.067221   30501 scope.go:115] "RemoveContainer" containerID="49837e3cd84b3ee34685a8e275cdf012f3d196eed6dc83f72b157eb3f14ed185"
Feb 05 18:56:14 truenas k3s[30501]: I0205 18:56:14.075888   30501 scope.go:115] "RemoveContainer" containerID="9b34ad7b3d0a2c1b6d69de286ce8f235b24ad152baf2d08469b8387d974b17e7"
Feb 05 18:56:14 truenas k3s[30501]: I0205 18:56:14.084832   30501 scope.go:115] "RemoveContainer" containerID="4994444d96dd89c5c4700a36e399f2d42306b556006aeade2c0ff6dbd81c0a28"
Feb 05 18:56:15 truenas k3s[30501]: E0205 18:56:15.091183   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:56:17 truenas k3s[30501]: I0205 18:56:17.182985   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:56:17 truenas k3s[30501]: I0205 18:56:17.183035   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:56:17 truenas k3s[30501]: I0205 18:56:17.183061   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:56:17 truenas k3s[30501]: E0205 18:56:17.184879   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:56:19 truenas k3s[30501]: I0205 18:56:19.105627   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="21ae9e972d5c5291381bb974f6bb33763ac56017a390e0575497fb4068459783"
Feb 05 18:56:19 truenas k3s[30501]: I0205 18:56:19.491134   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:56:19 truenas k3s[30501]: E0205 18:56:19.491716   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:56:22 truenas k3s[30501]: I0205 18:56:22.796316   30501 scope.go:115] "RemoveContainer" containerID="01ffaa9112ba5201e3a58329c78c6924ed48dd6478aa6118f2f8f5f0008a5d3a"
Feb 05 18:56:27 truenas k3s[30501]: I0205 18:56:27.107134   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:56:27 truenas k3s[30501]: I0205 18:56:27.107182   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:56:27 truenas k3s[30501]: I0205 18:56:27.107203   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:56:28 truenas k3s[30501]: I0205 18:56:28.611205   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:56:28 truenas k3s[30501]: E0205 18:56:28.611875   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:56:29 truenas k3s[30501]: W0205 18:56:29.140263   30501 container.go:488] Failed to get RecentStats("/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod945e6d80_a04f_4842_ac56_5ed0b16f8f7e.slice/cri-containerd-2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0.scope") while determining the next housekeeping: unable to find data in memory cache
Feb 05 18:56:29 truenas k3s[30501]: I0205 18:56:29.169942   30501 scope.go:115] "RemoveContainer" containerID="4b08aecf3c989e1b7f4cdba395aef667c7fbc1b1e8de57dd7d40de60bb3b2029"
Feb 05 18:56:29 truenas k3s[30501]: I0205 18:56:29.174710   30501 scope.go:115] "RemoveContainer" containerID="0c8fa78404868ded7c3c0bddfc24fddb39b3b0a3d39d922ba37a2b91060a2736"
Feb 05 18:56:29 truenas k3s[30501]: I0205 18:56:29.179193   30501 scope.go:115] "RemoveContainer" containerID="e09a8d0b4da9168a6e221211ca87414551e6c9a369d74bbeeb93c1e4f92a1ea2"
Feb 05 18:56:29 truenas k3s[30501]: I0205 18:56:29.182739   30501 scope.go:115] "RemoveContainer" containerID="5158520921fcad2fa81d6702fcf5a7af922d6ff41212a4d1f13c73f85bc92c9f"
Feb 05 18:56:29 truenas k3s[30501]: I0205 18:56:29.186394   30501 scope.go:115] "RemoveContainer" containerID="b00602433b06f7f9eb53392d112d1ea165f1c3e86710546466d7b54a58641971"
Feb 05 18:56:29 truenas k3s[30501]: I0205 18:56:29.189791   30501 scope.go:115] "RemoveContainer" containerID="e91adcf96b8d168e0b3a041711c719d77d2496c8472cd4482315a5c0dfade044"
Feb 05 18:56:32 truenas k3s[30501]: W0205 18:56:32.248084   30501 manager.go:1174] Failed to process watch event {EventType:0 Name:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod37a40dae_33a2_4237_ac94_71da566d2ed1.slice/cri-containerd-df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2.scope WatchSource:0}: task df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2 not found: not found
Feb 05 18:56:33 truenas k3s[30501]: I0205 18:56:33.856024   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:56:33 truenas k3s[30501]: E0205 18:56:33.856865   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:56:35 truenas k3s[30501]: I0205 18:56:35.220601   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="4b516877f8ec1b2a3047277caec6eb7fa0abae28892d4ee7e50988be63a9a35f"
Feb 05 18:56:35 truenas k3s[30501]: I0205 18:56:35.223416   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="800df4f7a84ed09b6e17e9f697dccc13ea49401bc6bf3875faba63af67da2eb5"
Feb 05 18:56:35 truenas k3s[30501]: W0205 18:56:35.368617   30501 manager.go:1174] Failed to process watch event {EventType:0 Name:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod37a40dae_33a2_4237_ac94_71da566d2ed1.slice/cri-containerd-68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d.scope WatchSource:0}: task 68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d not found: not found
Feb 05 18:56:35 truenas k3s[30501]: E0205 18:56:35.381363   30501 cadvisor_stats_provider.go:442] "Partial failure issuing cadvisor.ContainerInfoV2" err="partial failures: [\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod945e6d80_a04f_4842_ac56_5ed0b16f8f7e.slice/cri-containerd-2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0.scope\": RecentStats: unable to find data in memory cache]"
Feb 05 18:56:37 truenas k3s[30501]: I0205 18:56:37.222881   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:56:37 truenas k3s[30501]: I0205 18:56:37.222937   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:56:37 truenas k3s[30501]: E0205 18:56:37.224164   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:56:38 truenas k3s[30501]: W0205 18:56:38.494419   30501 manager.go:1174] Failed to process watch event {EventType:0 Name:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod945e6d80_a04f_4842_ac56_5ed0b16f8f7e.slice/cri-containerd-d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f.scope WatchSource:0}: task d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f not found: not found
Feb 05 18:56:41 truenas k3s[30501]: I0205 18:56:41.743563   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:56:41 truenas k3s[30501]: E0205 18:56:41.744681   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:56:43 truenas k3s[30501]: I0205 18:56:43.095131   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:56:43 truenas k3s[30501]: I0205 18:56:43.095181   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:56:43 truenas k3s[30501]: I0205 18:56:43.095203   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:56:43 truenas k3s[30501]: E0205 18:56:43.096475   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:56:43 truenas k3s[30501]: E0205 18:56:43.722188   30501 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0" cmd=[/csi-node-driver-registrar --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock --mode=kubelet-registration-probe]
Feb 05 18:56:43 truenas k3s[30501]: E0205 18:56:43.723068   30501 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0" cmd=[/csi-node-driver-registrar --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock --mode=kubelet-registration-probe]
Feb 05 18:56:43 truenas k3s[30501]: E0205 18:56:43.723856   30501 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0" cmd=[/csi-node-driver-registrar --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock --mode=kubelet-registration-probe]
Feb 05 18:56:44 truenas k3s[30501]: I0205 18:56:44.287087   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:56:44 truenas k3s[30501]: E0205 18:56:44.287715   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:56:45 truenas k3s[30501]: I0205 18:56:45.304000   30501 scope.go:115] "RemoveContainer" containerID="a16125bb917852c4751f670102faa48e7141a3c50294c67856d2bd1172c99a35"
Feb 05 18:56:45 truenas k3s[30501]: I0205 18:56:45.325364   30501 scope.go:115] "RemoveContainer" containerID="6a4189291116a44b58f420c1f9b45d3e4c8a25786fd61c7dad655b75f2b63a14"
Feb 05 18:56:45 truenas k3s[30501]: I0205 18:56:45.335709   30501 scope.go:115] "RemoveContainer" containerID="f00c44fbd88ceac69717972c2cafe515bd583f50c6f28d9c8dc21186317b701d"
Feb 05 18:56:46 truenas k3s[30501]: I0205 18:56:46.312278   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="5e0d4a476c9e73e6411829380286760a70cd371092e738b70308c31a54e86669"
Feb 05 18:56:46 truenas k3s[30501]: I0205 18:56:46.527273   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:56:46 truenas k3s[30501]: E0205 18:56:46.528129   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:56:47 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:56:47.359+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:56:48 truenas k3s[30501]: I0205 18:56:48.643734   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:56:48 truenas k3s[30501]: E0205 18:56:48.644375   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:56:49 truenas k3s[30501]: I0205 18:56:49.609538   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:56:53 truenas k3s[30501]: E0205 18:56:53.722652   30501 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0" cmd=[/csi-node-driver-registrar --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock --mode=kubelet-registration-probe]
Feb 05 18:56:53 truenas k3s[30501]: E0205 18:56:53.723378   30501 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0" cmd=[/csi-node-driver-registrar --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock --mode=kubelet-registration-probe]
Feb 05 18:56:53 truenas k3s[30501]: E0205 18:56:53.723965   30501 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0" cmd=[/csi-node-driver-registrar --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock --mode=kubelet-registration-probe]
Feb 05 18:56:54 truenas k3s[30501]: I0205 18:56:54.539847   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:56:54 truenas k3s[30501]: E0205 18:56:54.540688   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:56:55 truenas k3s[30501]: I0205 18:56:55.361806   30501 scope.go:115] "RemoveContainer" containerID="423d6e7406c383def6d90563fa76e31f0179476a5c45a26f3c4bd73996f61dcd"
Feb 05 18:56:59 truenas k3s[30501]: I0205 18:56:59.383495   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:56:59 truenas k3s[30501]: I0205 18:56:59.383559   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:57:00 truenas k3s[30501]: I0205 18:57:00.384529   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="5361cfb687e3b4d76880fdda25ea90c485296be8417809c7849b0630fbd8ede7"
Feb 05 18:57:00 truenas k3s[30501]: E0205 18:57:00.557097   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:57:02 truenas k3s[30501]: I0205 18:57:02.807185   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:57:02 truenas k3s[30501]: E0205 18:57:02.808183   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:57:03 truenas k3s[30501]: E0205 18:57:03.594744   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:57:06 truenas k3s[30501]: I0205 18:57:06.324162   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:57:06 truenas k3s[30501]: E0205 18:57:06.324765   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:57:07 truenas k3s[30501]: E0205 18:57:07.247792   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:57:08 truenas k3s[30501]: I0205 18:57:08.019780   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:57:08 truenas k3s[30501]: I0205 18:57:08.019843   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:57:08 truenas k3s[30501]: I0205 18:57:08.019877   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:57:12 truenas k3s[30501]: E0205 18:57:12.842634   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:57:14 truenas k3s[30501]: I0205 18:57:14.279646   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:57:14 truenas k3s[30501]: I0205 18:57:14.279706   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:57:14 truenas k3s[30501]: I0205 18:57:14.279740   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:57:14 truenas k3s[30501]: I0205 18:57:14.279772   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:57:14 truenas k3s[30501]: I0205 18:57:14.279806   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:57:14 truenas k3s[30501]: E0205 18:57:14.282458   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:57:17 truenas k3s[30501]: I0205 18:57:17.471231   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:57:17 truenas k3s[30501]: E0205 18:57:17.472107   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:57:19 truenas k3s[30501]: I0205 18:57:19.223311   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:57:19 truenas k3s[30501]: I0205 18:57:19.223407   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:57:19 truenas k3s[30501]: I0205 18:57:19.223454   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:57:19 truenas k3s[30501]: E0205 18:57:19.225462   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:57:23 truenas k3s[30501]: E0205 18:57:23.090650   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:57:23 truenas k3s[30501]: time="2024-02-05T18:57:23+01:00" level=info msg="COMPACT compactRev=48309 targetCompactRev=48500 currentRev=49500"
Feb 05 18:57:23 truenas k3s[30501]: time="2024-02-05T18:57:23+01:00" level=info msg="COMPACT deleted 189 rows from 191 revisions in 58.075244ms - compacted to 48500/49500"
Feb 05 18:57:23 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:57:23.961+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:57:25 truenas k3s[30501]: I0205 18:57:25.554945   30501 scope.go:115] "RemoveContainer" containerID="285151fad8b937d4cd4049cb1d0ae2205928179c0ef18441103c1b9e4ecb2090"
Feb 05 18:57:25 truenas k3s[30501]: I0205 18:57:25.558834   30501 scope.go:115] "RemoveContainer" containerID="603e865f63605f5c61df1220eb0b52b2c116bb900f060f70b2f437fcba24ad21"
Feb 05 18:57:25 truenas k3s[30501]: I0205 18:57:25.562154   30501 scope.go:115] "RemoveContainer" containerID="3e9628902535adcb762136e02e9efc3e8301630d50f04b5cb5f2b4f27e307e4c"
Feb 05 18:57:27 truenas k3s[30501]: I0205 18:57:27.103015   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:57:27 truenas k3s[30501]: I0205 18:57:27.103080   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:57:27 truenas k3s[30501]: I0205 18:57:27.103120   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:57:27 truenas k3s[30501]: I0205 18:57:27.103158   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:57:27 truenas k3s[30501]: E0205 18:57:27.105164   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:57:28 truenas k3s[30501]: I0205 18:57:28.619012   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:57:28 truenas k3s[30501]: I0205 18:57:28.619065   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:57:28 truenas k3s[30501]: I0205 18:57:28.619090   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:57:28 truenas k3s[30501]: I0205 18:57:28.619110   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:57:28 truenas k3s[30501]: I0205 18:57:28.619129   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:57:28 truenas k3s[30501]: E0205 18:57:28.620869   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:57:29 truenas k3s[30501]: I0205 18:57:29.789649   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:57:29 truenas k3s[30501]: E0205 18:57:29.790089   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:57:31 truenas k3s[30501]: I0205 18:57:31.318796   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:57:31 truenas k3s[30501]: E0205 18:57:31.319151   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:57:31 truenas k3s[30501]: I0205 18:57:31.579018   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="482368d31eb6677c2f35c64ec2fbe52576099a9a0425e4820041e8add0a74b32"
Feb 05 18:57:32 truenas k3s[30501]: I0205 18:57:32.786727   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:57:32 truenas k3s[30501]: I0205 18:57:32.786750   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:57:32 truenas k3s[30501]: I0205 18:57:32.786764   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:57:32 truenas k3s[30501]: E0205 18:57:32.787399   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:57:34 truenas k3s[30501]: I0205 18:57:34.163236   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:57:34 truenas k3s[30501]: I0205 18:57:34.163316   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:57:34 truenas k3s[30501]: I0205 18:57:34.163372   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:57:34 truenas k3s[30501]: E0205 18:57:34.165001   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:57:36 truenas k3s[30501]: I0205 18:57:36.597141   30501 scope.go:115] "RemoveContainer" containerID="8c27d97ac6987b9c6d516589eb07259dba9b4348c2b0449af8cc889cb482ad1d"
Feb 05 18:57:37 truenas k3s[30501]: I0205 18:57:37.602554   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="5ddbe7c905118a166ac084805e45c796d597402dc00f24d21ad6e00aac0d851a"
Feb 05 18:57:39 truenas k3s[30501]: I0205 18:57:39.429504   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:57:39 truenas k3s[30501]: I0205 18:57:39.429529   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:57:39 truenas k3s[30501]: I0205 18:57:39.429538   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:57:39 truenas k3s[30501]: I0205 18:57:39.429548   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:57:39 truenas k3s[30501]: E0205 18:57:39.430192   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:57:40 truenas k3s[30501]: E0205 18:57:40.537421   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:57:42 truenas k3s[30501]: I0205 18:57:42.081239   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:57:42 truenas k3s[30501]: E0205 18:57:42.081567   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:57:44 truenas k3s[30501]: I0205 18:57:44.871042   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:57:44 truenas k3s[30501]: I0205 18:57:44.871091   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:57:44 truenas k3s[30501]: I0205 18:57:44.871113   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:57:44 truenas k3s[30501]: E0205 18:57:44.872551   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:57:46 truenas k3s[30501]: I0205 18:57:46.074999   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:57:46 truenas k3s[30501]: I0205 18:57:46.075050   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:57:46 truenas k3s[30501]: I0205 18:57:46.075073   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:57:46 truenas k3s[30501]: E0205 18:57:46.077107   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:57:48 truenas k3s[30501]: I0205 18:57:48.022895   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:57:48 truenas k3s[30501]: E0205 18:57:48.028579   30501 remote_runtime.go:302] "CreateContainer in sandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to get sandbox container task: no running task found: task 5ddbe7c905118a166ac084805e45c796d597402dc00f24d21ad6e00aac0d851a not found: not found" podSandboxID="5ddbe7c905118a166ac084805e45c796d597402dc00f24d21ad6e00aac0d851a"
Feb 05 18:57:48 truenas k3s[30501]: E0205 18:57:48.028791   30501 kuberuntime_manager.go:872] container &Container{Name:csi-node-driver-registrar,Image:registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0,Command:[],Args:[--v=5 --csi-address=$(ADDRESS) --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{EnvVar{Name:ADDRESS,Value:/plugin/csi.sock,ValueFrom:nil,},EnvVar{Name:DRIVER_REG_SOCK_PATH,Value:/var/lib/kubelet/plugins/zfs-localpv/csi.sock,ValueFrom:nil,},EnvVar{Name:KUBE_NODE_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:spec.nodeName,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:NODE_DRIVER,Value:openebs-zfs,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:plugin-dir,ReadOnly:false,MountPath:/plugin,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:registration-dir,ReadOnly:false,MountPath:/registration,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-hqgx2,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sh -c rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807): CreateContainerError: failed to get sandbox container task: no running task found: task 5ddbe7c905118a166ac084805e45c796d597402dc00f24d21ad6e00aac0d851a not found: not found
Feb 05 18:57:48 truenas k3s[30501]: E0205 18:57:48.028888   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CreateContainerError: \"failed to get sandbox container task: no running task found: task 5ddbe7c905118a166ac084805e45c796d597402dc00f24d21ad6e00aac0d851a not found: not found\"" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:57:49 truenas k3s[30501]: I0205 18:57:49.351210   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:57:49 truenas k3s[30501]: I0205 18:57:49.351265   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:57:49 truenas k3s[30501]: I0205 18:57:49.351288   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:57:49 truenas k3s[30501]: I0205 18:57:49.351309   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:57:49 truenas k3s[30501]: E0205 18:57:49.353438   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:57:50 truenas k3s[30501]: I0205 18:57:50.715235   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:57:50 truenas k3s[30501]: E0205 18:57:50.715845   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:57:51 truenas k3s[30501]: I0205 18:57:51.706948   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:57:51 truenas k3s[30501]: I0205 18:57:51.707000   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:57:51 truenas k3s[30501]: I0205 18:57:51.707026   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:57:51 truenas k3s[30501]: I0205 18:57:51.707050   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:57:51 truenas k3s[30501]: I0205 18:57:51.707071   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:57:51 truenas k3s[30501]: E0205 18:57:51.708731   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:57:52 truenas k3s[30501]: I0205 18:57:52.823207   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:57:52 truenas k3s[30501]: E0205 18:57:52.824295   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:57:55 truenas k3s[30501]: I0205 18:57:55.531279   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:57:55 truenas k3s[30501]: E0205 18:57:55.531864   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:57:57 truenas k3s[30501]: I0205 18:57:57.030341   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:57:57 truenas k3s[30501]: I0205 18:57:57.030366   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:57:57 truenas k3s[30501]: I0205 18:57:57.030376   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:57:57 truenas k3s[30501]: E0205 18:57:57.031005   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:57:59 truenas k3s[30501]: I0205 18:57:59.251343   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:57:59 truenas k3s[30501]: E0205 18:57:59.252061   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:58:03 truenas k3s[30501]: I0205 18:58:03.707520   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:58:03 truenas k3s[30501]: E0205 18:58:03.708416   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:58:04 truenas k3s[30501]: E0205 18:58:04.999739   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:58:05 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:58:05.384+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:58:06 truenas k3s[30501]: I0205 18:58:06.103206   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:58:06 truenas k3s[30501]: I0205 18:58:06.103258   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:58:06 truenas k3s[30501]: I0205 18:58:06.103280   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:58:06 truenas k3s[30501]: E0205 18:58:06.105263   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:58:07 truenas k3s[30501]: E0205 18:58:07.767953   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:58:08 truenas k3s[30501]: I0205 18:58:08.379795   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:58:08 truenas k3s[30501]: I0205 18:58:08.379849   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:58:08 truenas k3s[30501]: I0205 18:58:08.379871   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:58:08 truenas k3s[30501]: I0205 18:58:08.379891   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:58:08 truenas k3s[30501]: E0205 18:58:08.381462   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:58:08 truenas k3s[30501]: E0205 18:58:08.811237   30501 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = Unknown desc = failed to exec in container: failed to start exec \"f5e1ce2aa172bf8d506057822c544d4d98d7afce288d505b60242ee3a0bd6a83\": OCI runtime exec failed: exec failed: unable to start container process: exec: \"/bin/sh\": stat /bin/sh: no such file or directory: unknown" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d" cmd=[/bin/sh -c rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock]
Feb 05 18:58:08 truenas k3s[30501]: E0205 18:58:08.811297   30501 kuberuntime_container.go:596] "PreStop hook failed" err="rpc error: code = Unknown desc = failed to exec in container: failed to start exec \"f5e1ce2aa172bf8d506057822c544d4d98d7afce288d505b60242ee3a0bd6a83\": OCI runtime exec failed: exec failed: unable to start container process: exec: \"/bin/sh\": stat /bin/sh: no such file or directory: unknown" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807 containerName="csi-node-driver-registrar" containerID="containerd://1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 18:58:09 truenas k3s[30501]: I0205 18:58:09.758233   30501 scope.go:115] "RemoveContainer" containerID="829d71b02d303c2fe919a1b13fdd09f0aa576173395caf9cd2a9f11ef47391b7"
Feb 05 18:58:10 truenas k3s[30501]: I0205 18:58:10.707394   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:58:10 truenas k3s[30501]: I0205 18:58:10.707422   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:58:10 truenas k3s[30501]: I0205 18:58:10.707431   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:58:10 truenas k3s[30501]: I0205 18:58:10.707442   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:58:10 truenas k3s[30501]: I0205 18:58:10.707452   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:58:10 truenas k3s[30501]: E0205 18:58:10.708213   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:58:11 truenas k3s[30501]: I0205 18:58:11.776385   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="5a64c59c391605bb906d0ad859e736d34592e75dd76df42b4d948df3359fbeb4"
Feb 05 18:58:11 truenas k3s[30501]: I0205 18:58:11.859333   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 18:58:11 truenas k3s[30501]: I0205 18:58:11.859416   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 18:58:11 truenas k3s[30501]: I0205 18:58:11.859447   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 18:58:11 truenas k3s[30501]: E0205 18:58:11.860704   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:58:13 truenas k3s[30501]: I0205 18:58:13.259256   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:58:13 truenas k3s[30501]: E0205 18:58:13.260268   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:58:14 truenas k3s[30501]: I0205 18:58:14.547897   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:58:14 truenas k3s[30501]: E0205 18:58:14.548503   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:58:15 truenas k3s[30501]: I0205 18:58:15.835300   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:58:15 truenas k3s[30501]: E0205 18:58:15.835903   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:58:17 truenas k3s[30501]: I0205 18:58:17.550123   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:58:17 truenas k3s[30501]: I0205 18:58:17.550174   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:58:17 truenas k3s[30501]: I0205 18:58:17.550197   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:58:17 truenas k3s[30501]: E0205 18:58:17.551638   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:58:19 truenas k3s[30501]: I0205 18:58:19.319802   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 18:58:19 truenas k3s[30501]: I0205 18:58:19.319864   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 18:58:19 truenas k3s[30501]: I0205 18:58:19.319891   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 18:58:19 truenas k3s[30501]: E0205 18:58:19.321677   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:58:20 truenas k3s[30501]: I0205 18:58:20.517660   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:58:20 truenas k3s[30501]: E0205 18:58:20.518054   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:58:21 truenas k3s[30501]: I0205 18:58:21.934064   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 18:58:21 truenas k3s[30501]: E0205 18:58:21.934781   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:58:23 truenas k3s[30501]: I0205 18:58:23.139953   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:58:23 truenas k3s[30501]: I0205 18:58:23.139994   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:58:23 truenas k3s[30501]: I0205 18:58:23.140017   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:58:23 truenas k3s[30501]: E0205 18:58:23.141226   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:58:24 truenas k3s[30501]: I0205 18:58:24.607125   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:58:24 truenas k3s[30501]: I0205 18:58:24.607173   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:58:24 truenas k3s[30501]: I0205 18:58:24.607196   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:58:24 truenas k3s[30501]: I0205 18:58:24.607216   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:58:24 truenas k3s[30501]: E0205 18:58:24.608748   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:58:25 truenas k3s[30501]: E0205 18:58:25.941543   30501 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find sandbox \"3e5d09fe3918bc55e3f605120b68ff215b5016ed4f4202dcac51a9632d27a605\": not found" podSandboxID="3e5d09fe3918bc55e3f605120b68ff215b5016ed4f4202dcac51a9632d27a605"
Feb 05 18:58:25 truenas k3s[30501]: E0205 18:58:25.942181   30501 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find sandbox \"84c94334eb3fb4b2014f1504466d34161656d433a53ee94f1a8525d9f6e9700f\": not found" podSandboxID="84c94334eb3fb4b2014f1504466d34161656d433a53ee94f1a8525d9f6e9700f"
Feb 05 18:58:27 truenas k3s[30501]: I0205 18:58:27.094896   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:58:27 truenas k3s[30501]: I0205 18:58:27.094951   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:58:27 truenas k3s[30501]: I0205 18:58:27.094973   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:58:27 truenas k3s[30501]: I0205 18:58:27.094993   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:58:27 truenas k3s[30501]: I0205 18:58:27.095015   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:58:27 truenas k3s[30501]: E0205 18:58:27.096639   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:58:27 truenas k3s[30501]: E0205 18:58:27.882254   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:58:28 truenas k3s[30501]: I0205 18:58:28.747494   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:58:28 truenas k3s[30501]: E0205 18:58:28.748070   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:58:30 truenas k3s[30501]: I0205 18:58:30.111339   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:58:30 truenas k3s[30501]: E0205 18:58:30.112341   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:58:31 truenas k3s[30501]: I0205 18:58:31.435897   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:58:31 truenas k3s[30501]: I0205 18:58:31.435950   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:58:31 truenas k3s[30501]: I0205 18:58:31.435975   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:58:31 truenas k3s[30501]: E0205 18:58:31.437422   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:58:32 truenas k3s[30501]: I0205 18:58:32.738986   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 18:58:32 truenas k3s[30501]: I0205 18:58:32.739043   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 18:58:32 truenas k3s[30501]: E0205 18:58:32.740314   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:58:33 truenas k3s[30501]: I0205 18:58:33.863780   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:58:33 truenas k3s[30501]: E0205 18:58:33.864395   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:58:35 truenas k3s[30501]: I0205 18:58:35.179110   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 18:58:35 truenas k3s[30501]: I0205 18:58:35.179164   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 18:58:35 truenas k3s[30501]: I0205 18:58:35.179186   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 18:58:35 truenas k3s[30501]: E0205 18:58:35.180541   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:58:36 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:58:36.081+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:58:36 truenas k3s[30501]: I0205 18:58:36.136832   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:58:36 truenas k3s[30501]: E0205 18:58:36.137695   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:58:37 truenas k3s[30501]: I0205 18:58:37.827893   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:58:37 truenas k3s[30501]: I0205 18:58:37.827943   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:58:37 truenas k3s[30501]: I0205 18:58:37.827965   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:58:37 truenas k3s[30501]: E0205 18:58:37.829792   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:58:39 truenas k3s[30501]: I0205 18:58:39.603281   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:58:39 truenas k3s[30501]: I0205 18:58:39.603345   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:58:39 truenas k3s[30501]: I0205 18:58:39.603402   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:58:39 truenas k3s[30501]: I0205 18:58:39.603438   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:58:39 truenas k3s[30501]: E0205 18:58:39.605684   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:58:40 truenas k3s[30501]: I0205 18:58:40.727596   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:58:40 truenas k3s[30501]: I0205 18:58:40.727653   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:58:40 truenas k3s[30501]: I0205 18:58:40.727675   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:58:40 truenas k3s[30501]: I0205 18:58:40.727695   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:58:40 truenas k3s[30501]: I0205 18:58:40.727715   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:58:40 truenas k3s[30501]: E0205 18:58:40.729305   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:58:41 truenas k3s[30501]: I0205 18:58:41.935239   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:58:43 truenas k3s[30501]: I0205 18:58:43.507461   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:58:43 truenas k3s[30501]: E0205 18:58:43.508312   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:58:47 truenas k3s[30501]: I0205 18:58:47.925657   30501 scope.go:115] "RemoveContainer" containerID="00615de5b0bf01f5e9a2dabd2fa5d2afe2a30b5ad98503ae06682955bb67a528"
Feb 05 18:58:48 truenas k3s[30501]: I0205 18:58:48.934664   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="14ea2b28061681cec48d95e6053b3e7a7f3c7bea8faf6bbd92d4c309fcb25565"
Feb 05 18:58:49 truenas k3s[30501]: I0205 18:58:49.323722   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:58:49 truenas k3s[30501]: E0205 18:58:49.324333   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:58:50 truenas k3s[30501]: I0205 18:58:50.787100   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:58:50 truenas k3s[30501]: I0205 18:58:50.787153   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:58:50 truenas k3s[30501]: I0205 18:58:50.787176   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:58:50 truenas k3s[30501]: E0205 18:58:50.788604   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:58:52 truenas k3s[30501]: I0205 18:58:52.277485   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 18:58:52 truenas k3s[30501]: I0205 18:58:52.277522   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 18:58:52 truenas k3s[30501]: E0205 18:58:52.278381   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:58:53 truenas k3s[30501]: I0205 18:58:53.577827   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:58:53 truenas k3s[30501]: E0205 18:58:53.578368   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:58:57 truenas k3s[30501]: I0205 18:58:57.399669   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 18:58:57 truenas k3s[30501]: I0205 18:58:57.399722   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 18:58:57 truenas k3s[30501]: I0205 18:58:57.399746   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 18:58:57 truenas k3s[30501]: E0205 18:58:57.401127   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:58:58 truenas k3s[30501]: I0205 18:58:58.409947   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:58:58 truenas k3s[30501]: I0205 18:58:58.410035   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:58:58 truenas k3s[30501]: I0205 18:58:58.410072   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:58:58 truenas k3s[30501]: E0205 18:58:58.412780   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:59:00 truenas k3s[30501]: I0205 18:59:00.039372   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:59:00 truenas k3s[30501]: I0205 18:59:00.039430   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:59:00 truenas k3s[30501]: I0205 18:59:00.039452   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:59:00 truenas k3s[30501]: I0205 18:59:00.039472   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:59:00 truenas k3s[30501]: E0205 18:59:00.041043   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:59:01 truenas k3s[30501]: I0205 18:59:01.431002   30501 scope.go:115] "RemoveContainer" containerID="11079882cd8b56a9ad15b07c4921d4c73e6521791e42c3eff3db82307340f594"
Feb 05 18:59:01 truenas k3s[30501]: I0205 18:59:01.436414   30501 scope.go:115] "RemoveContainer" containerID="9fe5e3098a48a548bf74fbd4c5ce152803131658dfe4b6cca34a18a1b5a5db4b"
Feb 05 18:59:03 truenas k3s[30501]: I0205 18:59:03.483149   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:59:03 truenas k3s[30501]: I0205 18:59:03.483197   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:59:03 truenas k3s[30501]: I0205 18:59:03.483220   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:59:03 truenas k3s[30501]: I0205 18:59:03.483243   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:59:03 truenas k3s[30501]: I0205 18:59:03.483263   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:59:03 truenas k3s[30501]: E0205 18:59:03.485325   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:59:05 truenas k3s[30501]: I0205 18:59:05.618949   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:59:05 truenas k3s[30501]: E0205 18:59:05.619549   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:59:06 truenas k3s[30501]: I0205 18:59:06.907826   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:59:06 truenas k3s[30501]: E0205 18:59:06.908718   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:59:10 truenas k3s[30501]: I0205 18:59:10.763390   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:59:10 truenas k3s[30501]: I0205 18:59:10.763462   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:59:10 truenas k3s[30501]: I0205 18:59:10.763495   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:59:10 truenas k3s[30501]: E0205 18:59:10.764940   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:59:12 truenas k3s[30501]: I0205 18:59:12.282788   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 18:59:12 truenas k3s[30501]: I0205 18:59:12.282848   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 18:59:12 truenas k3s[30501]: E0205 18:59:12.284378   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:59:13 truenas k3s[30501]: I0205 18:59:13.187179   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:59:13 truenas k3s[30501]: E0205 18:59:13.188052   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:59:15 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:59:15.143+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:59:16 truenas k3s[30501]: I0205 18:59:16.175116   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 18:59:16 truenas k3s[30501]: I0205 18:59:16.175168   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 18:59:16 truenas k3s[30501]: I0205 18:59:16.175193   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 18:59:16 truenas k3s[30501]: E0205 18:59:16.176453   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:59:17 truenas k3s[30501]: I0205 18:59:17.419258   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:59:17 truenas k3s[30501]: I0205 18:59:17.419313   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:59:17 truenas k3s[30501]: I0205 18:59:17.419337   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:59:17 truenas k3s[30501]: E0205 18:59:17.420783   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:59:18 truenas k3s[30501]: I0205 18:59:18.843330   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:59:18 truenas k3s[30501]: I0205 18:59:18.843409   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:59:18 truenas k3s[30501]: I0205 18:59:18.843431   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:59:18 truenas k3s[30501]: I0205 18:59:18.843455   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:59:18 truenas k3s[30501]: E0205 18:59:18.845000   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:59:21 truenas k3s[30501]: I0205 18:59:21.726739   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:59:21 truenas k3s[30501]: I0205 18:59:21.726776   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:59:21 truenas k3s[30501]: I0205 18:59:21.726792   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:59:21 truenas k3s[30501]: I0205 18:59:21.726806   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:59:21 truenas k3s[30501]: I0205 18:59:21.726822   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:59:21 truenas k3s[30501]: E0205 18:59:21.728085   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:59:22 truenas k3s[30501]: I0205 18:59:22.995329   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:59:23 truenas k3s[30501]: E0205 18:59:22.996011   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:59:25 truenas k3s[30501]: I0205 18:59:25.707319   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:59:25 truenas k3s[30501]: E0205 18:59:25.707890   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:59:29 truenas k3s[30501]: I0205 18:59:29.427412   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:59:29 truenas k3s[30501]: I0205 18:59:29.427465   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:59:29 truenas k3s[30501]: I0205 18:59:29.427487   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:59:29 truenas k3s[30501]: E0205 18:59:29.429027   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:59:31 truenas k3s[30501]: I0205 18:59:31.995144   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:59:32 truenas k3s[30501]: E0205 18:59:31.996047   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:59:33 truenas k3s[30501]: I0205 18:59:33.259016   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 18:59:33 truenas k3s[30501]: I0205 18:59:33.259073   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 18:59:33 truenas k3s[30501]: E0205 18:59:33.260294   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:59:34 truenas k3s[30501]: I0205 18:59:34.318504   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:59:34 truenas k3s[30501]: I0205 18:59:34.318535   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:59:34 truenas k3s[30501]: I0205 18:59:34.318551   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:59:34 truenas k3s[30501]: I0205 18:59:34.318561   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:59:34 truenas k3s[30501]: E0205 18:59:34.319287   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:59:35 truenas k3s[30501]: I0205 18:59:35.671951   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 18:59:35 truenas k3s[30501]: I0205 18:59:35.672002   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 18:59:35 truenas k3s[30501]: I0205 18:59:35.672025   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 18:59:35 truenas k3s[30501]: E0205 18:59:35.673518   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:59:36 truenas k3s[30501]: I0205 18:59:36.971382   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:59:36 truenas k3s[30501]: I0205 18:59:36.971438   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:59:36 truenas k3s[30501]: I0205 18:59:36.971461   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:59:36 truenas k3s[30501]: E0205 18:59:36.972878   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:59:38 truenas k3s[30501]: I0205 18:59:38.475106   30501 scope.go:115] "RemoveContainer" containerID="1b98c6fadd0d66c5d312640f0c7e8e85867e3b2249c12c5fae38916f2cb0b90e"
Feb 05 18:59:38 truenas k3s[30501]: E0205 18:59:38.476164   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:59:39 truenas k3s[30501]: I0205 18:59:39.565739   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:59:39 truenas k3s[30501]: I0205 18:59:39.566776   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:59:39 truenas k3s[30501]: I0205 18:59:39.566835   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:59:39 truenas k3s[30501]: I0205 18:59:39.566870   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:59:39 truenas k3s[30501]: I0205 18:59:39.566901   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:59:39 truenas k3s[30501]: E0205 18:59:39.569718   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:59:40 truenas k3s[30501]: I0205 18:59:40.743886   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:59:40 truenas k3s[30501]: E0205 18:59:40.744497   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:59:42 truenas k3s[30501]: I0205 18:59:42.209938   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:59:42 truenas k3s[30501]: E0205 18:59:42.210489   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:59:43 truenas k3s[30501]: I0205 18:59:43.423327   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:59:43 truenas k3s[30501]: I0205 18:59:43.423413   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:59:43 truenas k3s[30501]: I0205 18:59:43.423437   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:59:43 truenas k3s[30501]: E0205 18:59:43.425485   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:59:46 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T18:59:46.028+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 18:59:46 truenas k3s[30501]: I0205 18:59:46.374952   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:59:46 truenas k3s[30501]: E0205 18:59:46.375580   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 18:59:47 truenas k3s[30501]: I0205 18:59:47.963034   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 18:59:47 truenas k3s[30501]: I0205 18:59:47.963085   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 18:59:47 truenas k3s[30501]: E0205 18:59:47.964231   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 18:59:48 truenas k3s[30501]: I0205 18:59:48.915320   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 18:59:48 truenas k3s[30501]: I0205 18:59:48.915399   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 18:59:48 truenas k3s[30501]: I0205 18:59:48.915428   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 18:59:48 truenas k3s[30501]: E0205 18:59:48.916906   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 18:59:50 truenas k3s[30501]: I0205 18:59:50.707879   30501 scope.go:115] "RemoveContainer" containerID="1b98c6fadd0d66c5d312640f0c7e8e85867e3b2249c12c5fae38916f2cb0b90e"
Feb 05 18:59:50 truenas k3s[30501]: E0205 18:59:50.708849   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 18:59:52 truenas k3s[30501]: I0205 18:59:52.039235   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 18:59:52 truenas k3s[30501]: I0205 18:59:52.039283   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 18:59:52 truenas k3s[30501]: I0205 18:59:52.039305   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 18:59:52 truenas k3s[30501]: E0205 18:59:52.040652   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 18:59:53 truenas k3s[30501]: I0205 18:59:53.399399   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 18:59:53 truenas k3s[30501]: I0205 18:59:53.399466   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 18:59:53 truenas k3s[30501]: I0205 18:59:53.399502   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 18:59:53 truenas k3s[30501]: I0205 18:59:53.399535   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 18:59:53 truenas k3s[30501]: E0205 18:59:53.401408   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 18:59:54 truenas k3s[30501]: I0205 18:59:54.743777   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 18:59:54 truenas k3s[30501]: I0205 18:59:54.743850   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 18:59:54 truenas k3s[30501]: I0205 18:59:54.743875   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 18:59:54 truenas k3s[30501]: I0205 18:59:54.743920   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 18:59:54 truenas k3s[30501]: I0205 18:59:54.743942   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 18:59:54 truenas k3s[30501]: E0205 18:59:54.746067   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 18:59:55 truenas k3s[30501]: I0205 18:59:55.879021   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 18:59:55 truenas k3s[30501]: E0205 18:59:55.879666   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 18:59:56 truenas k3s[30501]: I0205 18:59:56.947739   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 18:59:56 truenas k3s[30501]: I0205 18:59:56.947806   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 18:59:56 truenas k3s[30501]: I0205 18:59:56.947834   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 18:59:56 truenas k3s[30501]: E0205 18:59:56.949389   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 18:59:58 truenas k3s[30501]: I0205 18:59:58.263099   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 18:59:58 truenas k3s[30501]: E0205 18:59:58.263782   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 18:59:59 truenas k3s[30501]: I0205 18:59:59.815918   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 18:59:59 truenas k3s[30501]: E0205 18:59:59.816775   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 19:00:01 truenas k3s[30501]: I0205 19:00:01.662917   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 19:00:01 truenas k3s[30501]: I0205 19:00:01.662981   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 19:00:01 truenas k3s[30501]: E0205 19:00:01.664408   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 19:00:02 truenas k3s[30501]: I0205 19:00:02.607447   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 19:00:02 truenas k3s[30501]: I0205 19:00:02.607497   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 19:00:02 truenas k3s[30501]: I0205 19:00:02.607518   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 19:00:02 truenas k3s[30501]: E0205 19:00:02.608908   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 19:00:03 truenas k3s[30501]: I0205 19:00:03.971215   30501 scope.go:115] "RemoveContainer" containerID="1b98c6fadd0d66c5d312640f0c7e8e85867e3b2249c12c5fae38916f2cb0b90e"
Feb 05 19:00:03 truenas k3s[30501]: E0205 19:00:03.972268   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 19:00:05 truenas k3s[30501]: I0205 19:00:05.743175   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 19:00:05 truenas k3s[30501]: I0205 19:00:05.743212   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 19:00:05 truenas k3s[30501]: I0205 19:00:05.743229   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 19:00:05 truenas k3s[30501]: E0205 19:00:05.744224   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 19:00:06 truenas k3s[30501]: I0205 19:00:06.799459   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 19:00:06 truenas k3s[30501]: I0205 19:00:06.799524   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 19:00:06 truenas k3s[30501]: I0205 19:00:06.799560   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 19:00:06 truenas k3s[30501]: I0205 19:00:06.799589   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 19:00:06 truenas k3s[30501]: E0205 19:00:06.801447   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 19:00:07 truenas k3s[30501]: I0205 19:00:07.750946   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 19:00:07 truenas k3s[30501]: I0205 19:00:07.751015   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 19:00:07 truenas k3s[30501]: I0205 19:00:07.751052   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 19:00:07 truenas k3s[30501]: I0205 19:00:07.751090   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 19:00:07 truenas k3s[30501]: I0205 19:00:07.751125   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 19:00:07 truenas k3s[30501]: E0205 19:00:07.752845   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 19:00:09 truenas k3s[30501]: I0205 19:00:09.790865   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 19:00:09 truenas k3s[30501]: I0205 19:00:09.790958   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 19:00:09 truenas k3s[30501]: I0205 19:00:09.790980   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 19:00:09 truenas k3s[30501]: E0205 19:00:09.792741   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 19:00:11 truenas k3s[30501]: I0205 19:00:11.835812   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 19:00:11 truenas k3s[30501]: E0205 19:00:11.836409   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 19:00:13 truenas k3s[30501]: I0205 19:00:13.811457   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 19:00:13 truenas k3s[30501]: E0205 19:00:13.812210   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 19:00:15 truenas k3s[30501]: I0205 19:00:15.027384   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 19:00:15 truenas k3s[30501]: I0205 19:00:15.027437   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 19:00:15 truenas k3s[30501]: E0205 19:00:15.028565   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 19:00:16 truenas k3s[30501]: I0205 19:00:16.383292   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 19:00:16 truenas k3s[30501]: I0205 19:00:16.383343   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 19:00:16 truenas k3s[30501]: I0205 19:00:16.383388   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 19:00:16 truenas k3s[30501]: E0205 19:00:16.384838   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 19:00:17 truenas k3s[30501]: I0205 19:00:17.699159   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 19:00:17 truenas k3s[30501]: E0205 19:00:17.700152   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 19:00:18 truenas k3s[30501]: I0205 19:00:18.915303   30501 scope.go:115] "RemoveContainer" containerID="1b98c6fadd0d66c5d312640f0c7e8e85867e3b2249c12c5fae38916f2cb0b90e"
Feb 05 19:00:18 truenas k3s[30501]: E0205 19:00:18.916299   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 19:00:19 truenas k3s[30501]: I0205 19:00:19.971278   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 19:00:19 truenas k3s[30501]: I0205 19:00:19.971308   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 19:00:19 truenas k3s[30501]: I0205 19:00:19.971317   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 19:00:20 truenas k3s[30501]: E0205 19:00:19.971914   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 19:00:21 truenas k3s[30501]: I0205 19:00:21.728011   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 19:00:21 truenas k3s[30501]: I0205 19:00:21.728062   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 19:00:21 truenas k3s[30501]: I0205 19:00:21.728090   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 19:00:21 truenas k3s[30501]: I0205 19:00:21.728113   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 19:00:21 truenas k3s[30501]: E0205 19:00:21.729668   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 19:00:23 truenas k3s[30501]: I0205 19:00:23.642832   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 19:00:23 truenas k3s[30501]: I0205 19:00:23.642885   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 19:00:23 truenas k3s[30501]: I0205 19:00:23.642921   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 19:00:23 truenas k3s[30501]: I0205 19:00:23.642944   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 19:00:23 truenas k3s[30501]: I0205 19:00:23.642965   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 19:00:23 truenas k3s[30501]: E0205 19:00:23.644660   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 19:00:23 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T19:00:23.795+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 19:00:25 truenas k3s[30501]: I0205 19:00:25.013704   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 19:00:25 truenas k3s[30501]: I0205 19:00:25.013734   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 19:00:25 truenas k3s[30501]: I0205 19:00:25.013748   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 19:00:25 truenas k3s[30501]: E0205 19:00:25.014693   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 19:00:26 truenas k3s[30501]: I0205 19:00:26.019123   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 19:00:26 truenas k3s[30501]: E0205 19:00:26.019723   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 19:00:27 truenas k3s[30501]: I0205 19:00:27.915790   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 19:00:32 truenas k3s[30501]: I0205 19:00:32.051173   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 19:00:32 truenas k3s[30501]: I0205 19:00:32.051220   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 19:00:32 truenas k3s[30501]: E0205 19:00:32.052391   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 19:00:32 truenas k3s[30501]: I0205 19:00:32.951212   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 19:00:32 truenas k3s[30501]: E0205 19:00:32.952080   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"coredns\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=coredns pod=coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)\"" pod="kube-system/coredns-59b4f5bbd5-7p628" podUID=586a341c-2cb0-4eb5-9f5d-690b594986b5
Feb 05 19:00:33 truenas k3s[30501]: I0205 19:00:33.996850   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 19:00:33 truenas k3s[30501]: I0205 19:00:33.996910   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 19:00:33 truenas k3s[30501]: I0205 19:00:33.996938   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 19:00:34 truenas k3s[30501]: E0205 19:00:33.998436   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 19:00:35 truenas k3s[30501]: I0205 19:00:35.363793   30501 scope.go:115] "RemoveContainer" containerID="f4ef8b9004a07ab889df309824a2a01915697502facf6bb6e3cc9dcac56f9730"
Feb 05 19:00:36 truenas k3s[30501]: I0205 19:00:36.651722   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 19:00:36 truenas k3s[30501]: I0205 19:00:36.651788   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 19:00:36 truenas k3s[30501]: I0205 19:00:36.651817   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 19:00:36 truenas k3s[30501]: E0205 19:00:36.653512   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 19:00:37 truenas k3s[30501]: I0205 19:00:37.951461   30501 scope.go:115] "RemoveContainer" containerID="1b98c6fadd0d66c5d312640f0c7e8e85867e3b2249c12c5fae38916f2cb0b90e"
Feb 05 19:00:37 truenas k3s[30501]: E0205 19:00:37.952441   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 19:00:39 truenas k3s[30501]: I0205 19:00:39.179178   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 19:00:39 truenas k3s[30501]: I0205 19:00:39.179255   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 19:00:39 truenas k3s[30501]: I0205 19:00:39.179296   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 19:00:39 truenas k3s[30501]: I0205 19:00:39.179317   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 19:00:39 truenas k3s[30501]: E0205 19:00:39.181044   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 19:00:40 truenas k3s[30501]: I0205 19:00:40.385620   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="7d7a2d979834c7bf97414ce29f8c57dd841385eda8443e21063c99636bb6fdaa"
Feb 05 19:00:40 truenas k3s[30501]: I0205 19:00:40.927076   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 19:00:40 truenas k3s[30501]: E0205 19:00:40.927639   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 19:00:42 truenas k3s[30501]: I0205 19:00:42.450960   30501 scope.go:115] "RemoveContainer" containerID="376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f"
Feb 05 19:00:42 truenas k3s[30501]: I0205 19:00:42.451010   30501 scope.go:115] "RemoveContainer" containerID="6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78"
Feb 05 19:00:42 truenas k3s[30501]: I0205 19:00:42.451032   30501 scope.go:115] "RemoveContainer" containerID="d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392"
Feb 05 19:00:42 truenas k3s[30501]: I0205 19:00:42.451052   30501 scope.go:115] "RemoveContainer" containerID="c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb"
Feb 05 19:00:42 truenas k3s[30501]: I0205 19:00:42.451071   30501 scope.go:115] "RemoveContainer" containerID="8226cbb67459232207651ba854446e78428c3f140adf44e92bd77375f67abe75"
Feb 05 19:00:43 truenas k3s[30501]: I0205 19:00:43.543202   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 19:00:43 truenas k3s[30501]: I0205 19:00:43.543250   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 19:00:43 truenas k3s[30501]: I0205 19:00:43.543309   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 19:00:43 truenas k3s[30501]: E0205 19:00:43.545056   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 19:00:45 truenas k3s[30501]: E0205 19:00:45.535849   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-resizer\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-resizer pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\", failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)\"]" pod="kube-system/openebs-zfs-controller-0" podUID=39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd
Feb 05 19:00:48 truenas k3s[30501]: I0205 19:00:48.195223   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 19:00:50 truenas k3s[30501]: I0205 19:00:50.127181   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 19:00:50 truenas k3s[30501]: I0205 19:00:50.127232   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 19:00:50 truenas k3s[30501]: I0205 19:00:50.127255   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 19:00:50 truenas k3s[30501]: E0205 19:00:50.128670   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 19:00:53 truenas k3s[30501]: I0205 19:00:53.442890   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 19:00:53 truenas k3s[30501]: I0205 19:00:53.442951   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 19:00:53 truenas k3s[30501]: E0205 19:00:53.444426   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 19:00:55 truenas k3s[30501]: I0205 19:00:55.461206   30501 scope.go:115] "RemoveContainer" containerID="40be0bfea9c900e7b904d61826cfd582d4f1a8cac0fc4d675c4529663310c1de"
Feb 05 19:00:57 truenas k3s[30501]: E0205 19:00:57.975244   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 19:00:58 truenas k3s[30501]: I0205 19:00:58.590101   30501 scope.go:115] "RemoveContainer" containerID="6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf"
Feb 05 19:00:58 truenas k3s[30501]: I0205 19:00:58.590152   30501 scope.go:115] "RemoveContainer" containerID="c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca"
Feb 05 19:00:58 truenas k3s[30501]: I0205 19:00:58.590176   30501 scope.go:115] "RemoveContainer" containerID="5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285"
Feb 05 19:00:58 truenas k3s[30501]: E0205 19:00:58.591460   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)\"]" pod="kube-system/csi-smb-controller-7fbbb8fb6f-khgtn" podUID=e2e5d854-ce9d-473b-a278-0750f97de2c9
Feb 05 19:00:59 truenas k3s[30501]: I0205 19:00:59.843891   30501 scope.go:115] "RemoveContainer" containerID="1b98c6fadd0d66c5d312640f0c7e8e85867e3b2249c12c5fae38916f2cb0b90e"
Feb 05 19:00:59 truenas k3s[30501]: E0205 19:00:59.844883   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)\"" pod="ix-prometheus/prometheus-645557855-89jrk" podUID=732278f4-3b22-4568-839f-00e4f5fc629f
Feb 05 19:01:00 truenas k3s[30501]: {"level":"warn","ts":"2024-02-05T19:01:00.504+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001976fc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 05 19:01:01 truenas k3s[30501]: I0205 19:01:01.243396   30501 scope.go:115] "RemoveContainer" containerID="fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea"
Feb 05 19:01:01 truenas k3s[30501]: E0205 19:01:01.244289   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)\"" pod="kube-system/snapshot-controller-546868dfb4-q67gk" podUID=8f029e33-1378-43ec-9400-7d11b73b870e
Feb 05 19:01:01 truenas k3s[30501]: I0205 19:01:01.494865   30501 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="0389b7ace9a5e565b9150cfeaa078d6ce40209dea8a19941d6c8712675521342"
Feb 05 19:01:02 truenas k3s[30501]: I0205 19:01:02.587151   30501 scope.go:115] "RemoveContainer" containerID="94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed"
Feb 05 19:01:02 truenas k3s[30501]: I0205 19:01:02.587196   30501 scope.go:115] "RemoveContainer" containerID="428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327"
Feb 05 19:01:02 truenas k3s[30501]: I0205 19:01:02.587218   30501 scope.go:115] "RemoveContainer" containerID="df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2"
Feb 05 19:01:02 truenas k3s[30501]: I0205 19:01:02.587238   30501 scope.go:115] "RemoveContainer" containerID="68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d"
Feb 05 19:01:02 truenas k3s[30501]: E0205 19:01:02.588799   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-provisioner\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-provisioner pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"csi-snapshotter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-snapshotter pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)\"]" pod="kube-system/csi-nfs-controller-7b74694749-wctvk" podUID=37a40dae-33a2-4237-ac94-71da566d2ed1
Feb 05 19:01:03 truenas k3s[30501]: I0205 19:01:03.891986   30501 scope.go:115] "RemoveContainer" containerID="bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388"
Feb 05 19:01:03 truenas k3s[30501]: I0205 19:01:03.892056   30501 scope.go:115] "RemoveContainer" containerID="2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0"
Feb 05 19:01:03 truenas k3s[30501]: I0205 19:01:03.892079   30501 scope.go:115] "RemoveContainer" containerID="d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f"
Feb 05 19:01:03 truenas k3s[30501]: E0205 19:01:03.893504   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\", failed to \"StartContainer\" for \"nfs\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nfs pod=csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)\"]" pod="kube-system/csi-nfs-node-9f4hq" podUID=945e6d80-a04f-4842-ac56-5ed0b16f8f7e
Feb 05 19:01:05 truenas k3s[30501]: I0205 19:01:05.111175   30501 scope.go:115] "RemoveContainer" containerID="e9515990dd1678be1cdd43d542c32461c396a13c674d1e421e9736aea5acff51"
Feb 05 19:01:05 truenas k3s[30501]: E0205 19:01:05.111776   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
Feb 05 19:01:07 truenas k3s[30501]: I0205 19:01:07.347380   30501 scope.go:115] "RemoveContainer" containerID="162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347"
Feb 05 19:01:07 truenas k3s[30501]: I0205 19:01:07.347437   30501 scope.go:115] "RemoveContainer" containerID="c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776"
Feb 05 19:01:07 truenas k3s[30501]: I0205 19:01:07.347466   30501 scope.go:115] "RemoveContainer" containerID="70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3"
Feb 05 19:01:07 truenas k3s[30501]: E0205 19:01:07.349114   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"liveness-probe\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=liveness-probe pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=node-driver-registrar pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\", failed to \"StartContainer\" for \"smb\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=smb pod=csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)\"]" pod="kube-system/csi-smb-node-knr62" podUID=a6c45698-02bc-438c-ab08-e2e6fb148572
Feb 05 19:01:08 truenas k3s[30501]: I0205 19:01:08.446622   30501 scope.go:115] "RemoveContainer" containerID="1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d"
Feb 05 19:01:08 truenas k3s[30501]: I0205 19:01:08.446672   30501 scope.go:115] "RemoveContainer" containerID="b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636"
Feb 05 19:01:08 truenas k3s[30501]: E0205 19:01:08.447898   30501 pod_workers.go:965] "Error syncing pod, skipping" err="[failed to \"StartContainer\" for \"csi-node-driver-registrar\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=csi-node-driver-registrar pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\", failed to \"StartContainer\" for \"openebs-zfs-plugin\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=openebs-zfs-plugin pod=openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)\"]" pod="kube-system/openebs-zfs-node-4cm89" podUID=63ec6a96-53e9-463e-89a1-63c509edd807
Feb 05 19:01:09 truenas k3s[30501]: I0205 19:01:09.902918   30501 scope.go:115] "RemoveContainer" containerID="e9515990dd1678be1cdd43d542c32461c396a13c674d1e421e9736aea5acff51"
Feb 05 19:01:09 truenas k3s[30501]: E0205 19:01:09.903542   30501 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"snapshot-controller\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=snapshot-controller pod=snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)\"" pod="kube-system/snapshot-controller-546868dfb4-6jclc" podUID=5b7ea80f-63fa-4f9e-975f-c73af32fb501
