
------------------------------
1) Describe 'Pods' resources
------------------------------

Name:                 csi-nfs-node-9f4hq
Namespace:            kube-system
Priority:             2000001000
Priority Class Name:  system-node-critical
Service Account:      csi-nfs-node-sa
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               app=csi-nfs-node
                      controller-revision-hash=598688b946
                      pod-template-generation=1
Annotations:          <none>
Status:               Running
IP:                   192.168.178.45
IPs:
  IP:           192.168.178.45
Controlled By:  DaemonSet/csi-nfs-node
Containers:
  liveness-probe:
    Container ID:  containerd://bbe8ec690abdd0b25901d3927b6018ea890dfb67bd43eb7e986b6851dd4e1388
    Image:         registry.k8s.io/sig-storage/livenessprobe:v2.10.0
    Image ID:      registry.k8s.io/sig-storage/livenessprobe@sha256:4dc0b87ccd69f9865b89234d8555d3a614ab0a16ed94a3016ffd27f8106132ce
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=/csi/csi.sock
      --probe-timeout=3s
      --health-port=29653
      --v=2
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:06 +0100
      Finished:     Mon, 05 Feb 2024 18:56:24 +0100
    Ready:          False
    Restart Count:  230
    Limits:
      memory:  100Mi
    Requests:
      cpu:        10m
      memory:     20Mi
    Environment:  <none>
    Mounts:
      /csi from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-4298v (ro)
  node-driver-registrar:
    Container ID:  containerd://2305bae79243a8e92fc322685b24b1ccb889ce1da0f9fd61f6f65cbaaeb464a0
    Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
    Image ID:      registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:f6717ce72a2615c7fbc746b4068f788e78579c54c43b8716e5ce650d97af2df1
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=2
      --csi-address=/csi/csi.sock
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:13 +0100
      Finished:     Mon, 05 Feb 2024 18:56:24 +0100
    Ready:          False
    Restart Count:  230
    Limits:
      memory:  100Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Liveness:  exec [/csi-node-driver-registrar --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) --mode=kubelet-registration-probe] delay=30s timeout=15s period=10s #success=1 #failure=3
    Environment:
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
    Mounts:
      /csi from socket-dir (rw)
      /registration from registration-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-4298v (ro)
  nfs:
    Container ID:  containerd://d9b5c5d8f41fb9ff215b48f3f22d7909ef8e3cd6991336c707349e2798c8787f
    Image:         registry.k8s.io/sig-storage/nfsplugin:v4.4.0
    Image ID:      registry.k8s.io/sig-storage/nfsplugin@sha256:971cb526bc0108cca7d8efda32102629cb362f99bbd4879fd381bea1b2014ab4
    Port:          29653/TCP
    Host Port:     29653/TCP
    Args:
      -v=5
      --nodeid=$(NODE_ID)
      --endpoint=$(CSI_ENDPOINT)
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:23 +0100
      Finished:     Mon, 05 Feb 2024 18:56:24 +0100
    Ready:          False
    Restart Count:  230
    Limits:
      memory:  300Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Liveness:  http-get http://:healthz/healthz delay=30s timeout=10s period=30s #success=1 #failure=5
    Environment:
      NODE_ID:        (v1:spec.nodeName)
      CSI_ENDPOINT:  unix:///csi/csi.sock
    Mounts:
      /csi from socket-dir (rw)
      /var/lib/kubelet/pods from pods-mount-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-4298v (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/csi-nfsplugin
    HostPathType:  DirectoryOrCreate
  pods-mount-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/pods
    HostPathType:  Directory
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry
    HostPathType:  Directory
  kube-api-access-4298v:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 op=Exists
                             node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
  Type     Reason     Age                     From     Message
  ----     ------     ----                    ----     -------
  Warning  Unhealthy  53m (x66 over 20h)      kubelet  Liveness probe errored: rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state
  Normal   Started    48m (x223 over 20h)     kubelet  Started container liveness-probe
  Warning  BackOff    3m27s (x3741 over 20h)  kubelet  Back-off restarting failed container liveness-probe in pod csi-nfs-node-9f4hq_kube-system(945e6d80-a04f-4842-ac56-5ed0b16f8f7e)


Name:                 csi-nfs-controller-7b74694749-wctvk
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      csi-nfs-controller-sa
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               app=csi-nfs-controller
                      pod-template-hash=7b74694749
Annotations:          <none>
Status:               Running
IP:                   192.168.178.45
IPs:
  IP:           192.168.178.45
Controlled By:  ReplicaSet/csi-nfs-controller-7b74694749
Containers:
  csi-provisioner:
    Container ID:  containerd://94416e99b501a45821a82a967dcc625262219e954a54cd496f5b60fb49619bed
    Image:         registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
    Image ID:      registry.k8s.io/sig-storage/csi-provisioner@sha256:d078dc174323407e8cc6f0f9abd4efaac5db27838f1564d0253d5e3233e3f17f
    Port:          <none>
    Host Port:     <none>
    Args:
      -v=2
      --csi-address=$(ADDRESS)
      --leader-election
      --leader-election-namespace=kube-system
      --extra-create-metadata=true
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:02 +0100
      Finished:     Mon, 05 Feb 2024 18:56:21 +0100
    Ready:          False
    Restart Count:  229
    Limits:
      memory:  400Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Environment:
      ADDRESS:  /csi/csi.sock
    Mounts:
      /csi from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-jkzc5 (ro)
  csi-snapshotter:
    Container ID:  containerd://428e2deed4a9a43424a95424ddbbb5f040ef920b9c5de6ce00e3733c8123e327
    Image:         registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2
    Image ID:      registry.k8s.io/sig-storage/csi-snapshotter@sha256:becc53e25b96573f61f7469923a92fb3e9d3a3781732159954ce0d9da07233a2
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=2
      --csi-address=$(ADDRESS)
      --leader-election-namespace=kube-system
      --leader-election
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:07 +0100
      Finished:     Mon, 05 Feb 2024 18:56:21 +0100
    Ready:          False
    Restart Count:  229
    Limits:
      memory:  200Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Environment:
      ADDRESS:  /csi/csi.sock
    Mounts:
      /csi from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-jkzc5 (ro)
  liveness-probe:
    Container ID:  containerd://df81745ff3d9c3e26bb11d73c6aea46e4dd562db3810d2751d8fe2a3b51008e2
    Image:         registry.k8s.io/sig-storage/livenessprobe:v2.10.0
    Image ID:      registry.k8s.io/sig-storage/livenessprobe@sha256:4dc0b87ccd69f9865b89234d8555d3a614ab0a16ed94a3016ffd27f8106132ce
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=/csi/csi.sock
      --probe-timeout=3s
      --health-port=29652
      --v=2
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:15 +0100
      Finished:     Mon, 05 Feb 2024 18:56:21 +0100
    Ready:          False
    Restart Count:  229
    Limits:
      memory:  100Mi
    Requests:
      cpu:        10m
      memory:     20Mi
    Environment:  <none>
    Mounts:
      /csi from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-jkzc5 (ro)
  nfs:
    Container ID:  containerd://68718cb4d4118ae80bddb050f7a29aef50f5745a8be67a68f63e1acb16e7461d
    Image:         registry.k8s.io/sig-storage/nfsplugin:v4.4.0
    Image ID:      registry.k8s.io/sig-storage/nfsplugin@sha256:971cb526bc0108cca7d8efda32102629cb362f99bbd4879fd381bea1b2014ab4
    Port:          29652/TCP
    Host Port:     29652/TCP
    Args:
      -v=5
      --nodeid=$(NODE_ID)
      --endpoint=$(CSI_ENDPOINT)
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:20 +0100
      Finished:     Mon, 05 Feb 2024 18:56:21 +0100
    Ready:          False
    Restart Count:  229
    Limits:
      memory:  200Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Liveness:  http-get http://:healthz/healthz delay=30s timeout=10s period=30s #success=1 #failure=5
    Environment:
      NODE_ID:        (v1:spec.nodeName)
      CSI_ENDPOINT:  unix:///csi/csi.sock
    Mounts:
      /csi from socket-dir (rw)
      /var/lib/kubelet/pods from pods-mount-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-jkzc5 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  pods-mount-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/pods
    HostPathType:  Directory
  socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-jkzc5:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                             node-role.kubernetes.io/controlplane:NoSchedule op=Exists
                             node-role.kubernetes.io/master:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age                   From     Message
  ----     ------     ----                  ----     -------
  Warning  Unhealthy  15m (x19 over 20h)    kubelet  Liveness probe failed: Get "http://192.168.178.45:29652/healthz": dial tcp 192.168.178.45:29652: connect: connection refused
  Normal   Started    10m (x229 over 20h)   kubelet  Started container nfs
  Warning  BackOff    22s (x3669 over 20h)  kubelet  Back-off restarting failed container csi-provisioner in pod csi-nfs-controller-7b74694749-wctvk_kube-system(37a40dae-33a2-4237-ac94-71da566d2ed1)


Name:                 csi-smb-node-knr62
Namespace:            kube-system
Priority:             2000001000
Priority Class Name:  system-node-critical
Service Account:      csi-smb-node-sa
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               app=csi-smb-node
                      controller-revision-hash=754c5545b6
                      pod-template-generation=1
Annotations:          <none>
Status:               Running
IP:                   192.168.178.45
IPs:
  IP:           192.168.178.45
Controlled By:  DaemonSet/csi-smb-node
Containers:
  liveness-probe:
    Container ID:  containerd://162f32bc2de7eeb8022d1b7f70943dfcb2ced9025d3fc58298589881e01ca347
    Image:         registry.k8s.io/sig-storage/livenessprobe:v2.10.0
    Image ID:      registry.k8s.io/sig-storage/livenessprobe@sha256:4dc0b87ccd69f9865b89234d8555d3a614ab0a16ed94a3016ffd27f8106132ce
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=/csi/csi.sock
      --probe-timeout=3s
      --health-port=29643
      --v=2
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:32 +0100
      Finished:     Mon, 05 Feb 2024 18:56:39 +0100
    Ready:          False
    Restart Count:  229
    Limits:
      memory:  100Mi
    Requests:
      cpu:        10m
      memory:     20Mi
    Environment:  <none>
    Mounts:
      /csi from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6fl4x (ro)
  node-driver-registrar:
    Container ID:  containerd://c2e6e88f9531a917d89da93cc50182d00720d7698d437aa61eb4022632137776
    Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
    Image ID:      registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:f6717ce72a2615c7fbc746b4068f788e78579c54c43b8716e5ce650d97af2df1
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
      --v=2
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:36 +0100
      Finished:     Mon, 05 Feb 2024 18:56:39 +0100
    Ready:          False
    Restart Count:  229
    Limits:
      memory:  100Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Liveness:  exec [/csi-node-driver-registrar --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) --mode=kubelet-registration-probe] delay=30s timeout=15s period=10s #success=1 #failure=3
    Environment:
      ADDRESS:               /csi/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/smb.csi.k8s.io/csi.sock
    Mounts:
      /csi from socket-dir (rw)
      /registration from registration-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6fl4x (ro)
  smb:
    Container ID:  containerd://70f9f92d15aac7745ad2105e1028e25fb10f2d0f25c50d34c5d9273b388fd2d3
    Image:         registry.k8s.io/sig-storage/smbplugin:v1.11.0
    Image ID:      registry.k8s.io/sig-storage/smbplugin@sha256:65f664c6e9cce565805b13fb82b40018d56e39f41616134f572c52f3964c6ee4
    Port:          29643/TCP
    Host Port:     29643/TCP
    Args:
      --v=5
      --endpoint=$(CSI_ENDPOINT)
      --nodeid=$(KUBE_NODE_NAME)
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:39 +0100
      Finished:     Mon, 05 Feb 2024 18:56:39 +0100
    Ready:          False
    Restart Count:  229
    Limits:
      memory:  200Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Liveness:  http-get http://:healthz/healthz delay=30s timeout=10s period=30s #success=1 #failure=5
    Environment:
      CSI_ENDPOINT:    unix:///csi/csi.sock
      KUBE_NODE_NAME:   (v1:spec.nodeName)
    Mounts:
      /csi from socket-dir (rw)
      /var/lib/kubelet/ from mountpoint-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6fl4x (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/smb.csi.k8s.io
    HostPathType:  DirectoryOrCreate
  mountpoint-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/
    HostPathType:  DirectoryOrCreate
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
  kube-api-access-6fl4x:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 op=Exists
                             node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
  Type     Reason     Age                     From     Message
  ----     ------     ----                    ----     -------
  Warning  Unhealthy  58m (x99 over 20h)      kubelet  Liveness probe errored: rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state
  Normal   Pulled     53m (x220 over 20h)     kubelet  Container image "registry.k8s.io/sig-storage/livenessprobe:v2.10.0" already present on machine
  Warning  BackOff    3m18s (x3722 over 20h)  kubelet  Back-off restarting failed container liveness-probe in pod csi-smb-node-knr62_kube-system(a6c45698-02bc-438c-ab08-e2e6fb148572)


Name:                 snapshot-controller-546868dfb4-q67gk
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      snapshot-controller
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               app=snapshot-controller
                      pod-template-hash=546868dfb4
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.5.107"
                            ],
                            "mac": "da:a5:ec:80:2b:75",
                            "default": true,
                            "dns": {},
                            "gateway": [
                                "172.16.0.1"
                            ]
                        }]
Status:               Running
IP:                   172.16.5.107
IPs:
  IP:           172.16.5.107
Controlled By:  ReplicaSet/snapshot-controller-546868dfb4
Containers:
  snapshot-controller:
    Container ID:  containerd://fdc3db34638cd612fc87d47c3a53c509993a69540688cb2cc4246ceb51b880ea
    Image:         registry.k8s.io/sig-storage/snapshot-controller:v6.2.2
    Image ID:      registry.k8s.io/sig-storage/snapshot-controller@sha256:fb95b65bb88f319f0f7d5397c401a654164f11a191f466b4026fa36085c7141b
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=2
      --leader-election=true
      --leader-election-namespace=kube-system
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:51 +0100
      Finished:     Mon, 05 Feb 2024 18:56:52 +0100
    Ready:          False
    Restart Count:  238
    Limits:
      memory:  300Mi
    Requests:
      cpu:        10m
      memory:     20Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-sd7vm (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  kube-api-access-sd7vm:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node-role.kubernetes.io/control-plane=true:NoSchedule
                             node-role.kubernetes.io/controlplane=true:NoSchedule
                             node-role.kubernetes.io/master=true:NoSchedule
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason          Age                   From     Message
  ----     ------          ----                  ----     -------
  Normal   AddedInterface  57m                   multus   Add eth0 [172.16.5.46/16] from ix-net
  Normal   AddedInterface  52m                   multus   Add eth0 [172.16.5.53/16] from ix-net
  Normal   AddedInterface  46m                   multus   Add eth0 [172.16.5.59/16] from ix-net
  Normal   AddedInterface  41m                   multus   Add eth0 [172.16.5.65/16] from ix-net
  Normal   AddedInterface  36m                   multus   Add eth0 [172.16.5.71/16] from ix-net
  Normal   SandboxChanged  30m (x234 over 20h)   kubelet  Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface  30m                   multus   Add eth0 [172.16.5.77/16] from ix-net
  Normal   Pulled          26m (x234 over 20h)   kubelet  Container image "registry.k8s.io/sig-storage/snapshot-controller:v6.2.2" already present on machine
  Normal   AddedInterface  25m                   multus   Add eth0 [172.16.5.83/16] from ix-net
  Normal   AddedInterface  19m                   multus   Add eth0 [172.16.5.89/16] from ix-net
  Normal   AddedInterface  14m                   multus   Add eth0 [172.16.5.95/16] from ix-net
  Normal   AddedInterface  9m20s                 multus   Add eth0 [172.16.5.101/16] from ix-net
  Normal   AddedInterface  3m47s                 multus   Add eth0 [172.16.5.107/16] from ix-net
  Warning  BackOff         73s (x3871 over 20h)  kubelet  Back-off restarting failed container snapshot-controller in pod snapshot-controller-546868dfb4-q67gk_kube-system(8f029e33-1378-43ec-9400-7d11b73b870e)


Name:                 csi-smb-controller-7fbbb8fb6f-khgtn
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      csi-smb-controller-sa
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               app=csi-smb-controller
                      pod-template-hash=7fbbb8fb6f
Annotations:          <none>
Status:               Running
IP:                   192.168.178.45
IPs:
  IP:           192.168.178.45
Controlled By:  ReplicaSet/csi-smb-controller-7fbbb8fb6f
Containers:
  csi-provisioner:
    Container ID:  containerd://6668a7e26d53d4d95e793100401626d0d5bc35318c89fbaba6db11fde7e4ccaf
    Image:         registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
    Image ID:      registry.k8s.io/sig-storage/csi-provisioner@sha256:d078dc174323407e8cc6f0f9abd4efaac5db27838f1564d0253d5e3233e3f17f
    Port:          <none>
    Host Port:     <none>
    Args:
      -v=2
      --csi-address=$(ADDRESS)
      --leader-election
      --leader-election-namespace=kube-system
      --extra-create-metadata=true
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:57:10 +0100
      Finished:     Mon, 05 Feb 2024 18:57:20 +0100
    Ready:          False
    Restart Count:  230
    Limits:
      cpu:     1
      memory:  300Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Environment:
      ADDRESS:  /csi/csi.sock
    Mounts:
      /csi from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snlcz (ro)
  liveness-probe:
    Container ID:  containerd://c77b599b9b65c027a2c8425f877317af4645d413fef1b20607614cd522f8faca
    Image:         registry.k8s.io/sig-storage/livenessprobe:v2.10.0
    Image ID:      registry.k8s.io/sig-storage/livenessprobe@sha256:4dc0b87ccd69f9865b89234d8555d3a614ab0a16ed94a3016ffd27f8106132ce
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=/csi/csi.sock
      --probe-timeout=3s
      --health-port=29642
      --v=2
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:57:15 +0100
      Finished:     Mon, 05 Feb 2024 18:57:20 +0100
    Ready:          False
    Restart Count:  230
    Limits:
      cpu:     1
      memory:  100Mi
    Requests:
      cpu:        10m
      memory:     20Mi
    Environment:  <none>
    Mounts:
      /csi from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snlcz (ro)
  smb:
    Container ID:  containerd://5baf53a9df83a473dc36868d5bb7371539ab6f7d889f03e0b2d608dfbf5cb285
    Image:         registry.k8s.io/sig-storage/smbplugin:v1.11.0
    Image ID:      registry.k8s.io/sig-storage/smbplugin@sha256:65f664c6e9cce565805b13fb82b40018d56e39f41616134f572c52f3964c6ee4
    Ports:         29642/TCP, 29644/TCP
    Host Ports:    29642/TCP, 29644/TCP
    Args:
      --v=5
      --endpoint=$(CSI_ENDPOINT)
      --metrics-address=0.0.0.0:29644
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:57:20 +0100
      Finished:     Mon, 05 Feb 2024 18:57:20 +0100
    Ready:          False
    Restart Count:  230
    Limits:
      memory:  200Mi
    Requests:
      cpu:     10m
      memory:  20Mi
    Liveness:  http-get http://:healthz/healthz delay=30s timeout=10s period=30s #success=1 #failure=5
    Environment:
      CSI_ENDPOINT:  unix:///csi/csi.sock
    Mounts:
      /csi from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snlcz (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-snlcz:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                             node-role.kubernetes.io/controlplane:NoSchedule op=Exists
                             node-role.kubernetes.io/master:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason          Age                   From     Message
  ----     ------          ----                  ----     -------
  Normal   SandboxChanged  25m (x227 over 20h)   kubelet  Pod sandbox changed, it will be killed and re-created.
  Normal   Pulled          20m (x227 over 20h)   kubelet  Container image "registry.k8s.io/sig-storage/csi-provisioner:v3.5.0" already present on machine
  Warning  BackOff         26s (x3691 over 20h)  kubelet  Back-off restarting failed container csi-provisioner in pod csi-smb-controller-7fbbb8fb6f-khgtn_kube-system(e2e5d854-ce9d-473b-a278-0750f97de2c9)


Name:                 openebs-zfs-node-4cm89
Namespace:            kube-system
Priority:             900001000
Priority Class Name:  openebs-zfs-csi-node-critical
Service Account:      openebs-zfs-node-sa
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               app=openebs-zfs-node
                      controller-revision-hash=6456c99fd9
                      openebs.io/component-name=openebs-zfs-node
                      openebs.io/version=2.3.0
                      pod-template-generation=1
                      role=openebs-zfs
Annotations:          <none>
Status:               Running
IP:                   192.168.178.45
IPs:
  IP:           192.168.178.45
Controlled By:  DaemonSet/openebs-zfs-node
Containers:
  csi-node-driver-registrar:
    Container ID:  containerd://1cfb7ad4f773ce602d4a6c5865e9e62fa07bb442a2e8f9b33904070181d92c7d
    Image:         registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
    Image ID:      registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:f6717ce72a2615c7fbc746b4068f788e78579c54c43b8716e5ce650d97af2df1
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:58:07 +0100
      Finished:     Mon, 05 Feb 2024 18:58:08 +0100
    Ready:          False
    Restart Count:  221
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-hqgx2 (ro)
  openebs-zfs-plugin:
    Container ID:  containerd://b163a4fb54356148963003394256477f1244e11490086fe2ce30a7a562935636
    Image:         openebs/zfs-driver:2.3.0
    Image ID:      docker.io/openebs/zfs-driver@sha256:42b8d31edc7cdf4116ecd6369f89a1d477bb9b64e695d074d290789af1a461dd
    Port:          <none>
    Host Port:     <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    137
      Started:      Mon, 05 Feb 2024 18:57:03 +0100
      Finished:     Mon, 05 Feb 2024 18:57:34 +0100
    Ready:          False
    Restart Count:  216
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-hqgx2 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
  encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
  chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
  host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
  plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
  pods-mount-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/
    HostPathType:  Directory
  kube-api-access-hqgx2:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
  Type     Reason          Age                     From     Message
  ----     ------          ----                    ----     -------
  Warning  BackOff         43m (x3386 over 20h)    kubelet  Back-off restarting failed container openebs-zfs-plugin in pod openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)
  Normal   Pulled          38m (x374 over 20h)     kubelet  Container image "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0" already present on machine
  Warning  BackOff         8m21s (x3313 over 20h)  kubelet  Back-off restarting failed container csi-node-driver-registrar in pod openebs-zfs-node-4cm89_kube-system(63ec6a96-53e9-463e-89a1-63c509edd807)
  Normal   SandboxChanged  3m26s (x429 over 20h)   kubelet  Pod sandbox changed, it will be killed and re-created.


Name:             prometheus-645557855-89jrk
Namespace:        ix-prometheus
Priority:         0
Service Account:  default
Node:             ix-truenas/192.168.178.45
Start Time:       Sun, 04 Feb 2024 22:22:43 +0100
Labels:           app=prometheus-1.1.9
                  app.kubernetes.io/instance=prometheus
                  app.kubernetes.io/managed-by=Helm
                  app.kubernetes.io/name=prometheus
                  app.kubernetes.io/version=v2.49.1
                  helm-revision=1
                  helm.sh/chart=prometheus-1.1.9
                  pod-template-hash=645557855
                  pod.name=prometheus
                  release=prometheus
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.108"
                        ],
                        "mac": "ca:67:7e:3d:54:bf",
                        "default": true,
                        "dns": {},
                        "gateway": [
                            "172.16.0.1"
                        ]
                    }]
                  rollme: 8i1wn
Status:           Running
IP:               172.16.5.108
IPs:
  IP:           172.16.5.108
Controlled By:  ReplicaSet/prometheus-645557855
Init Containers:
  prometheus-install-01-permissions:
    Container ID:  containerd://4488356ce1e56a72b7708f544f4f20969cc7f07de4e4509e27a0fb21478671dc
    Image:         bash:4.4.23
    Image ID:      docker.io/library/bash@sha256:b131958aa51b95bf71f92e3c11a3f9d5233db25ffad1bc3241866e7f59766d90
    Port:          <none>
    Host Port:     <none>
    Command:
      bash
    Args:
      -c
      for dir in /mnt/directories/*; do
        if [ ! -d "$dir" ]; then
          echo "[$dir] is not a directory, skipping"
          continue
        fi
      
        echo "Current Ownership and Permissions on ["$dir"]:"
        echo "chown: $(stat -c "%u %g" "$dir")"
        echo "chmod: $(stat -c "%a" "$dir")" 
        if [ $(stat -c %u "$dir") -eq 568 ] && [ $(stat -c %g "$dir") -eq 568 ]; then
          echo "Ownership is correct. Skipping..."
          fix_owner="false"
        else
          echo "Ownership is incorrect. Fixing..."
          fix_owner="true"
        fi
      
      
        if [ "$fix_owner" = "true" ]; then
          echo "Changing ownership to 568:568 on: ["$dir"]"
          chown -R 568:568 "$dir"
          echo "Finished changing ownership"
          echo "Ownership after changes:"
          stat -c "%u %g" "$dir"
        fi
      done
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 05 Feb 2024 18:59:09 +0100
      Finished:     Mon, 05 Feb 2024 18:59:09 +0100
    Ready:          True
    Restart Count:  1
    Limits:
      cpu:     1
      memory:  512Mi
    Requests:
      cpu:     10m
      memory:  50Mi
    Environment:
      TZ:                      UTC
      UMASK:                   002
      UMASK_SET:               002
      NVIDIA_VISIBLE_DEVICES:  void
      PUID:                    568
      USER_ID:                 568
      UID:                     568
      PGID:                    568
      GROUP_ID:                568
      GID:                     568
    Mounts:
      /mnt/directories/config from config (rw)
      /mnt/directories/data from data (rw)
  prometheus-init-init-config:
    Container ID:  containerd://89ef8bb3ade88be3b4516d6f590f73e5c6df8ffef006773013c169676ec153c0
    Image:         prom/prometheus:v2.49.1
    Image ID:      docker.io/prom/prometheus@sha256:beb5e30ffba08d9ae8a7961b9a2145fc8af6296ff2a4f463df7cd722fcbfc789
    Port:          <none>
    Host Port:     <none>
    Command:
      sh
    Args:
      -c
      if [ ! -f /config/prometheus.yml ]; then
        touch /config/prometheus.yml
      fi
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 05 Feb 2024 18:59:26 +0100
      Finished:     Mon, 05 Feb 2024 18:59:26 +0100
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     500m
      memory:  256Mi
    Requests:
      cpu:     10m
      memory:  50Mi
    Environment:
      TZ:                      UTC
      UMASK:                   002
      UMASK_SET:               002
      NVIDIA_VISIBLE_DEVICES:  void
      S6_READ_ONLY_ROOT:       1
    Mounts:
      /config from config (rw)
Containers:
  prometheus:
    Container ID:  containerd://1b98c6fadd0d66c5d312640f0c7e8e85867e3b2249c12c5fae38916f2cb0b90e
    Image:         prom/prometheus:v2.49.1
    Image ID:      docker.io/prom/prometheus@sha256:beb5e30ffba08d9ae8a7961b9a2145fc8af6296ff2a4f463df7cd722fcbfc789
    Port:          30002/TCP
    Host Port:     0/TCP
    Args:
      --web.listen-address=0.0.0.0:30002
      --storage.tsdb.path=/data
      --config.file=/config/prometheus.yml
      --storage.tsdb.retention.time=15d
      --storage.tsdb.wal-compression
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 05 Feb 2024 18:58:45 +0100
      Finished:     Mon, 05 Feb 2024 18:58:45 +0100
    Ready:          False
    Restart Count:  235
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   http-get http://:30002/-/healthy delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  http-get http://:30002/-/ready delay=10s timeout=5s period=10s #success=2 #failure=5
    Startup:    http-get http://:30002/-/ready delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      TZ:                      UTC
      UMASK:                   002
      UMASK_SET:               002
      NVIDIA_VISIBLE_DEVICES:  void
      S6_READ_ONLY_ROOT:       1
    Mounts:
      /config from config (rw)
      /data from data (rw)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Apps/ix-applications/releases/prometheus/volumes/ix_volumes/config
    HostPathType:  
  data:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Apps/ix-applications/releases/prometheus/volumes/ix_volumes/data
    HostPathType:  
QoS Class:         Burstable
Node-Selectors:    <none>
Tolerations:       node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                   node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason          Age                     From     Message
  ----     ------          ----                    ----     -------
  Normal   AddedInterface  59m                     multus   Add eth0 [172.16.5.44/16] from ix-net
  Normal   AddedInterface  53m                     multus   Add eth0 [172.16.5.50/16] from ix-net
  Normal   AddedInterface  48m                     multus   Add eth0 [172.16.5.56/16] from ix-net
  Normal   AddedInterface  43m                     multus   Add eth0 [172.16.5.62/16] from ix-net
  Normal   Pulled          43m (x228 over 20h)     kubelet  Container image "prom/prometheus:v2.49.1" already present on machine
  Normal   AddedInterface  38m                     multus   Add eth0 [172.16.5.68/16] from ix-net
  Normal   AddedInterface  33m                     multus   Add eth0 [172.16.5.74/16] from ix-net
  Normal   Pulled          33m (x230 over 20h)     kubelet  Container image "bash:4.4.23" already present on machine
  Normal   Started         28m (x231 over 20h)     kubelet  Started container prometheus
  Normal   AddedInterface  27m                     multus   Add eth0 [172.16.5.79/16] from ix-net
  Normal   Pulled          23m (x239 over 20h)     kubelet  Container image "prom/prometheus:v2.49.1" already present on machine
  Normal   AddedInterface  22m                     multus   Add eth0 [172.16.5.84/16] from ix-net
  Normal   AddedInterface  17m                     multus   Add eth0 [172.16.5.90/16] from ix-net
  Normal   AddedInterface  12m                     multus   Add eth0 [172.16.5.96/16] from ix-net
  Normal   AddedInterface  7m33s                   multus   Add eth0 [172.16.5.102/16] from ix-net
  Warning  BackOff         3m11s (x3048 over 20h)  kubelet  Back-off restarting failed container prometheus in pod prometheus-645557855-89jrk_ix-prometheus(732278f4-3b22-4568-839f-00e4f5fc629f)
  Normal   AddedInterface  2m23s                   multus   Add eth0 [172.16.5.108/16] from ix-net


Name:                 openebs-zfs-controller-0
Namespace:            kube-system
Priority:             900000000
Priority Class Name:  openebs-zfs-csi-controller-critical
Service Account:      openebs-zfs-controller-sa
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               app=openebs-zfs-controller
                      controller-revision-hash=openebs-zfs-controller-77d96d5597
                      openebs.io/component-name=openebs-zfs-controller
                      openebs.io/version=2.3.0
                      role=openebs-zfs
                      statefulset.kubernetes.io/pod-name=openebs-zfs-controller-0
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.5.106"
                            ],
                            "mac": "02:10:6e:5e:76:4c",
                            "default": true,
                            "dns": {},
                            "gateway": [
                                "172.16.0.1"
                            ]
                        }]
Status:               Running
IP:                   172.16.5.106
IPs:
  IP:           172.16.5.106
Controlled By:  StatefulSet/openebs-zfs-controller
Containers:
  csi-resizer:
    Container ID:  containerd://376eaa224f3d3a9ab61dae7e1cc28c0a94c5063d99d767a63f0b18599d0e0a5f
    Image:         registry.k8s.io/sig-storage/csi-resizer:v1.8.0
    Image ID:      registry.k8s.io/sig-storage/csi-resizer@sha256:2e2b44393539d744a55b9370b346e8ebd95a77573064f3f9a8caf18c22f4d0d0
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:55:54 +0100
      Finished:     Mon, 05 Feb 2024 18:56:08 +0100
    Ready:          False
    Restart Count:  216
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snm6j (ro)
  csi-snapshotter:
    Container ID:  containerd://6a68d50fb44ddcbe874fc78f20766b8e4ddbcc534bcc208bb49e23ff6d4bbb78
    Image:         registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2
    Image ID:      registry.k8s.io/sig-storage/csi-snapshotter@sha256:becc53e25b96573f61f7469923a92fb3e9d3a3781732159954ce0d9da07233a2
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:55:57 +0100
      Finished:     Mon, 05 Feb 2024 18:56:08 +0100
    Ready:          False
    Restart Count:  216
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snm6j (ro)
  snapshot-controller:
    Container ID:  containerd://d9555cf46a21d54677221f33eb8cb82fea2f16a8225966ca8fd89d9f1c0bb392
    Image:         registry.k8s.io/sig-storage/snapshot-controller:v6.2.2
    Image ID:      registry.k8s.io/sig-storage/snapshot-controller@sha256:fb95b65bb88f319f0f7d5397c401a654164f11a191f466b4026fa36085c7141b
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --leader-election=true
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:00 +0100
      Finished:     Mon, 05 Feb 2024 18:56:08 +0100
    Ready:          False
    Restart Count:  216
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snm6j (ro)
  csi-provisioner:
    Container ID:  containerd://c97db55b98c4558dc288e2a01d183291c502bb5a58d19353722a2d1f4fdc03eb
    Image:         registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
    Image ID:      registry.k8s.io/sig-storage/csi-provisioner@sha256:d078dc174323407e8cc6f0f9abd4efaac5db27838f1564d0253d5e3233e3f17f
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 18:56:07 +0100
      Finished:     Mon, 05 Feb 2024 18:56:08 +0100
    Ready:          False
    Restart Count:  216
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:  kube-system (v1:metadata.namespace)
      POD_NAME:   openebs-zfs-controller-0 (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snm6j (ro)
  openebs-zfs-plugin:
    Container ID:  containerd://0f76d0d51500159dddec9fde4cf10788f5c2e45a33b470b61426a2541bb1b9d9
    Image:         openebs/zfs-driver:2.3.0
    Image ID:      docker.io/openebs/zfs-driver@sha256:42b8d31edc7cdf4116ecd6369f89a1d477bb9b64e695d074d290789af1a461dd
    Port:          <none>
    Host Port:     <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    State:          Running
      Started:      Mon, 05 Feb 2024 19:00:45 +0100
    Last State:     Terminated
      Reason:       Error
      Exit Code:    137
      Started:      Mon, 05 Feb 2024 18:54:54 +0100
      Finished:     Mon, 05 Feb 2024 18:55:24 +0100
    Ready:          True
    Restart Count:  216
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snm6j (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-snm6j:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason          Age                     From     Message
  ----     ------          ----                    ----     -------
  Normal   AddedInterface  56m                     multus   Add eth0 [172.16.5.48/16] from ix-net
  Normal   AddedInterface  56m                     multus   Add eth0 [172.16.5.49/16] from ix-net
  Normal   AddedInterface  51m                     multus   Add eth0 [172.16.5.54/16] from ix-net
  Normal   AddedInterface  50m                     multus   Add eth0 [172.16.5.55/16] from ix-net
  Normal   AddedInterface  45m                     multus   Add eth0 [172.16.5.60/16] from ix-net
  Normal   AddedInterface  44m                     multus   Add eth0 [172.16.5.61/16] from ix-net
  Normal   AddedInterface  39m                     multus   Add eth0 [172.16.5.66/16] from ix-net
  Normal   AddedInterface  39m                     multus   Add eth0 [172.16.5.67/16] from ix-net
  Normal   AddedInterface  34m                     multus   Add eth0 [172.16.5.72/16] from ix-net
  Normal   AddedInterface  33m                     multus   Add eth0 [172.16.5.73/16] from ix-net
  Normal   SandboxChanged  28m (x419 over 20h)     kubelet  Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface  28m                     multus   Add eth0 [172.16.5.78/16] from ix-net
  Normal   AddedInterface  27m                     multus   Add eth0 [172.16.5.80/16] from ix-net
  Normal   Pulled          22m (x404 over 20h)     kubelet  Container image "registry.k8s.io/sig-storage/csi-resizer:v1.8.0" already present on machine
  Normal   AddedInterface  22m                     multus   Add eth0 [172.16.5.85/16] from ix-net
  Normal   AddedInterface  21m                     multus   Add eth0 [172.16.5.86/16] from ix-net
  Normal   AddedInterface  16m                     multus   Add eth0 [172.16.5.91/16] from ix-net
  Normal   AddedInterface  16m                     multus   Add eth0 [172.16.5.93/16] from ix-net
  Normal   AddedInterface  11m                     multus   Add eth0 [172.16.5.97/16] from ix-net
  Normal   AddedInterface  10m                     multus   Add eth0 [172.16.5.99/16] from ix-net
  Normal   AddedInterface  5m35s                   multus   Add eth0 [172.16.5.104/16] from ix-net
  Normal   AddedInterface  4m26s                   multus   Add eth0 [172.16.5.106/16] from ix-net
  Warning  BackOff         2m57s (x3210 over 20h)  kubelet  Back-off restarting failed container csi-resizer in pod openebs-zfs-controller-0_kube-system(39953722-45d6-4f0c-b0bc-2bf8b3ec0cdd)


Name:                 snapshot-controller-546868dfb4-6jclc
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      snapshot-controller
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               app=snapshot-controller
                      pod-template-hash=546868dfb4
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.5.109"
                            ],
                            "mac": "02:1d:64:81:68:58",
                            "default": true,
                            "dns": {},
                            "gateway": [
                                "172.16.0.1"
                            ]
                        }]
Status:               Running
IP:                   172.16.5.109
IPs:
  IP:           172.16.5.109
Controlled By:  ReplicaSet/snapshot-controller-546868dfb4
Containers:
  snapshot-controller:
    Container ID:  containerd://e9515990dd1678be1cdd43d542c32461c396a13c674d1e421e9736aea5acff51
    Image:         registry.k8s.io/sig-storage/snapshot-controller:v6.2.2
    Image ID:      registry.k8s.io/sig-storage/snapshot-controller@sha256:fb95b65bb88f319f0f7d5397c401a654164f11a191f466b4026fa36085c7141b
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=2
      --leader-election=true
      --leader-election-namespace=kube-system
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 05 Feb 2024 19:00:29 +0100
      Finished:     Mon, 05 Feb 2024 19:00:30 +0100
    Ready:          False
    Restart Count:  239
    Limits:
      memory:  300Mi
    Requests:
      cpu:        10m
      memory:     20Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6tkxb (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  kube-api-access-6tkxb:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node-role.kubernetes.io/control-plane=true:NoSchedule
                             node-role.kubernetes.io/controlplane=true:NoSchedule
                             node-role.kubernetes.io/master=true:NoSchedule
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason          Age                    From     Message
  ----     ------          ----                   ----     -------
  Normal   AddedInterface  57m                    multus   Add eth0 [172.16.5.47/16] from ix-net
  Normal   AddedInterface  52m                    multus   Add eth0 [172.16.5.52/16] from ix-net
  Normal   AddedInterface  47m                    multus   Add eth0 [172.16.5.58/16] from ix-net
  Normal   AddedInterface  42m                    multus   Add eth0 [172.16.5.63/16] from ix-net
  Normal   AddedInterface  36m                    multus   Add eth0 [172.16.5.69/16] from ix-net
  Normal   AddedInterface  31m                    multus   Add eth0 [172.16.5.75/16] from ix-net
  Normal   AddedInterface  26m                    multus   Add eth0 [172.16.5.81/16] from ix-net
  Normal   AddedInterface  21m                    multus   Add eth0 [172.16.5.87/16] from ix-net
  Normal   AddedInterface  16m                    multus   Add eth0 [172.16.5.92/16] from ix-net
  Normal   SandboxChanged  10m (x238 over 20h)    kubelet  Pod sandbox changed, it will be killed and re-created.
  Normal   AddedInterface  10m                    multus   Add eth0 [172.16.5.98/16] from ix-net
  Normal   Pulled          6m12s (x238 over 20h)  kubelet  Container image "registry.k8s.io/sig-storage/snapshot-controller:v6.2.2" already present on machine
  Normal   AddedInterface  5m53s                  multus   Add eth0 [172.16.5.103/16] from ix-net
  Warning  BackOff         71s (x3874 over 20h)   kubelet  Back-off restarting failed container snapshot-controller in pod snapshot-controller-546868dfb4-6jclc_kube-system(5b7ea80f-63fa-4f9e-975f-c73af32fb501)
  Normal   AddedInterface  30s                    multus   Add eth0 [172.16.5.109/16] from ix-net


Name:                 coredns-59b4f5bbd5-7p628
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      coredns
Node:                 ix-truenas/192.168.178.45
Start Time:           Sun, 04 Feb 2024 22:22:40 +0100
Labels:               k8s-app=kube-dns
                      pod-template-hash=59b4f5bbd5
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.5.110"
                            ],
                            "mac": "c6:0f:6d:d0:dd:40",
                            "default": true,
                            "dns": {},
                            "gateway": [
                                "172.16.0.1"
                            ]
                        }]
Status:               Running
IP:                   172.16.5.110
IPs:
  IP:           172.16.5.110
Controlled By:  ReplicaSet/coredns-59b4f5bbd5
Containers:
  coredns:
    Container ID:  containerd://eb87c59bc99123d96d716c72f5ed00bc73e6c700c35a11b315531879a0d13cac
    Image:         rancher/mirrored-coredns-coredns:1.10.1
    Image ID:      docker.io/rancher/mirrored-coredns-coredns@sha256:a11fafae1f8037cbbd66c5afa40ba2423936b72b4fd50a7034a7e8b955163594
    Ports:         53/UDP, 53/TCP, 9153/TCP
    Host Ports:    0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 05 Feb 2024 19:00:50 +0100
      Finished:     Mon, 05 Feb 2024 19:00:51 +0100
    Ready:          False
    Restart Count:  240
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-kn64t (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
  custom-config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns-custom
    Optional:  true
  kube-api-access-kn64t:
    Type:                     Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:   3607
    ConfigMapName:            kube-root-ca.crt
    ConfigMapOptional:        <nil>
    DownwardAPI:              true
QoS Class:                    Burstable
Node-Selectors:               kubernetes.io/os=linux
Tolerations:                  CriticalAddonsOnly op=Exists
                              node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                              node-role.kubernetes.io/master:NoSchedule op=Exists
                              node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                              node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Topology Spread Constraints:  kubernetes.io/hostname:DoNotSchedule when max skew 1 is exceeded for selector k8s-app=kube-dns
Events:
  Type     Reason          Age                     From     Message
  ----     ------          ----                    ----     -------
  Warning  Unhealthy       58m (x3 over 58m)       kubelet  Readiness probe failed: Get "http://172.16.5.39:8181/ready": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
  Normal   AddedInterface  58m                     multus   Add eth0 [172.16.5.45/16] from ix-net
  Warning  Unhealthy       53m (x4 over 53m)       kubelet  Readiness probe failed: Get "http://172.16.5.45:8181/ready": dial tcp 172.16.5.45:8181: connect: connection refused
  Normal   AddedInterface  52m                     multus   Add eth0 [172.16.5.51/16] from ix-net
  Normal   Pulled          48m (x231 over 20h)     kubelet  Container image "rancher/mirrored-coredns-coredns:1.10.1" already present on machine
  Normal   AddedInterface  47m                     multus   Add eth0 [172.16.5.57/16] from ix-net
  Normal   AddedInterface  42m                     multus   Add eth0 [172.16.5.64/16] from ix-net
  Normal   AddedInterface  36m                     multus   Add eth0 [172.16.5.70/16] from ix-net
  Normal   AddedInterface  31m                     multus   Add eth0 [172.16.5.76/16] from ix-net
  Normal   AddedInterface  25m                     multus   Add eth0 [172.16.5.82/16] from ix-net
  Normal   AddedInterface  20m                     multus   Add eth0 [172.16.5.88/16] from ix-net
  Normal   AddedInterface  15m                     multus   Add eth0 [172.16.5.94/16] from ix-net
  Normal   AddedInterface  10m                     multus   Add eth0 [172.16.5.100/16] from ix-net
  Normal   AddedInterface  5m19s                   multus   Add eth0 [172.16.5.105/16] from ix-net
  Warning  BackOff         3m21s (x3898 over 20h)  kubelet  Back-off restarting failed container coredns in pod coredns-59b4f5bbd5-7p628_kube-system(586a341c-2cb0-4eb5-9f5d-690b594986b5)
  Normal   AddedInterface  10s                     multus   Add eth0 [172.16.5.110/16] from ix-net
