+--------------------------------------------------------------------------------+
+                      Kubernetes Configuration @1647217990                      +
+--------------------------------------------------------------------------------+
{
  "id": 1,
  "pool": "pool 0",
  "cluster_cidr": "172.16.0.0/16",
  "service_cidr": "172.17.0.0/16",
  "cluster_dns_ip": "172.17.0.10",
  "route_v4_interface": "eno1",
  "route_v4_gateway": "192.168.1.1",
  "route_v6_interface": null,
  "route_v6_gateway": null,
  "node_ip": "0.0.0.0",
  "configure_gpus": true,
  "servicelb": true,
  "dataset": "pool 0/ix-applications"
}
debug finished in 0 seconds for Kubernetes Configuration


+--------------------------------------------------------------------------------+
+                     k3s kubectl describe nodes @1647217990                     +
+--------------------------------------------------------------------------------+
Name:               ix-truenas
Roles:              control-plane,master
Labels:             beta.kubernetes.io/arch=amd64
                    beta.kubernetes.io/os=linux
                    kubernetes.io/arch=amd64
                    kubernetes.io/hostname=ix-truenas
                    kubernetes.io/os=linux
                    node-role.kubernetes.io/control-plane=true
                    node-role.kubernetes.io/master=true
                    openebs.io/nodeid=ix-truenas
                    openebs.io/nodename=ix-truenas
Annotations:        csi.volume.kubernetes.io/nodeid: {"zfs.csi.openebs.io":"ix-truenas"}
                    k3s.io/node-args:
                      ["server","--flannel-backend","none","--disable","traefik,metrics-server,local-storage","--disable-kube-proxy","--disable-network-policy",...
                    k3s.io/node-config-hash: 2H7UIYDJVBJERVFTED67XBR2DDTQXJJHAVCJI5PD432G2INS6S3A====
                    k3s.io/node-env: {"K3S_DATA_DIR":"/mnt/pool 0/ix-applications/k3s/data/94499178817b4c5ce4973492f034bfaed731d47d3de22485bfc180619d2a1cc7"}
                    node.alpha.kubernetes.io/ttl: 0
                    volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp:  Sun, 13 Mar 2022 19:30:03 -0400
Taints:             <none>
Unschedulable:      false
Lease:
  HolderIdentity:  ix-truenas
  AcquireTime:     <unset>
  RenewTime:       Sun, 13 Mar 2022 20:33:05 -0400
Conditions:
  Type             Status  LastHeartbeatTime                 LastTransitionTime                Reason                       Message
  ----             ------  -----------------                 ------------------                ------                       -------
  MemoryPressure   False   Sun, 13 Mar 2022 20:31:01 -0400   Sun, 13 Mar 2022 19:30:00 -0400   KubeletHasSufficientMemory   kubelet has sufficient memory available
  DiskPressure     False   Sun, 13 Mar 2022 20:31:01 -0400   Sun, 13 Mar 2022 19:30:00 -0400   KubeletHasNoDiskPressure     kubelet has no disk pressure
  PIDPressure      False   Sun, 13 Mar 2022 20:31:01 -0400   Sun, 13 Mar 2022 19:30:00 -0400   KubeletHasSufficientPID      kubelet has sufficient PID available
  Ready            True    Sun, 13 Mar 2022 20:31:01 -0400   Sun, 13 Mar 2022 19:30:21 -0400   KubeletReady                 kubelet is posting ready status. AppArmor enabled
Addresses:
  InternalIP:  192.168.1.147
  Hostname:    ix-truenas
Capacity:
  cpu:                16
  ephemeral-storage:  3951097600Ki
  hugepages-2Mi:      0
  memory:             74197432Ki
  pods:               110
Allocatable:
  cpu:                16
  ephemeral-storage:  3843627742266
  hugepages-2Mi:      0
  memory:             74197432Ki
  pods:               110
System Info:
  Machine ID:                 85282b3449064b30a8b96ce3a3a6ff7a
  System UUID:                4c4c4544-0039-3610-8042-b4c04f544b31
  Boot ID:                    326ef6c7-65c8-4813-bc2d-9ea366410bd0
  Kernel Version:             5.10.93+truenas
  OS Image:                   Debian GNU/Linux 11 (bullseye)
  Operating System:           linux
  Architecture:               amd64
  Container Runtime Version:  docker://20.10.12
  Kubelet Version:            v1.21.0-k3s1
  Kube-Proxy Version:         v1.21.0-k3s1
PodCIDR:                      172.16.0.0/16
PodCIDRs:                     172.16.0.0/16
Non-terminated Pods:          (3 in total)
  Namespace                   Name                        CPU Requests  CPU Limits  Memory Requests  Memory Limits  Age
  ---------                   ----                        ------------  ----------  ---------------  -------------  ---
  kube-system                 coredns-7448499f4d-qzbr4    100m (0%)     0 (0%)      70Mi (0%)        170Mi (0%)     62m
  kube-system                 openebs-zfs-node-l5t7c      0 (0%)        0 (0%)      0 (0%)           0 (0%)         62m
  kube-system                 openebs-zfs-controller-0    0 (0%)        0 (0%)      0 (0%)           0 (0%)         62m
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  Resource           Requests   Limits
  --------           --------   ------
  cpu                100m (0%)  0 (0%)
  memory             70Mi (0%)  170Mi (0%)
  ephemeral-storage  0 (0%)     0 (0%)
  hugepages-2Mi      0 (0%)     0 (0%)
Events:              <none>
debug finished in 1 seconds for k3s kubectl describe nodes


+--------------------------------------------------------------------------------+
+k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide @1647217991+
+--------------------------------------------------------------------------------+
NAMESPACE     NAME                           READY   STATUS    RESTARTS   AGE   IP              NODE         NOMINATED NODE   READINESS GATES
kube-system   pod/coredns-7448499f4d-qzbr4   1/1     Running   0          62m   172.16.0.20     ix-truenas   <none>           <none>
kube-system   pod/openebs-zfs-node-l5t7c     2/2     Running   0          62m   192.168.1.147   ix-truenas   <none>           <none>
kube-system   pod/openebs-zfs-controller-0   5/5     Running   0          62m   172.16.0.21     ix-truenas   <none>           <none>

NAMESPACE      NAME                    TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                                     AGE   SELECTOR
default        service/kubernetes      ClusterIP   172.17.0.1       <none>        443/TCP                                                     63m   <none>
kube-system    service/kube-dns        ClusterIP   172.17.0.10      <none>        53/UDP,53/TCP,9153/TCP                                      63m   k8s-app=kube-dns
ix-plex-test   service/plex-test-udp   ClusterIP   172.17.95.145    <none>        1900/UDP,32410/UDP,32412/UDP,32413/UDP,32414/UDP            40m   app.kubernetes.io/instance=plex-test,app.kubernetes.io/name=plex
ix-plex-test   service/plex-test-tcp   NodePort    172.17.218.255   <none>        32400:32400/TCP,80:60360/TCP,443:63168/TCP,1900:40763/TCP   40m   app.kubernetes.io/instance=plex-test,app.kubernetes.io/name=plex

NAMESPACE     NAME                              DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE   CONTAINERS                                     IMAGES                                                                             SELECTOR
kube-system   daemonset.apps/openebs-zfs-node   1         1         1       1            1           <none>          63m   csi-node-driver-registrar,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0,openebs/zfs-driver:1.9.3   app=openebs-zfs-node

NAMESPACE      NAME                        READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS   IMAGES                                     SELECTOR
kube-system    deployment.apps/coredns     1/1     1            1           63m   coredns      rancher/coredns-coredns:1.8.3              k8s-app=kube-dns
ix-plex-test   deployment.apps/plex-test   0/0     0            0           40m   plex         plexinc/pms-docker:1.25.6.5577-c8bd13540   app.kubernetes.io/instance=plex-test,app.kubernetes.io/name=plex

NAMESPACE     NAME                                      READY   AGE   CONTAINERS                                                                           IMAGES
kube-system   statefulset.apps/openebs-zfs-controller   1/1     63m   csi-resizer,csi-snapshotter,snapshot-controller,csi-provisioner,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-resizer:v1.1.0,k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0,k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0,k8s.gcr.io/sig-storage/csi-provisioner:v2.1.0,openebs/zfs-driver:1.9.3

NAMESPACE   NAME                                                          PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
            storageclass.storage.k8s.io/openebs-zfspv-default (default)   zfs.csi.openebs.io   Delete          Immediate           true                   63m
            storageclass.storage.k8s.io/ix-storage-class-plex-test        zfs.csi.openebs.io   Retain          Immediate           true                   40m

NAMESPACE   NAME                        STATUS   AGE
            namespace/default           Active   63m
            namespace/kube-system       Active   63m
            namespace/kube-public       Active   63m
            namespace/kube-node-lease   Active   63m
            namespace/openebs           Active   63m
            namespace/ix-plex-test      Active   40m
debug finished in 0 seconds for k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide


+--------------------------------------------------------------------------------+
+         k3s kubectl describe deployments --all-namespaces @1647217991          +
+--------------------------------------------------------------------------------+
Name:                   coredns
Namespace:              kube-system
CreationTimestamp:      Sun, 13 Mar 2022 19:30:07 -0400
Labels:                 k8s-app=kube-dns
                        kubernetes.io/name=CoreDNS
                        objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:            deployment.kubernetes.io/revision: 1
                        objectset.rio.cattle.io/applied:
                          H4sIAAAAAAAA/6xV32/bNhD+V4Z7ln8obhNXwB66OFuDLZ6RH30pjOJMnSzOFI8jKTdGoP99OEl2nDRNu2FPpsnvjt/38e70AOj0R/JBs4UM0Lkw2qaQwEbbHDKYkTO8q8hGSKCiiD...
                        objectset.rio.cattle.io/id: 
                        objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                        objectset.rio.cattle.io/owner-name: coredns
                        objectset.rio.cattle.io/owner-namespace: kube-system
Selector:               k8s-app=kube-dns
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  1 max unavailable, 25% max surge
Pod Template:
  Labels:           k8s-app=kube-dns
  Service Account:  coredns
  Containers:
   coredns:
    Image:       rancher/coredns-coredns:1.8.3
    Ports:       53/UDP, 53/TCP, 9153/TCP
    Host Ports:  0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
  Volumes:
   config-volume:
    Type:               ConfigMap (a volume populated by a ConfigMap)
    Name:               coredns
    Optional:           false
  Priority Class Name:  system-cluster-critical
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   coredns-7448499f4d (1/1 replicas created)
Events:          <none>


Name:               plex-test
Namespace:          ix-plex-test
CreationTimestamp:  Sun, 13 Mar 2022 19:52:25 -0400
Labels:             app.kubernetes.io/instance=plex-test
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=plex
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: plex-test
                    meta.helm.sh/release-namespace: ix-plex-test
Selector:           app.kubernetes.io/instance=plex-test,app.kubernetes.io/name=plex
Replicas:           0 desired | 0 updated | 0 total | 0 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:       app.kubernetes.io/instance=plex-test
                app.kubernetes.io/name=plex
  Annotations:  rollme: tVd7Y
  Containers:
   plex:
    Image:       plexinc/pms-docker:1.25.6.5577-c8bd13540
    Ports:       32400/TCP, 32469/TCP, 1900/UDP, 32410/UDP, 32412/UDP, 32413/UDP, 32414/UDP
    Host Ports:  0/TCP, 0/TCP, 0/UDP, 0/UDP, 0/UDP, 0/UDP, 0/UDP
    Liveness:    exec [/bin/plex_probe_check.sh] delay=0s timeout=1s period=15s #success=1 #failure=5
    Readiness:   exec [/bin/plex_probe_check.sh] delay=0s timeout=1s period=15s #success=1 #failure=5
    Startup:     exec [/bin/plex_probe_check.sh] delay=5s timeout=1s period=15s #success=1 #failure=40
    Environment:
      KUBE_NAMESPACE:         (v1:metadata.namespace)
      POD_IP:                 (v1:status.podIP)
      TZ:                    America/New_York
      PLEX_CLAIM:            
      PMS_INTERNAL_ADDRESS:  http://plex-test:32400
      PMS_IMAGE:             plexinc/pms-docker:1.25.6.5577-c8bd13540
    Mounts:
      /bin/plex_probe_check.sh from plex-probe-check (ro,path="entrypoint.sh")
      /config from config (rw)
      /config/Library/Application Support/Plex Media Server/Logs from shared-logs (rw)
      /data from data (rw)
      /shared from shared (rw)
      /transcode from transcode (rw)
  Volumes:
   config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/pool 0/ix-applications/releases/plex-test/volumes/ix_volumes/ix-plex_config
    HostPathType:  
   data:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/pool 0/ix-applications/releases/plex-test/volumes/ix_volumes/ix-plex_data
    HostPathType:  
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shared-logs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   transcode:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/pool 0/ix-applications/releases/plex-test/volumes/ix_volumes/ix-plex_transcode
    HostPathType:  
   plex-probe-check:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      plex-probe-check
    Optional:  false
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   plex-test-8574fd5c48 (0/0 replicas created)
Events:
  Type    Reason             Age   From                   Message
  ----    ------             ----  ----                   -------
  Normal  ScalingReplicaSet  40m   deployment-controller  Scaled up replica set plex-test-8574fd5c48 to 1
  Normal  ScalingReplicaSet  16s   deployment-controller  Scaled down replica set plex-test-8574fd5c48 to 0
debug finished in 0 seconds for k3s kubectl describe deployments --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe pods --all-namespaces @1647217991             +
+--------------------------------------------------------------------------------+
Name:                 coredns-7448499f4d-qzbr4
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Node:                 ix-truenas/192.168.1.147
Start Time:           Sun, 13 Mar 2022 19:30:21 -0400
Labels:               k8s-app=kube-dns
                      pod-template-hash=7448499f4d
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.20"
                            ],
                            "mac": "42:55:91:8c:ca:7a",
                            "default": true,
                            "dns": {}
                        }]
                      k8s.v1.cni.cncf.io/networks-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.20"
                            ],
                            "mac": "42:55:91:8c:ca:7a",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.0.20
IPs:
  IP:           172.16.0.20
Controlled By:  ReplicaSet/coredns-7448499f4d
Containers:
  coredns:
    Container ID:  docker://cceb279ff6c8faf57edfdeb245085aa9ea1461b3a703a54b967da1589ae3a100
    Image:         rancher/coredns-coredns:1.8.3
    Image ID:      docker://sha256:3885a5b7f138c3e8779a18f672fd55c3c7cf0166b67cf5143ab0cd137eb38418
    Ports:         53/UDP, 53/TCP, 9153/TCP
    Host Ports:    0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    State:          Running
      Started:      Sun, 13 Mar 2022 19:31:12 -0400
    Ready:          True
    Restart Count:  0
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-9qzsv (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
  kube-api-access-9qzsv:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              beta.kubernetes.io/os=linux
Tolerations:                 CriticalAddonsOnly op=Exists
                             node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                             node-role.kubernetes.io/master:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:                 openebs-zfs-node-l5t7c
Namespace:            kube-system
Priority:             2000001000
Priority Class Name:  system-node-critical
Node:                 ix-truenas/192.168.1.147
Start Time:           Sun, 13 Mar 2022 19:30:21 -0400
Labels:               app=openebs-zfs-node
                      controller-revision-hash=6b75b8b65d
                      openebs.io/component-name=openebs-zfs-node
                      openebs.io/version=ci
                      pod-template-generation=1
                      role=openebs-zfs
Annotations:          <none>
Status:               Running
IP:                   192.168.1.147
IPs:
  IP:           192.168.1.147
Controlled By:  DaemonSet/openebs-zfs-node
Containers:
  csi-node-driver-registrar:
    Container ID:  docker://559f44e3d962eeeb348c5d8f1a8cb92e1a40ca5a88a6b4fd250a1f50d8746cd7
    Image:         k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    State:          Running
      Started:      Sun, 13 Mar 2022 19:31:12 -0400
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snzf2 (ro)
  openebs-zfs-plugin:
    Container ID:  docker://53f501e667a4612be3ca6c7ce2ea42fd04d5d1782361f9153a54569c70917743
    Image:         openebs/zfs-driver:1.9.3
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:1f6f18059a646ca1ce8fc8f5a305b0f902a13723b8e162206ce23f8359bd1d0b
    Port:          <none>
    Host Port:     <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    State:          Running
      Started:      Sun, 13 Mar 2022 19:32:00 -0400
    Ready:          True
    Restart Count:  0
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-snzf2 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
  encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
  chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
  host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
  plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
  pods-mount-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/
    HostPathType:  Directory
  kube-api-access-snzf2:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:                      <none>


Name:                 openebs-zfs-controller-0
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Node:                 ix-truenas/192.168.1.147
Start Time:           Sun, 13 Mar 2022 19:30:22 -0400
Labels:               app=openebs-zfs-controller
                      controller-revision-hash=openebs-zfs-controller-bffcd445b
                      openebs.io/component-name=openebs-zfs-controller
                      openebs.io/version=ci
                      role=openebs-zfs
                      statefulset.kubernetes.io/pod-name=openebs-zfs-controller-0
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.21"
                            ],
                            "mac": "06:94:a8:58:ed:e0",
                            "default": true,
                            "dns": {}
                        }]
                      k8s.v1.cni.cncf.io/networks-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.21"
                            ],
                            "mac": "06:94:a8:58:ed:e0",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.0.21
IPs:
  IP:           172.16.0.21
Controlled By:  StatefulSet/openebs-zfs-controller
Containers:
  csi-resizer:
    Container ID:  docker://37f085bbc2a71b46fb3bcf501889c5f53702b3eded449249698cb44fbf3908f1
    Image:         k8s.gcr.io/sig-storage/csi-resizer:v1.1.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Sun, 13 Mar 2022 19:31:13 -0400
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mc6r8 (ro)
  csi-snapshotter:
    Container ID:  docker://3b13637b34206f65ab2ac394dab19dcafca6bfcc7699131db3ac7834eb0412d7
    Image:         k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-snapshotter@sha256:51f2dfde5bccac7854b3704689506aeecfb793328427b91115ba253a93e60782
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Sun, 13 Mar 2022 19:32:02 -0400
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mc6r8 (ro)
  snapshot-controller:
    Container ID:  docker://2396cd1c27bf502b8515b3b695808587b2b6836065e8acb2070f7e241c81e61b
    Image:         k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/snapshot-controller@sha256:00fcc441ea9f72899c25eed61d602272a2a58c5f0014332bdcb5ac24acef08e4
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --leader-election=true
    State:          Running
      Started:      Sun, 13 Mar 2022 19:32:07 -0400
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mc6r8 (ro)
  csi-provisioner:
    Container ID:  docker://45e54ca4ff9ac9bc23b06ed238f05e0ca425ff65aa49ce57a334d73d8a4c43cb
    Image:         k8s.gcr.io/sig-storage/csi-provisioner:v2.1.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-provisioner@sha256:20c828075d1e36f679d6a91e905b0927141eef5e15be0c9a1ca4a6a0ed9313d2
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --default-fstype=ext4
    State:          Running
      Started:      Sun, 13 Mar 2022 19:32:25 -0400
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mc6r8 (ro)
  openebs-zfs-plugin:
    Container ID:  docker://0d87f3a31d7de7588349944061394a46d8a70d9d7bedf73873bd404abc8a18d4
    Image:         openebs/zfs-driver:1.9.3
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:1f6f18059a646ca1ce8fc8f5a305b0f902a13723b8e162206ce23f8359bd1d0b
    Port:          <none>
    Host Port:     <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    State:          Running
      Started:      Sun, 13 Mar 2022 19:32:36 -0400
    Ready:          True
    Restart Count:  0
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mc6r8 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-mc6r8:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type    Reason   Age   From     Message
  ----    ------   ----  ----     -------
  Normal  Created  60m   kubelet  Created container csi-provisioner
  Normal  Started  60m   kubelet  Started container csi-provisioner
  Normal  Pulled   60m   kubelet  Container image "openebs/zfs-driver:1.9.3" already present on machine
  Normal  Created  60m   kubelet  Created container openebs-zfs-plugin
  Normal  Started  60m   kubelet  Started container openebs-zfs-plugin
debug finished in 1 seconds for k3s kubectl describe pods --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe services --all-namespaces @1647217992           +
+--------------------------------------------------------------------------------+
Name:              kubernetes
Namespace:         default
Labels:            component=apiserver
                   provider=kubernetes
Annotations:       <none>
Selector:          <none>
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.1
IPs:               172.17.0.1
Port:              https  443/TCP
TargetPort:        6443/TCP
Endpoints:         192.168.1.147:6443
Session Affinity:  None
Events:            <none>


Name:              kube-dns
Namespace:         kube-system
Labels:            k8s-app=kube-dns
                   kubernetes.io/cluster-service=true
                   kubernetes.io/name=CoreDNS
                   objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:       objectset.rio.cattle.io/applied:
                     {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"objectset.rio.cattle.io/id":"","objectset.rio.cattle.io/owner-gvk":"k3s.ca...
                   objectset.rio.cattle.io/id: 
                   objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                   objectset.rio.cattle.io/owner-name: coredns
                   objectset.rio.cattle.io/owner-namespace: kube-system
                   prometheus.io/port: 9153
                   prometheus.io/scrape: true
Selector:          k8s-app=kube-dns
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.10
IPs:               172.17.0.10
Port:              dns  53/UDP
TargetPort:        53/UDP
Endpoints:         172.16.0.20:53
Port:              dns-tcp  53/TCP
TargetPort:        53/TCP
Endpoints:         172.16.0.20:53
Port:              metrics  9153/TCP
TargetPort:        9153/TCP
Endpoints:         172.16.0.20:9153
Session Affinity:  None
Events:            <none>


Name:              plex-test-udp
Namespace:         ix-plex-test
Labels:            app.kubernetes.io/instance=plex-test
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=plex
                   app.kubernetes.io/version=1.25.6.5577
                   helm.sh/chart=plex-1.6.24
Annotations:       meta.helm.sh/release-name: plex-test
                   meta.helm.sh/release-namespace: ix-plex-test
Selector:          app.kubernetes.io/instance=plex-test,app.kubernetes.io/name=plex
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.95.145
IPs:               172.17.95.145
Port:              plex-dlna-udp  1900/UDP
TargetPort:        plex-dlna-udp/UDP
Endpoints:         <none>
Port:              plex-gdm1  32410/UDP
TargetPort:        plex-gdm1/UDP
Endpoints:         <none>
Port:              plex-gdm2  32412/UDP
TargetPort:        plex-gdm2/UDP
Endpoints:         <none>
Port:              plex-gdm3  32413/UDP
TargetPort:        plex-gdm3/UDP
Endpoints:         <none>
Port:              plex-gdm4  32414/UDP
TargetPort:        plex-gdm4/UDP
Endpoints:         <none>
Session Affinity:  None
Events:            <none>


Name:                     plex-test-tcp
Namespace:                ix-plex-test
Labels:                   app.kubernetes.io/instance=plex-test
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=plex
                          app.kubernetes.io/version=1.25.6.5577
                          helm.sh/chart=plex-1.6.24
Annotations:              meta.helm.sh/release-name: plex-test
                          meta.helm.sh/release-namespace: ix-plex-test
Selector:                 app.kubernetes.io/instance=plex-test,app.kubernetes.io/name=plex
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.218.255
IPs:                      172.17.218.255
Port:                     pms  32400/TCP
TargetPort:               pms/TCP
NodePort:                 pms  32400/TCP
Endpoints:                <none>
Port:                     http  80/TCP
TargetPort:               pms/TCP
NodePort:                 http  60360/TCP
Endpoints:                <none>
Port:                     https  443/TCP
TargetPort:               pms/TCP
NodePort:                 https  63168/TCP
Endpoints:                <none>
Port:                     plex-dlna  1900/TCP
TargetPort:               plex-dlna/TCP
NodePort:                 plex-dlna  40763/TCP
Endpoints:                <none>
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>
debug finished in 0 seconds for k3s kubectl describe services --all-namespaces


+--------------------------------------------------------------------------------+
+         k3s kubectl describe statefulset --all-namespaces @1647217992          +
+--------------------------------------------------------------------------------+
Name:               openebs-zfs-controller
Namespace:          kube-system
CreationTimestamp:  Sun, 13 Mar 2022 19:30:07 -0400
Selector:           app=openebs-zfs-controller,role=openebs-zfs
Labels:             objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
Annotations:        objectset.rio.cattle.io/applied:
                      H4sIAAAAAAAA/8xWXW8aORT9K6urPrTSmI8mu2lH4mEa5gE1JSigSlWFIuO5M3gxtte+MwuN+O8rzwCdpqFJVrvdfQPb99xzjs8F3wG38iM6L42GGLi1vlv1IYKV1BnEMCVOmJdqig...
                    objectset.rio.cattle.io/id: 
                    objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                    objectset.rio.cattle.io/owner-name: zfs-operator
                    objectset.rio.cattle.io/owner-namespace: kube-system
Replicas:           1 desired | 1 total
Update Strategy:    RollingUpdate
  Partition:        0
Pods Status:        1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-controller
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-controller-sa
  Containers:
   csi-resizer:
    Image:      k8s.gcr.io/sig-storage/csi-resizer:v1.1.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   csi-snapshotter:
    Image:      k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   snapshot-controller:
    Image:      k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --leader-election=true
    Environment:  <none>
    Mounts:       <none>
   csi-provisioner:
    Image:      k8s.gcr.io/sig-storage/csi-provisioner:v2.1.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --default-fstype=ext4
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:1.9.3
    Port:       <none>
    Host Port:  <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
  Volumes:
   socket-dir:
    Type:               EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:             
    SizeLimit:          <unset>
  Priority Class Name:  system-cluster-critical
Volume Claims:          <none>
Events:                 <none>
debug finished in 0 seconds for k3s kubectl describe statefulset --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe job --all-namespaces @1647217992              +
+--------------------------------------------------------------------------------+
No resources found
debug finished in 1 seconds for k3s kubectl describe job --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe cronjob --all-namespaces @1647217993            +
+--------------------------------------------------------------------------------+
No resources found
debug finished in 0 seconds for k3s kubectl describe cronjob --all-namespaces


+--------------------------------------------------------------------------------+
+          k3s kubectl describe daemonsets --all-namespaces @1647217993          +
+--------------------------------------------------------------------------------+
Name:           openebs-zfs-node
Selector:       app=openebs-zfs-node
Node-Selector:  <none>
Labels:         objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                openebs.io/component-name=openebs-zfs-node
                openebs.io/version=ci
Annotations:    deprecated.daemonset.template.generation: 1
                objectset.rio.cattle.io/applied:
                  H4sIAAAAAAAA/6xWUY/aOBD+K5XVk1opIVDabhtpHyjkrqveAoK2L9UKOfYk+HDsyHayy63y3092AgQI2z1p3xJ75puZbz6P/Yhwzn6C0kwKFCKc5zooB8hDGyYoCtEEQybFEgzyUA...
                objectset.rio.cattle.io/id: 
                objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                objectset.rio.cattle.io/owner-name: zfs-operator
                objectset.rio.cattle.io/owner-namespace: kube-system
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-node
                    openebs.io/component-name=openebs-zfs-node
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-node-sa
  Containers:
   csi-node-driver-registrar:
    Image:      k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:1.9.3
    Port:       <none>
    Host Port:  <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
  Volumes:
   device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
   encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
   chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
   host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
   registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
   plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
   pods-mount-dir:
    Type:               HostPath (bare host directory volume)
    Path:               /var/lib/kubelet/
    HostPathType:       Directory
  Priority Class Name:  system-node-critical
Events:                 <none>
debug finished in 0 seconds for k3s kubectl describe daemonsets --all-namespaces


+--------------------------------------------------------------------------------+
+                            docker ps -a @1647217993                            +
+--------------------------------------------------------------------------------+
CONTAINER ID   IMAGE               COMMAND                  CREATED             STATUS             PORTS     NAMES
0d87f3a31d7d   10dbacde5cc2        "/usr/local/bin/zfs-…"   About an hour ago   Up About an hour             k8s_openebs-zfs-plugin_openebs-zfs-controller-0_kube-system_4e5509b0-fcae-4486-b3ae-0278682ecd79_0
45e54ca4ff9a   e0d187f105d6        "/csi-provisioner --…"   About an hour ago   Up About an hour             k8s_csi-provisioner_openebs-zfs-controller-0_kube-system_4e5509b0-fcae-4486-b3ae-0278682ecd79_0
2396cd1c27bf   f1d8a00ae690        "/snapshot-controlle…"   About an hour ago   Up About an hour             k8s_snapshot-controller_openebs-zfs-controller-0_kube-system_4e5509b0-fcae-4486-b3ae-0278682ecd79_0
3b13637b3420   da32a49a903a        "/csi-snapshotter --…"   About an hour ago   Up About an hour             k8s_csi-snapshotter_openebs-zfs-controller-0_kube-system_4e5509b0-fcae-4486-b3ae-0278682ecd79_0
53f501e667a4   10dbacde5cc2        "/usr/local/bin/zfs-…"   About an hour ago   Up About an hour             k8s_openebs-zfs-plugin_openebs-zfs-node-l5t7c_kube-system_f5fad81a-b088-466e-9c3d-4e116edeb641_0
37f085bbc2a7   a8fe79377034        "/csi-resizer --v=5 …"   About an hour ago   Up About an hour             k8s_csi-resizer_openebs-zfs-controller-0_kube-system_4e5509b0-fcae-4486-b3ae-0278682ecd79_0
cceb279ff6c8   3885a5b7f138        "/coredns -conf /etc…"   About an hour ago   Up About an hour             k8s_coredns_coredns-7448499f4d-qzbr4_kube-system_e150fb2e-ded4-43fa-a37f-75b61c350a30_0
559f44e3d962   ef2b13b2a066        "/csi-node-driver-re…"   About an hour ago   Up About an hour             k8s_csi-node-driver-registrar_openebs-zfs-node-l5t7c_kube-system_f5fad81a-b088-466e-9c3d-4e116edeb641_0
2c6a0b546948   rancher/pause:3.1   "/pause"                 About an hour ago   Up About an hour             k8s_POD_openebs-zfs-controller-0_kube-system_4e5509b0-fcae-4486-b3ae-0278682ecd79_0
3aa01d013da8   rancher/pause:3.1   "/pause"                 About an hour ago   Up About an hour             k8s_POD_coredns-7448499f4d-qzbr4_kube-system_e150fb2e-ded4-43fa-a37f-75b61c350a30_0
93dcfe8865d1   rancher/pause:3.1   "/pause"                 About an hour ago   Up About an hour             k8s_POD_openebs-zfs-node-l5t7c_kube-system_f5fad81a-b088-466e-9c3d-4e116edeb641_0
debug finished in 0 seconds for docker ps -a


+--------------------------------------------------------------------------------+
+                          docker images -a @1647217993                          +
+--------------------------------------------------------------------------------+
REPOSITORY                                         TAG       IMAGE ID       CREATED         SIZE
romses/mineos                                      1.18      502637d8e9f3   3 months ago    825MB
ghcr.io/truecharts/alpine                          <none>    f937e4e4c343   3 months ago    23.1MB
openebs/zfs-driver                                 1.9.3     10dbacde5cc2   5 months ago    175MB
rancher/coredns-coredns                            1.8.3     3885a5b7f138   12 months ago   43.5MB
k8s.gcr.io/sig-storage/csi-snapshotter             v4.0.0    da32a49a903a   15 months ago   49.5MB
k8s.gcr.io/sig-storage/snapshot-controller         v4.0.0    f1d8a00ae690   15 months ago   46.6MB
k8s.gcr.io/sig-storage/csi-resizer                 v1.1.0    a8fe79377034   15 months ago   49.2MB
k8s.gcr.io/sig-storage/csi-provisioner             v2.1.0    e0d187f105d6   15 months ago   51.6MB
k8s.gcr.io/sig-storage/csi-node-driver-registrar   v2.1.0    ef2b13b2a066   15 months ago   19.6MB
rancher/klipper-lb                                 v0.1.2    897ce3c5fc8f   2 years ago     6.1MB
rancher/pause                                      3.1       da86e6ba6ca1   4 years ago     742kB
debug finished in 1 seconds for docker images -a


+--------------------------------------------------------------------------------+
+                     K3s Logs (last 1000 lines) @1647217994                     +
+--------------------------------------------------------------------------------+
Mar 13 19:26:27 newnas.local k3s[85484]: E0313 19:26:27.831097   85484 status.go:71] apiserver received an error that is not an metav1.Status: &status.statusError{Code:2, Message:"unable to open database file: no such file or directory", Details:[]*anypb.Any(nil), XXX_NoUnkeyedLiteral:struct {}{}, XXX_unrecognized:[]uint8(nil), XXX_sizecache:0}: rpc error: code = Unknown desc = unable to open database file: no such file or directory
Mar 13 19:26:27 newnas.local k3s[85484]: E0313 19:26:27.831693   85484 replica_set.go:532] sync "kube-system/coredns-7448499f4d" failed with rpc error: code = Unknown desc = unable to open database file: no such file or directory
Mar 13 19:26:27 newnas.local k3s[85484]: time="2022-03-13T19:26:27.832225325-04:00" level=error msg="error in txn: unable to open database file: no such file or directory"
Mar 13 19:26:27 newnas.local k3s[85484]: E0313 19:26:27.832379   85484 status.go:71] apiserver received an error that is not an metav1.Status: &status.statusError{Code:2, Message:"unable to open database file: no such file or directory", Details:[]*anypb.Any(nil), XXX_NoUnkeyedLiteral:struct {}{}, XXX_unrecognized:[]uint8(nil), XXX_sizecache:0}: rpc error: code = Unknown desc = unable to open database file: no such file or directory
Mar 13 19:26:27 newnas.local k3s[85484]: E0313 19:26:27.847912   85484 daemon_controller.go:320] kube-system/openebs-zfs-node failed with : error storing status for daemon set &v1.DaemonSet{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"openebs-zfs-node", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"c5c936f7-6075-4b8a-a129-cf6813a48cfb", ResourceVersion:"528", Generation:1, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:63782810270, loc:(*time.Location)(0x8e40740)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{"objectset.rio.cattle.io/hash":"4b1efd47d7f37713706abe9f8b0de3f0adf02e2c", "openebs.io/component-name":"openebs-zfs-node", "openebs.io/version":"ci"}, Annotations:map[string]string{"deprecated.daemonset.template.generation":"1", "objectset.rio.cattle.io/applied":"H4sIAAAAAAAA/6xWUY/aOBD+K5XVk1opIVDabhtpHyjkrqveAoK2L9UKOfYk+HDsyHayy63y3092AgQI2z1p3xJ75puZbz6P/Yhwzn6C0kwKFCKc5zooB8hDGyYoCtEEQybFEgzyUAYGU2wwCh8RFkIabJgU2v7K+B8gRoPpKSZ7BBvDocdkwCwG8i7uy3sByk/LDQrRZqhbO+XAe/WNCXo9olSK30IInAEK0b+J9mUOChupnuWjc0ys46aIwddbbSBDlYc4joE/Wdka6zUK0ft4AAl9f0WvkuHV1WB41f+IY/icfIr7FIZJH9Ok/w7eEZtMDgJibZ2JzHIpQJhd3s2eb/MXksKxebnvD2E2u8tOT5SkcyC2IA0ciKUnfEQZNmT9975WnOddsFXlIQNZzrEB59XSAf+d84vV7SEl+YmPS21XGJHCYCZAaRT+ekRYpfYD+X55/QF5yPeJZj6mVIHW16/fjCaTRbRcvnVblisOxleQMm2UE7afY7O+fv1msrj5GS1Wi+iv1XI2/raaj75/fYvuPASidJGaWhpA5KES88KuBDkvUiYCollPS7JBlbe37kJtu5ZYBZzFQZNZA6UDyxSXBPO87MT99uNLtJrOJtFqOrqNdoh/KplZjhIGnC4g2X/PsbEythz2bAOmFqWqWoAOq862ld9RE+48xDKcOtF90r2UKNs9zVJfG6lwCjZT11+fKlaC2vOswvJdb9DrowZhXnA+l5yRLQrRTTKVZq5Ag7Djh7MEyJZwp8FcwdLI3H7Cw67/WYbt0PqFgpiJQK9ta+3JU9krXyWvgnZ320Re3rGJ1hTfWVJ2nFwsxzIkeZHBrSyEqYWY2c+G56aNaI9U//uUKdfEI9t2UgePI4k6vzvreZC7TczaXr9+M5tH0+jL8qCHWu0gaC6ZMC2L8fJmFU0n89nN9HttVGd2ClILoUv+Z7FeQHtd2bVEWAj2EAbBU8esI/kWAE6ttLrMR7fRcj4aR+eSP5J7s+YkU2shHPQ+94bP0XPHCNyrQwMpFDPbsRQGHoybr5zL+7liJeOQQqQJ5rUyQqMK8FC+26L1SvWSUqRQHgwplIxAt+FaZhBsYKsP5iCI8t3SmbW259SOkMPJWispjV+v6SJuLN2YOY+l3bPErSmZ47QhBH2V2nyX492FcIC3Lr6NYKcCYDoTfLuj6wT9dAB3R/rCKFNA7B/mLS4l1b4z3x3RO88Fn4K5l2rTapp0feZY62ntW1/Z9XwhihlGHLIGZWkfEWJhO+5OX+P9+KnbbQPW5Tyi/KiTZpu7S8jlLtW29ao46W4XRrvJp0gzNVZgHwrVBQEQKRKW3mI3uykkuODm1l794fDjh65TkZd+zEQLr6WRSxk+XeJBBZf8L9y+q2b4brsCdFR+Pqv/X7yj2/55IU9O8XOCPU3WuZQrDxU5xQaWtjRItxZcSc6ZSH+4jfpt+fBD4BIzjmP3bhv0+39Y3CbW4sihqqr/AgAA//9ZF4yTiwwAAA", "objectset.rio.cattle.io/id":"", "objectset.rio.cattle.io/owner-gvk":"k3s.cattle.io/v1, Kind=Addon", "objectset.rio.cattle.io/owner-name":"zfs-operator", "objectset.rio.cattle.io/owner-namespace":"kube-system"}, OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry{v1.ManagedFieldsEntry{Manager:"k3s", Operation:"Update", APIVersion:"apps/v1", Time:(*v1.Time)(0xc015be3b80), FieldsType:"FieldsV1", FieldsV1:(*v1.FieldsV1)(0xc015be3c00)}}}, Spec:v1.DaemonSetSpec{Selector:(*v1.LabelSelector)(0xc015be3c80), Template:v1.PodTemplateSpec{ObjectMeta:v1.ObjectMeta{Name:"", GenerateName:"", Namespace:"", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{"app":"openebs-zfs-node", "openebs.io/component-name":"openebs-zfs-node", "openebs.io/version":"ci", "role":"openebs-zfs"}, Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.PodSpec{Volumes:[]v1.Volume{v1.Volume{Name:"device-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc015be3d00), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"encr-keys", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc015be3d80), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"chroot-zfs", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(nil), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(0xc01763aec0), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"host-root", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc015be3e00), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"registration-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc015be3e80), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"plugin-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc015be3f00), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"pods-mount-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc015be3f80), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}}, InitContainers:[]v1.Container(nil), Containers:[]v1.Container{v1.Container{Name:"csi-node-driver-registrar", Image:"k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0", Command:[]string(nil), Args:[]string{"--v=5", "--csi-address=$(ADDRESS)", "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"}, WorkingDir:"", Ports:[]v1.ContainerPort(nil), EnvFrom:[]v1.EnvFromSource(nil), Env:[]v1.EnvVar{v1.EnvVar{Name:"ADDRESS", Value:"/plugin/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"DRIVER_REG_SOCK_PATH", Value:"/var/lib/kubelet/plugins/zfs-localpv/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"KUBE_NODE_NAME", Value:"", ValueFrom:(*v1.EnvVarSource)(0xc007eec060)}, v1.EnvVar{Name:"NODE_DRIVER", Value:"openebs-zfs", ValueFrom:(*v1.EnvVarSource)(nil)}}, Resources:v1.ResourceRequirements{Limits:v1.ResourceList(nil), Requests:v1.ResourceList(nil)}, VolumeMounts:[]v1.VolumeMount{v1.VolumeMount{Name:"plugin-dir", ReadOnly:false, MountPath:"/plugin", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"registration-dir", ReadOnly:false, MountPath:"/registration", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}}, VolumeDevices:[]v1.VolumeDevice(nil), LivenessProbe:(*v1.Probe)(nil), ReadinessProbe:(*v1.Probe)(nil), StartupProbe:(*v1.Probe)(nil), Lifecycle:(*v1.Lifecycle)(0xc00b6a9390), TerminationMessagePath:"/dev/termination-log", TerminationMessagePolicy:"File", ImagePullPolicy:"IfNotPresent", SecurityContext:(*v1.SecurityContext)(nil), Stdin:false, StdinOnce:false, TTY:false}, v1.Container{Name:"openebs-zfs-plugin", Image:"openebs/zfs-driver:1.9.3", Command:[]string(nil), Args:[]string{"--nodename=$(OPENEBS_NODE_NAME)", "--endpoint=$(OPENEBS_CSI_ENDPOINT)", "--plugin=$(OPENEBS_NODE_DRIVER)"}, WorkingDir:"", Ports:[]v1.ContainerPort(nil), EnvFrom:[]v1.EnvFromSource(nil), Env:[]v1.EnvVar{v1.EnvVar{Name:"OPENEBS_NODE_NAME", Value:"", ValueFrom:(*v1.EnvVarSource)(0xc007eec120)}, v1.EnvVar{Name:"OPENEBS_CSI_ENDPOINT", Value:"unix:///plugin/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"OPENEBS_NODE_DRIVER", Value:"agent", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"OPENEBS_NAMESPACE", Value:"openebs", ValueFrom:(*v1.EnvVarSource)(nil)}}, Resources:v1.ResourceRequirements{Limits:v1.ResourceList(nil), Requests:v1.ResourceList(nil)}, VolumeMounts:[]v1.VolumeMount{v1.VolumeMount{Name:"plugin-dir", ReadOnly:false, MountPath:"/plugin", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"device-dir", ReadOnly:false, MountPath:"/dev", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"encr-keys", ReadOnly:false, MountPath:"/home/keys", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"chroot-zfs", ReadOnly:false, MountPath:"/sbin/zfs", SubPath:"zfs", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"host-root", ReadOnly:true, MountPath:"/host", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(0xc00b6a93c0), SubPathExpr:""}, v1.VolumeMount{Name:"pods-mount-dir", ReadOnly:false, MountPath:"/var/lib/kubelet/", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(0xc00b6a93d0), SubPathExpr:""}}, VolumeDevices:[]v1.VolumeDevice(nil), LivenessProbe:(*v1.Probe)(nil), ReadinessProbe:(*v1.Probe)(nil), StartupProbe:(*v1.Probe)(nil), Lifecycle:(*v1.Lifecycle)(nil), TerminationMessagePath:"/dev/termination-log", TerminationMessagePolicy:"File", ImagePullPolicy:"IfNotPresent", SecurityContext:(*v1.SecurityContext)(0xc00b77e6c0), Stdin:false, StdinOnce:false, TTY:false}}, EphemeralContainers:[]v1.EphemeralContainer(nil), RestartPolicy:"Always", TerminationGracePeriodSeconds:(*int64)(0xc010e26f90), ActiveDeadlineSeconds:(*int64)(nil), DNSPolicy:"ClusterFirst", NodeSelector:map[string]string(nil), ServiceAccountName:"openebs-zfs-node-sa", DeprecatedServiceAccount:"openebs-zfs-node-sa", AutomountServiceAccountToken:(*bool)(nil), NodeName:"", HostNetwork:true, HostPID:false, HostIPC:false, ShareProcessNamespace:(*bool)(nil), SecurityContext:(*v1.PodSecurityContext)(0xc000e23570), ImagePullSecrets:[]v1.LocalObjectReference(nil), Hostname:"", Subdomain:"", Affinity:(*v1.Affinity)(nil), SchedulerName:"default-scheduler", Tolerations:[]v1.Toleration(nil), HostAliases:[]v1.HostAlias(nil), PriorityClassName:"system-node-critical", Priority:(*int32)(nil), DNSConfig:(*v1.PodDNSConfig)(nil), ReadinessGates:[]v1.PodReadinessGate(nil), RuntimeClassName:(*string)(nil), EnableServiceLinks:(*bool)(nil), PreemptionPolicy:(*v1.PreemptionPolicy)(nil), Overhead:v1.ResourceList(nil), TopologySpreadConstraints:[]v1.TopologySpreadConstraint(nil), SetHostnameAsFQDN:(*bool)(nil)}}, UpdateStrategy:v1.DaemonSetUpdateStrategy{Type:"RollingUpdate", RollingUpdate:(*v1.RollingUpdateDaemonSet)(0xc00b6a93e0)}, MinReadySeconds:0, RevisionHistoryLimit:(*int32)(0xc010e26f9c)}, Status:v1.DaemonSetStatus{CurrentNumberScheduled:1, NumberMisscheduled:0, DesiredNumberScheduled:1, NumberReady:1, ObservedGeneration:1, UpdatedNumberScheduled:1, NumberAvailable:1, NumberUnavailable:0, CollisionCount:(*int32)(nil), Conditions:[]v1.DaemonSetCondition(nil)}}: Operation cannot be fulfilled on daemonsets.apps "openebs-zfs-node": the object has been modified; please apply your changes to the latest version and try again
Mar 13 19:26:27 newnas.local k3s[85484]: E0313 19:26:27.922444   85484 kuberuntime_container.go:534] "PreStop hook failed" err="command '/bin/sh -c rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock' exited with 126: " pod="kube-system/openebs-zfs-node-dvdqr" podUID=d0250d8e-250a-4afb-83f0-4733388cf253 containerName="csi-node-driver-registrar" containerID="docker://f1c83259b8336fc0bd9e02dc88013cac555d43ff92a8a19e8b61ccc06042f3e5"
Mar 13 19:26:28 newnas.local k3s[85484]: I0313 19:26:28.483639   85484 csi_plugin.go:178] kubernetes.io/csi: registrationHandler.DeRegisterPlugin request for plugin zfs.csi.openebs.io
Mar 13 19:26:28 newnas.local k3s[85484]: time="2022-03-13T19:26:28.485746823-04:00" level=info msg="Cluster-Http-Server 2022/03/13 19:26:28 http: TLS handshake error from 127.0.0.1:42588: EOF"
Mar 13 19:26:28 newnas.local k3s[85484]: time="2022-03-13T19:26:28.498535814-04:00" level=info msg="Cluster-Http-Server 2022/03/13 19:26:28 http: TLS handshake error from 127.0.0.1:42590: EOF"
Mar 13 19:26:28 newnas.local k3s[85484]: time="2022-03-13T19:26:28.551118866-04:00" level=info msg="Cluster-Http-Server 2022/03/13 19:26:28 http: TLS handshake error from 127.0.0.1:42592: EOF"
Mar 13 19:26:28 newnas.local k3s[85484]: I0313 19:26:28.739803   85484 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": unexpected command output  with error: fork/exec /mnt/pool 0/ix-applications/k3s/data/94499178817b4c5ce4973492f034bfaed731d47d3de22485bfc180619d2a1cc7/bin/nsenter: no such file or directory"
Mar 13 19:26:28 newnas.local k3s[85484]: time="2022-03-13T19:26:28.804242338-04:00" level=info msg="Cluster-Http-Server 2022/03/13 19:26:28 http: TLS handshake error from 127.0.0.1:42594: EOF"
Mar 13 19:26:28 newnas.local k3s[85484]: E0313 19:26:28.804309   85484 csi_plugin.go:180] kubernetes.io/csi: registrationHandler.DeRegisterPlugin failed: kubernetes.io/csi: Error uninstalling CSI driver: error uninstalling CSI driver from CSINode object error updating CSINode: timed out waiting for the condition; caused by: Get "https://127.0.0.1:6443/apis/storage.k8s.io/v1/csinodes/ix-truenas": open /mnt/pool 0/ix-applications/k3s/agent/client-kubelet.crt: no such file or directory
Mar 13 19:26:28 newnas.local k3s[85484]: I0313 19:26:28.975934   85484 scope.go:111] "RemoveContainer" containerID="4c047c5d6751e9488cde89246ec287d1e34059e6ff72df25b2f18f8544a4d4f0"
Mar 13 19:26:30 newnas.local k3s[85484]: I0313 19:26:30.037273   85484 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/3de30deb-19f7-4f61-990d-5b4884319e7c-config-volume\") pod \"3de30deb-19f7-4f61-990d-5b4884319e7c\" (UID: \"3de30deb-19f7-4f61-990d-5b4884319e7c\") "
Mar 13 19:26:30 newnas.local k3s[85484]: I0313 19:26:30.037334   85484 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"kube-api-access-p686h\" (UniqueName: \"kubernetes.io/projected/3de30deb-19f7-4f61-990d-5b4884319e7c-kube-api-access-p686h\") pod \"3de30deb-19f7-4f61-990d-5b4884319e7c\" (UID: \"3de30deb-19f7-4f61-990d-5b4884319e7c\") "
Mar 13 19:26:30 newnas.local k3s[85484]: W0313 19:26:30.038139   85484 empty_dir.go:520] Warning: Failed to clear quota on /var/lib/kubelet/pods/3de30deb-19f7-4f61-990d-5b4884319e7c/volumes/kubernetes.io~configmap/config-volume: clearQuota called, but quotas disabled
Mar 13 19:26:30 newnas.local k3s[85484]: I0313 19:26:30.038723   85484 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/configmap/3de30deb-19f7-4f61-990d-5b4884319e7c-config-volume" (OuterVolumeSpecName: "config-volume") pod "3de30deb-19f7-4f61-990d-5b4884319e7c" (UID: "3de30deb-19f7-4f61-990d-5b4884319e7c"). InnerVolumeSpecName "config-volume". PluginName "kubernetes.io/configmap", VolumeGidValue ""
Mar 13 19:26:30 newnas.local k3s[85484]: I0313 19:26:30.040217   85484 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/3de30deb-19f7-4f61-990d-5b4884319e7c-kube-api-access-p686h" (OuterVolumeSpecName: "kube-api-access-p686h") pod "3de30deb-19f7-4f61-990d-5b4884319e7c" (UID: "3de30deb-19f7-4f61-990d-5b4884319e7c"). InnerVolumeSpecName "kube-api-access-p686h". PluginName "kubernetes.io/projected", VolumeGidValue ""
Mar 13 19:26:30 newnas.local k3s[85484]: I0313 19:26:30.114803   85484 scope.go:111] "RemoveContainer" containerID="4c047c5d6751e9488cde89246ec287d1e34059e6ff72df25b2f18f8544a4d4f0"
Mar 13 19:26:30 newnas.local k3s[85484]: E0313 19:26:30.115550   85484 remote_runtime.go:334] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 4c047c5d6751e9488cde89246ec287d1e34059e6ff72df25b2f18f8544a4d4f0" containerID="4c047c5d6751e9488cde89246ec287d1e34059e6ff72df25b2f18f8544a4d4f0"
Mar 13 19:26:30 newnas.local k3s[85484]: I0313 19:26:30.115600   85484 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:4c047c5d6751e9488cde89246ec287d1e34059e6ff72df25b2f18f8544a4d4f0} err="failed to get container status \"4c047c5d6751e9488cde89246ec287d1e34059e6ff72df25b2f18f8544a4d4f0\": rpc error: code = Unknown desc = Error: No such container: 4c047c5d6751e9488cde89246ec287d1e34059e6ff72df25b2f18f8544a4d4f0"
Mar 13 19:26:30 newnas.local k3s[85484]: I0313 19:26:30.137575   85484 reconciler.go:319] "Volume detached for volume \"kube-api-access-p686h\" (UniqueName: \"kubernetes.io/projected/3de30deb-19f7-4f61-990d-5b4884319e7c-kube-api-access-p686h\") on node \"ix-truenas\" DevicePath \"\""
Mar 13 19:26:30 newnas.local k3s[85484]: I0313 19:26:30.137606   85484 reconciler.go:319] "Volume detached for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/3de30deb-19f7-4f61-990d-5b4884319e7c-config-volume\") on node \"ix-truenas\" DevicePath \"\""
Mar 13 19:26:31 newnas.local k3s[85484]: time="2022-03-13T19:26:31.410448826-04:00" level=info msg="Cluster-Http-Server 2022/03/13 19:26:31 http: TLS handshake error from 127.0.0.1:42598: EOF"
Mar 13 19:26:31 newnas.local k3s[85484]: E0313 19:26:31.410446   85484 event.go:273] Unable to write event: '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"coredns-7448499f4d-8whz7.16dc146504233cab", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"coredns-7448499f4d-8whz7", UID:"3de30deb-19f7-4f61-990d-5b4884319e7c", APIVersion:"v1", ResourceVersion:"473", FieldPath:"spec.containers{coredns}"}, Reason:"Killing", Message:"Stopping container coredns", Source:v1.EventSource{Component:"kubelet", Host:"ix-truenas"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc083bd48eff29eab, ext:526528137638, loc:(*time.Location)(0x8e40740)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc083bd48eff29eab, ext:526528137638, loc:(*time.Location)(0x8e40740)}}, Count:1, Type:"Normal", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, Series:(*v1.EventSeries)(nil), Action:"", Related:(*v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'Post "https://127.0.0.1:6443/api/v1/namespaces/kube-system/events": open /mnt/pool 0/ix-applications/k3s/agent/client-kubelet.crt: no such file or directory'(may retry after sleeping)
Mar 13 19:26:35 newnas.local k3s[85484]: time="2022-03-13T19:26:35.868936397-04:00" level=error msg="Failed to process config: lstat /mnt/pool 0/ix-applications/k3s/server/manifests: no such file or directory"
Mar 13 19:26:38 newnas.local systemd[1]: Stopping Lightweight Kubernetes...
Mar 13 19:26:38 newnas.local k3s[85484]: time="2022-03-13T19:26:38.120199618-04:00" level=fatal msg="controllers exited"
Mar 13 19:26:38 newnas.local k3s[85484]: time="2022-03-13T19:26:38.120187526-04:00" level=info msg="Shutting down /v1, Kind=Secret workers"
Mar 13 19:26:38 newnas.local k3s[85484]: I0313 19:26:38.120251   85484 controller.go:185] Shutting down kubernetes service endpoint reconciler
Mar 13 19:26:38 newnas.local systemd[1]: k3s.service: Main process exited, code=exited, status=1/FAILURE
Mar 13 19:26:38 newnas.local systemd[1]: k3s.service: Failed with result 'exit-code'.
Mar 13 19:26:38 newnas.local systemd[1]: Stopped Lightweight Kubernetes.
Mar 13 19:29:54 newnas.local systemd[1]: Starting Lightweight Kubernetes...
Mar 13 19:29:54 newnas.local k3s[142022]: time="2022-03-13T19:29:54-04:00" level=info msg="Acquiring lock file /mnt/pool 0/ix-applications/k3s/data/.lock"
Mar 13 19:29:54 newnas.local k3s[142022]: time="2022-03-13T19:29:54-04:00" level=info msg="Preparing data dir /mnt/pool 0/ix-applications/k3s/data/94499178817b4c5ce4973492f034bfaed731d47d3de22485bfc180619d2a1cc7"
Mar 13 19:29:57 newnas.local k3s[142022]: time="2022-03-13T19:29:57.675141067-04:00" level=info msg="Starting k3s TS-22.02-RC.2 (7f6d3a72)"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.232226929-04:00" level=info msg="Configuring sqlite3 database connection pooling: maxIdleConns=2, maxOpenConns=0, connMaxLifetime=0s"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.232335528-04:00" level=info msg="Configuring database table schema and indexes, this may take a moment..."
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.235722141-04:00" level=info msg="Database tables and indexes are up to date"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.238148675-04:00" level=info msg="Kine listening on unix://kine.sock"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.263340570-04:00" level=info msg="certificate CN=system:admin,O=system:masters signed by CN=k3s-client-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.264405679-04:00" level=info msg="certificate CN=system:kube-controller-manager signed by CN=k3s-client-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.265232048-04:00" level=info msg="certificate CN=system:kube-scheduler signed by CN=k3s-client-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.266094385-04:00" level=info msg="certificate CN=kube-apiserver signed by CN=k3s-client-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.267138431-04:00" level=info msg="certificate CN=system:kube-proxy signed by CN=k3s-client-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.267826151-04:00" level=info msg="certificate CN=system:k3s-controller signed by CN=k3s-client-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.268646193-04:00" level=info msg="certificate CN=cloud-controller-manager signed by CN=k3s-client-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.270245849-04:00" level=info msg="certificate CN=kube-apiserver signed by CN=k3s-server-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.271539311-04:00" level=info msg="certificate CN=system:auth-proxy signed by CN=k3s-request-header-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.272751181-04:00" level=info msg="certificate CN=etcd-server signed by CN=etcd-server-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.273788674-04:00" level=info msg="certificate CN=etcd-client signed by CN=etcd-server-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.275219921-04:00" level=info msg="certificate CN=etcd-peer signed by CN=etcd-peer-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.694854414-04:00" level=info msg="certificate CN=k3s,O=k3s signed by CN=k3s-server-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:29:58 +0000 UTC"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.697076777-04:00" level=info msg="Active TLS secret  (ver=) (count 8): map[listener.cattle.io/cn-0.0.0.0:0.0.0.0 listener.cattle.io/cn-127.0.0.1:127.0.0.1 listener.cattle.io/cn-172.17.0.1:172.17.0.1 listener.cattle.io/cn-192.168.1.147:192.168.1.147 listener.cattle.io/cn-kubernetes:kubernetes listener.cattle.io/cn-kubernetes.default:kubernetes.default listener.cattle.io/cn-kubernetes.default.svc.cluster.local:kubernetes.default.svc.cluster.local listener.cattle.io/cn-localhost:localhost listener.cattle.io/fingerprint:SHA1=AF9012260895B46BE7DF9AB8D37DA235D6520938]"
Mar 13 19:29:58 newnas.local k3s[142022]: time="2022-03-13T19:29:58.701635860-04:00" level=info msg="Running kube-apiserver --advertise-address=0.0.0.0 --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/log/k3s_server_audit.log --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/mnt/pool 0/ix-applications/k3s/server/tls/temporary-certs --client-ca-file=/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.crt --enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount --etcd-servers=unix://kine.sock --insecure-port=0 --kubelet-certificate-authority=/mnt/pool 0/ix-applications/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/mnt/pool 0/ix-applications/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/mnt/pool 0/ix-applications/k3s/server/tls/client-kube-apiserver.key --profiling=false --proxy-client-cert-file=/mnt/pool 0/ix-applications/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/mnt/pool 0/ix-applications/k3s/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/mnt/pool 0/ix-applications/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/mnt/pool 0/ix-applications/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/mnt/pool 0/ix-applications/k3s/server/tls/service.key --service-cluster-ip-range=172.17.0.0/16 --service-node-port-range=9000-65535 --storage-backend=etcd3 --tls-cert-file=/mnt/pool 0/ix-applications/k3s/server/tls/serving-kube-apiserver.crt --tls-private-key-file=/mnt/pool 0/ix-applications/k3s/server/tls/serving-kube-apiserver.key"
Mar 13 19:29:58 newnas.local k3s[142022]: Flag --insecure-port has been deprecated, This flag has no effect now and will be removed in v1.24.
Mar 13 19:29:58 newnas.local k3s[142022]: I0313 19:29:58.707262  142022 server.go:656] external host was not specified, using 192.168.1.147
Mar 13 19:29:58 newnas.local k3s[142022]: I0313 19:29:58.707781  142022 server.go:195] Version: v1.21.0-k3s1
Mar 13 19:29:59 newnas.local k3s[142022]: I0313 19:29:59.763868  142022 shared_informer.go:240] Waiting for caches to sync for node_authorizer
Mar 13 19:29:59 newnas.local k3s[142022]: I0313 19:29:59.766457  142022 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
Mar 13 19:29:59 newnas.local k3s[142022]: I0313 19:29:59.766475  142022 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
Mar 13 19:29:59 newnas.local k3s[142022]: I0313 19:29:59.768566  142022 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
Mar 13 19:29:59 newnas.local k3s[142022]: I0313 19:29:59.768581  142022 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
Mar 13 19:29:59 newnas.local k3s[142022]: I0313 19:29:59.798308  142022 instance.go:283] Using reconciler: lease
Mar 13 19:29:59 newnas.local k3s[142022]: I0313 19:29:59.846819  142022 rest.go:130] the default service ipfamily for this cluster is: IPv4
Mar 13 19:30:00 newnas.local k3s[142022]: W0313 19:30:00.280079  142022 genericapiserver.go:425] Skipping API node.k8s.io/v1alpha1 because it has no resources.
Mar 13 19:30:00 newnas.local k3s[142022]: W0313 19:30:00.293150  142022 genericapiserver.go:425] Skipping API rbac.authorization.k8s.io/v1alpha1 because it has no resources.
Mar 13 19:30:00 newnas.local k3s[142022]: W0313 19:30:00.297031  142022 genericapiserver.go:425] Skipping API scheduling.k8s.io/v1alpha1 because it has no resources.
Mar 13 19:30:00 newnas.local k3s[142022]: W0313 19:30:00.305778  142022 genericapiserver.go:425] Skipping API storage.k8s.io/v1alpha1 because it has no resources.
Mar 13 19:30:00 newnas.local k3s[142022]: W0313 19:30:00.308718  142022 genericapiserver.go:425] Skipping API flowcontrol.apiserver.k8s.io/v1alpha1 because it has no resources.
Mar 13 19:30:00 newnas.local k3s[142022]: W0313 19:30:00.315180  142022 genericapiserver.go:425] Skipping API apps/v1beta2 because it has no resources.
Mar 13 19:30:00 newnas.local k3s[142022]: W0313 19:30:00.315202  142022 genericapiserver.go:425] Skipping API apps/v1beta1 because it has no resources.
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.336876  142022 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.336912  142022 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.355224052-04:00" level=info msg="Running kube-scheduler --address=127.0.0.1 --bind-address=127.0.0.1 --kubeconfig=/mnt/pool 0/ix-applications/k3s/server/cred/scheduler.kubeconfig --leader-elect=false --port=10251 --profiling=false --secure-port=0"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.355343074-04:00" level=info msg="Waiting for API server to become available"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.356909572-04:00" level=info msg="Running kube-controller-manager --address=127.0.0.1 --allocate-node-cidrs=true --bind-address=127.0.0.1 --cluster-cidr=172.16.0.0/16 --cluster-signing-cert-file=/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.crt --cluster-signing-key-file=/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.key --kubeconfig=/mnt/pool 0/ix-applications/k3s/server/cred/controller.kubeconfig --leader-elect=false --node-cidr-mask-size=16 --port=10252 --profiling=false --root-ca-file=/mnt/pool 0/ix-applications/k3s/server/tls/server-ca.crt --secure-port=0 --service-account-private-key-file=/mnt/pool 0/ix-applications/k3s/server/tls/service.key --terminated-pod-gc-threshold=5 --use-service-account-credentials=true"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.358924821-04:00" level=info msg="Node token is available at /mnt/pool 0/ix-applications/k3s/server/token"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.359166979-04:00" level=info msg="To join node to cluster: k3s agent -s https://192.168.1.147:6443 -t ${NODE_TOKEN}"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.360896083-04:00" level=info msg="Wrote kubeconfig /etc/rancher/k3s/k3s.yaml"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.360954233-04:00" level=info msg="Run: k3s kubectl"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.402702743-04:00" level=info msg="Cluster-Http-Server 2022/03/13 19:30:00 http: TLS handshake error from 127.0.0.1:42632: remote error: tls: bad certificate"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.409454371-04:00" level=info msg="Cluster-Http-Server 2022/03/13 19:30:00 http: TLS handshake error from 127.0.0.1:42638: remote error: tls: bad certificate"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.424078104-04:00" level=info msg="certificate CN=ix-truenas signed by CN=k3s-server-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:30:00 +0000 UTC"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.427723723-04:00" level=info msg="certificate CN=system:node:ix-truenas,O=system:nodes signed by CN=k3s-client-ca@1647214198: notBefore=2022-03-13 23:29:58 +0000 UTC notAfter=2023-03-13 23:30:00 +0000 UTC"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.433629848-04:00" level=info msg="Module overlay was already loaded"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.433665927-04:00" level=info msg="Module nf_conntrack was already loaded"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.433681388-04:00" level=info msg="Module br_netfilter was already loaded"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.433696347-04:00" level=info msg="Module iptable_nat was already loaded"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.448645563-04:00" level=info msg="Connecting to proxy" url="wss://127.0.0.1:6443/v1-k3s/connect"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.452450291-04:00" level=info msg="Handling backend connection request [ix-truenas]"
Mar 13 19:30:00 newnas.local k3s[142022]: time="2022-03-13T19:30:00.454463035-04:00" level=info msg="Running kubelet --address=0.0.0.0 --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --client-ca-file=/mnt/pool 0/ix-applications/k3s/agent/client-ca.crt --cluster-dns=172.17.0.10 --cluster-domain=cluster.local --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ix-truenas --kubeconfig=/mnt/pool 0/ix-applications/k3s/agent/kubelet.kubeconfig --network-plugin=cni --node-ip=0.0.0.0 --node-labels= --pod-infra-container-image=docker.io/rancher/pause:3.1 --pod-manifest-path=/mnt/pool 0/ix-applications/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/etc/resolv.conf --tls-cert-file=/mnt/pool 0/ix-applications/k3s/agent/serving-kubelet.crt --tls-private-key-file=/mnt/pool 0/ix-applications/k3s/agent/serving-kubelet.key"
Mar 13 19:30:00 newnas.local k3s[142022]: Flag --network-plugin has been deprecated, will be removed along with dockershim.
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.472342  142022 server.go:436] "Kubelet version" kubeletVersion="v1.21.0-k3s1"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.518353  142022 dynamic_cafile_content.go:167] Starting client-ca-bundle::/mnt/pool 0/ix-applications/k3s/agent/client-ca.crt
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.646107  142022 server.go:660] "--cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.646911  142022 container_manager_linux.go:291] "Container manager verified user specified cgroup-root exists" cgroupRoot=[]
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.647043  142022 container_manager_linux.go:296] "Creating Container Manager object based on Node Config" nodeConfig={RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:true NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalTopologyManagerScope:container ExperimentalCPUManagerReconcilePeriod:10s ExperimentalMemoryManagerPolicy:None ExperimentalMemoryManagerReservedMemory:[] ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none Rootless:false}
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.647284  142022 topology_manager.go:120] "Creating topology manager with policy per scope" topologyPolicyName="none" topologyScopeName="container"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.647320  142022 container_manager_linux.go:327] "Initializing Topology Manager" policy="none" scope="container"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.647333  142022 container_manager_linux.go:332] "Creating device plugin manager" devicePluginEnabled=true
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.648069  142022 kubelet.go:310] "Using dockershim is deprecated, please consider using a full-fledged CRI implementation"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.648639  142022 client.go:78] "Connecting to docker on the dockerEndpoint" endpoint="unix:///var/run/docker.sock"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.648699  142022 client.go:97] "Start docker client with request timeout" timeout="2m0s"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.658388  142022 docker_service.go:566] "Hairpin mode is set but kubenet is not enabled, falling back to HairpinVeth" hairpinMode=promiscuous-bridge
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.658439  142022 docker_service.go:242] "Hairpin mode is set" hairpinMode=hairpin-veth
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.658844  142022 cni.go:239] "Unable to update cni config" err="no networks found in /etc/cni/net.d"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.660932  142022 cni.go:239] "Unable to update cni config" err="no networks found in /etc/cni/net.d"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.661006  142022 docker_service.go:257] "Docker cri networking managed by the network plugin" networkPluginName="cni"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.661135  142022 cni.go:239] "Unable to update cni config" err="no networks found in /etc/cni/net.d"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.675196  142022 docker_service.go:264] "Docker Info" dockerInfo=&{ID:GHP7:EGSO:W52B:O4I2:RKIS:SVNR:4BJT:DUOL:NUC7:YD3C:K7O3:ACI3 Containers:10 ContainersRunning:0 ContainersPaused:0 ContainersStopped:10 Images:11 Driver:zfs DriverStatus:[[Zpool boot-pool] [Zpool Health ONLINE] [Parent Dataset boot-pool/ROOT/22.02.RELEASE] [Space Used By Parent 3635728384] [Space Available 26513686528] [Parent Quota no] [Compression on]] SystemStatus:[] Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:[] Log:[awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:true KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6tables:true Debug:false NFd:22 OomKillDisable:true NGoroutines:35 SystemTime:2022-03-13T19:30:00.661901543-04:00 LoggingDriver:json-file CgroupDriver:cgroupfs CgroupVersion:1 NEventsListener:0 KernelVersion:5.10.93+truenas OperatingSystem:Debian GNU/Linux 11 (bullseye) OSVersion:11 OSType:linux Architecture:x86_64 IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:0xc0006b9110 NCPU:16 MemTotal:75978170368 GenericResources:[] DockerRootDir:/mnt/pool\ 0/ix-applications/docker HTTPProxy: HTTPSProxy: NoProxy: Name:newnas.local Labels:[] ExperimentalBuild:false ServerVersion:20.10.12 ClusterStore: ClusterAdvertise: Runtimes:map[io.containerd.runc.v2:{Path:runc Args:[] Shim:<nil>} io.containerd.runtime.v1.linux:{Path:runc Args:[] Shim:<nil>} runc:{Path:runc Args:[] Shim:<nil>}] DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:[] Nodes:0 Managers:0 Cluster:<nil> Warnings:[]} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:7b11cfaabd73bb80907dd23182b9347b4245eb5d Expected:7b11cfaabd73bb80907dd23182b9347b4245eb5d} RuncCommit:{ID:v1.0.2-0-g52b36a2 Expected:v1.0.2-0-g52b36a2} InitCommit:{ID:de40ad0 Expected:de40ad0} SecurityOptions:[name=apparmor name=seccomp,profile=default] ProductLicense: DefaultAddressPools:[] Warnings:[]}
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.675225  142022 docker_service.go:277] "Setting cgroupDriver" cgroupDriver="cgroupfs"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.689947  142022 kubelet.go:275] "Adding static pod path" path="/mnt/pool 0/ix-applications/k3s/agent/pod-manifests"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.690101  142022 kubelet.go:286] "Watching apiserver"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.690450  142022 kubelet.go:465] "Kubelet client is not nil"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.700926  142022 kuberuntime_manager.go:222] "Container runtime initialized" containerRuntime="docker" version="20.10.12" apiVersion="1.41.0"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.703182  142022 server.go:1191] "Started kubelet"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.703520  142022 server.go:149] "Starting to listen" address="0.0.0.0" port=10250
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.703543  142022 kubelet.go:1313] "Image garbage collection failed once. Stats initialization may not have completed yet" err="failed to get imageFs info: unable to find data in memory cache"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.705256  142022 server.go:405] "Adding debug handlers to kubelet server"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.705873  142022 fs_resource_analyzer.go:67] "Starting FS ResourceAnalyzer"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.706079  142022 volume_manager.go:271] "Starting Kubelet Volume Manager"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.706166  142022 desired_state_of_world_populator.go:141] "Desired state populator starts to run"
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.715685  142022 remote_runtime.go:144] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"coredns-7448499f4d-8whz7_kube-system\" network: cni config uninitialized" podSandboxID="44fe819055da9eefe1b7b40e1fc66bb21e6a41fb4810d9e2a467171c2e628085"
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.715733  142022 kuberuntime_gc.go:176] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"coredns-7448499f4d-8whz7_kube-system\" network: cni config uninitialized" sandboxID="44fe819055da9eefe1b7b40e1fc66bb21e6a41fb4810d9e2a467171c2e628085"
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.717806  142022 nodelease.go:49] "Failed to get node when trying to set owner ref to the node lease" err="nodes \"ix-truenas\" not found" node="ix-truenas"
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.718039  142022 kubelet.go:2218] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.728254  142022 kubelet_network_linux.go:56] "Initialized protocol iptables rules." protocol=IPv4
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.806639  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.846647  142022 kubelet_network_linux.go:56] "Initialized protocol iptables rules." protocol=IPv6
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.846775  142022 status_manager.go:157] "Starting to sync pod status with apiserver"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.846848  142022 kubelet.go:1853] "Starting kubelet main sync loop"
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.847006  142022 kubelet.go:1877] "Skipping pod synchronization" err="[container runtime status check may not have completed yet, PLEG is not healthy: pleg has yet to be successful]"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.854715  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"coredns-7448499f4d-8whz7_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"44fe819055da9eefe1b7b40e1fc66bb21e6a41fb4810d9e2a467171c2e628085\""
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.866429  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.876615  142022 kubelet_node_status.go:71] "Attempting to register node" node="ix-truenas"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.905632  142022 cpu_manager.go:199] "Starting CPU manager" policy="none"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.905655  142022 cpu_manager.go:200] "Reconciling" reconcilePeriod="10s"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.905716  142022 state_mem.go:36] "Initialized new in-memory state store"
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.907751  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.923494  142022 policy_none.go:44] "None policy: Start"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.924786  142022 manager.go:600] "Failed to retrieve checkpoint" checkpoint="kubelet_internal_checkpoint" err="checkpoint is not found"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.925419  142022 plugin_manager.go:114] "Starting Kubelet Plugin Manager"
Mar 13 19:30:00 newnas.local k3s[142022]: E0313 19:30:00.925447  142022 eviction_manager.go:255] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ix-truenas\" not found"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.947260  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="44fe819055da9eefe1b7b40e1fc66bb21e6a41fb4810d9e2a467171c2e628085"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.947299  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="a2275507c8615921f21b1b58f83dbd67e5944b447b5acabe10d4a2ed467673f4"
Mar 13 19:30:00 newnas.local k3s[142022]: I0313 19:30:00.947324  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.007940  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: I0313 19:30:01.107065  142022 reconciler.go:157] "Reconciler: start to sync state"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.108283  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.209333  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.309939  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.411036  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.511140  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.611478  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.696174  142022 csi_plugin.go:295] Failed to initialize CSINode: error updating CSINode annotation: timed out waiting for the condition; caused by: nodes "ix-truenas" not found
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.712580  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.813348  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:01 newnas.local k3s[142022]: E0313 19:30:01.914002  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.014100  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.114466  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.215224  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.315493  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.416336  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.516511  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.620425  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.720532  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.821143  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:02 newnas.local k3s[142022]: I0313 19:30:02.853758  142022 kubelet_getters.go:311] "Path does not exist" path="/var/lib/kubelet/pods/742c3cb4-eaf0-4bf6-a757-fd1186bda100/volumes"
Mar 13 19:30:02 newnas.local k3s[142022]: I0313 19:30:02.853797  142022 kubelet_getters.go:311] "Path does not exist" path="/var/lib/kubelet/pods/d0250d8e-250a-4afb-83f0-4733388cf253/volumes"
Mar 13 19:30:02 newnas.local k3s[142022]: E0313 19:30:02.922166  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:03 newnas.local k3s[142022]: E0313 19:30:03.022656  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:03 newnas.local k3s[142022]: E0313 19:30:03.122913  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:03 newnas.local k3s[142022]: E0313 19:30:03.223400  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:03 newnas.local k3s[142022]: E0313 19:30:03.293597  142022 csi_plugin.go:295] Failed to initialize CSINode: error updating CSINode annotation: timed out waiting for the condition; caused by: nodes "ix-truenas" not found
Mar 13 19:30:03 newnas.local k3s[142022]: E0313 19:30:03.323821  142022 kubelet.go:2298] "Error getting node" err="node \"ix-truenas\" not found"
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.325372  142022 dynamic_cafile_content.go:167] Starting client-ca-bundle::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.crt
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.325384  142022 dynamic_cafile_content.go:167] Starting request-header::/mnt/pool 0/ix-applications/k3s/server/tls/request-header-ca.crt
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.325657  142022 dynamic_serving_content.go:130] Starting serving-cert::/mnt/pool 0/ix-applications/k3s/server/tls/serving-kube-apiserver.crt::/mnt/pool 0/ix-applications/k3s/server/tls/serving-kube-apiserver.key
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.326605  142022 tlsconfig.go:240] Starting DynamicServingCertificateController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.326608  142022 secure_serving.go:197] Serving securely on 127.0.0.1:6444
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.326927  142022 dynamic_serving_content.go:130] Starting aggregator-proxy-cert::/mnt/pool 0/ix-applications/k3s/server/tls/client-auth-proxy.crt::/mnt/pool 0/ix-applications/k3s/server/tls/client-auth-proxy.key
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.326949  142022 apiservice_controller.go:97] Starting APIServiceRegistrationController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.326960  142022 controller.go:83] Starting OpenAPI AggregationController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.326985  142022 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327010  142022 available_controller.go:475] Starting AvailableConditionController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327026  142022 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327024  142022 apf_controller.go:294] Starting API Priority and Fairness config controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327096  142022 crdregistration_controller.go:111] Starting crd-autoregister controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327115  142022 shared_informer.go:240] Waiting for caches to sync for crd-autoregister
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327268  142022 autoregister_controller.go:141] Starting autoregister controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327285  142022 cache.go:32] Waiting for caches to sync for autoregister controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327315  142022 establishing_controller.go:76] Starting EstablishingController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327357  142022 customresource_discovery_controller.go:209] Starting DiscoveryController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327402  142022 controller.go:86] Starting OpenAPI controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327591  142022 nonstructuralschema_controller.go:192] Starting NonStructuralSchemaConditionController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327658  142022 apiapproval_controller.go:186] Starting KubernetesAPIApprovalPolicyConformantConditionController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327674  142022 naming_controller.go:291] Starting NamingConditionController
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.327708  142022 crd_finalizer.go:266] Starting CRDFinalizer
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.328182  142022 cluster_authentication_trust_controller.go:440] Starting cluster_authentication_trust_controller controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.328376  142022 shared_informer.go:240] Waiting for caches to sync for cluster_authentication_trust_controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.328513  142022 dynamic_cafile_content.go:167] Starting client-ca-bundle::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.crt
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.328648  142022 dynamic_cafile_content.go:167] Starting request-header::/mnt/pool 0/ix-applications/k3s/server/tls/request-header-ca.crt
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.359522  142022 controller.go:611] quota admission added evaluator for: namespaces
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.363941  142022 shared_informer.go:247] Caches are synced for node_authorizer
Mar 13 19:30:03 newnas.local k3s[142022]: E0313 19:30:03.370318  142022 controller.go:151] Unable to perform initial Kubernetes service initialization: Service "kubernetes" is invalid: spec.clusterIPs: Invalid value: []string{"172.17.0.1"}: failed to allocated ip:172.17.0.1 with error:cannot allocate resources of type serviceipallocations at this time
Mar 13 19:30:03 newnas.local k3s[142022]: E0313 19:30:03.370988  142022 controller.go:156] Unable to remove old endpoints from kubernetes service: StorageError: key not found, Code: 1, Key: /registry/masterleases/192.168.1.147, ResourceVersion: 0, AdditionalErrorMsg:
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.406277  142022 trace.go:205] Trace[704028957]: "Create" url:/api/v1/nodes,user-agent:k3s/v1.21.0 (linux/amd64) kubernetes/bcdd3fe,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf,application/json,protocol:HTTP/1.1 (13-Mar-2022 19:30:01.293) (total time: 2113ms):
Mar 13 19:30:03 newnas.local k3s[142022]: Trace[704028957]: ---"Object stored in database" 2112ms (19:30:00.405)
Mar 13 19:30:03 newnas.local k3s[142022]: Trace[704028957]: [2.113144712s] [2.113144712s] END
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.406749  142022 kubelet_node_status.go:74] "Successfully registered node" node="ix-truenas"
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.427126  142022 apf_controller.go:299] Running API Priority and Fairness config worker
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.427156  142022 cache.go:39] Caches are synced for APIServiceRegistrationController controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.427207  142022 shared_informer.go:247] Caches are synced for crd-autoregister
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.427254  142022 cache.go:39] Caches are synced for AvailableConditionController controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.427338  142022 cache.go:39] Caches are synced for autoregister controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.428547  142022 shared_informer.go:247] Caches are synced for cluster_authentication_trust_controller
Mar 13 19:30:03 newnas.local k3s[142022]: I0313 19:30:03.429267  142022 trace.go:205] Trace[1553304334]: "Create" url:/api/v1/namespaces/default/events,user-agent:k3s/v1.21.0 (linux/amd64) kubernetes/bcdd3fe,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf,application/json,protocol:HTTP/1.1 (13-Mar-2022 19:30:00.710) (total time: 2718ms):
Mar 13 19:30:03 newnas.local k3s[142022]: Trace[1553304334]: ---"Object stored in database" 2717ms (19:30:00.428)
Mar 13 19:30:03 newnas.local k3s[142022]: Trace[1553304334]: [2.718287709s] [2.718287709s] END
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.325520  142022 controller.go:132] OpenAPI AggregationController: action for item : Nothing (removed from the queue).
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.325560  142022 controller.go:132] OpenAPI AggregationController: action for item k8s_internal_local_delegation_chain_0000000000: Nothing (removed from the queue).
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.331337  142022 storage_scheduling.go:132] created PriorityClass system-node-critical with value 2000001000
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.334455  142022 storage_scheduling.go:132] created PriorityClass system-cluster-critical with value 2000000000
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.334474  142022 storage_scheduling.go:148] all system priority classes are created successfully or already exist.
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.646340  142022 controller.go:611] quota admission added evaluator for: roles.rbac.authorization.k8s.io
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.673654  142022 controller.go:611] quota admission added evaluator for: rolebindings.rbac.authorization.k8s.io
Mar 13 19:30:04 newnas.local k3s[142022]: W0313 19:30:04.792185  142022 lease.go:233] Resetting endpoints for master service "kubernetes" to [192.168.1.147]
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.793085  142022 controller.go:611] quota admission added evaluator for: endpoints
Mar 13 19:30:04 newnas.local k3s[142022]: I0313 19:30:04.796619  142022 controller.go:611] quota admission added evaluator for: endpointslices.discovery.k8s.io
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.344705002-04:00" level=info msg="Kube API server is now running"
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.344773450-04:00" level=info msg="k3s is up and running"
Mar 13 19:30:05 newnas.local k3s[142022]: Flag --address has been deprecated, see --bind-address instead.
Mar 13 19:30:05 newnas.local systemd[1]: Started Lightweight Kubernetes.
Mar 13 19:30:05 newnas.local k3s[142022]: I0313 19:30:05.350553  142022 controllermanager.go:175] Version: v1.21.0-k3s1
Mar 13 19:30:05 newnas.local k3s[142022]: I0313 19:30:05.350946  142022 deprecated_insecure_serving.go:53] Serving insecurely on 127.0.0.1:10252
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.355225512-04:00" level=info msg="Creating CRD addons.k3s.cattle.io"
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.360734638-04:00" level=info msg="Creating CRD helmcharts.helm.cattle.io"
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.363805230-04:00" level=info msg="Creating CRD helmchartconfigs.helm.cattle.io"
Mar 13 19:30:05 newnas.local k3s[142022]: W0313 19:30:05.365944  142022 authorization.go:47] Authorization is disabled
Mar 13 19:30:05 newnas.local k3s[142022]: W0313 19:30:05.365963  142022 authentication.go:47] Authentication is disabled
Mar 13 19:30:05 newnas.local k3s[142022]: I0313 19:30:05.365977  142022 deprecated_insecure_serving.go:51] Serving healthz insecurely on 127.0.0.1:10251
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.371412741-04:00" level=info msg="Waiting for CRD addons.k3s.cattle.io to become available"
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.451239192-04:00" level=warning msg="Unable to watch for tunnel endpoints: unknown (get endpoints)"
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.481055009-04:00" level=info msg="labels have been set successfully on node: ix-truenas"
Mar 13 19:30:05 newnas.local k3s[142022]: I0313 19:30:05.661472  142022 cni.go:239] "Unable to update cni config" err="no networks found in /etc/cni/net.d"
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.874061553-04:00" level=info msg="Done waiting for CRD addons.k3s.cattle.io to become available"
Mar 13 19:30:05 newnas.local k3s[142022]: time="2022-03-13T19:30:05.874091539-04:00" level=info msg="Waiting for CRD helmcharts.helm.cattle.io to become available"
Mar 13 19:30:05 newnas.local k3s[142022]: E0313 19:30:05.934872  142022 kubelet.go:2218] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.377077843-04:00" level=info msg="Done waiting for CRD helmcharts.helm.cattle.io to become available"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.377119185-04:00" level=info msg="Waiting for CRD helmchartconfigs.helm.cattle.io to become available"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.880289426-04:00" level=info msg="Done waiting for CRD helmchartconfigs.helm.cattle.io to become available"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.896084082-04:00" level=info msg="Writing static file: /mnt/pool 0/ix-applications/k3s/server/static/charts/traefik-9.14.2.tgz"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.896469675-04:00" level=info msg="Writing static file: /mnt/pool 0/ix-applications/k3s/server/static/charts/traefik-crd-9.14.2.tgz"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.896864733-04:00" level=info msg="Writing manifest: /mnt/pool 0/ix-applications/k3s/server/manifests/multus-daemonset.yaml"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.897878898-04:00" level=info msg="Writing manifest: /mnt/pool 0/ix-applications/k3s/server/manifests/zfs-operator.yaml"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.898064704-04:00" level=info msg="Writing manifest: /mnt/pool 0/ix-applications/k3s/server/manifests/kuberouter.yaml"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.898195108-04:00" level=info msg="Writing manifest: /mnt/pool 0/ix-applications/k3s/server/manifests/rolebindings.yaml"
Mar 13 19:30:06 newnas.local k3s[142022]: time="2022-03-13T19:30:06.898396152-04:00" level=info msg="Writing manifest: /mnt/pool 0/ix-applications/k3s/server/manifests/coredns.yaml"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:06.999765848-04:00" level=info msg="Starting k3s.cattle.io/v1, Kind=Addon controller"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.008520298-04:00" level=info msg="Cluster dns configmap has been set successfully"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.010171589-04:00" level=info msg="Control-plane role label has been set successfully on node: ix-truenas"
Mar 13 19:30:07 newnas.local k3s[142022]: I0313 19:30:07.013584  142022 controller.go:611] quota admission added evaluator for: serviceaccounts
Mar 13 19:30:07 newnas.local k3s[142022]: I0313 19:30:07.060154  142022 controller.go:611] quota admission added evaluator for: deployments.apps
Mar 13 19:30:07 newnas.local k3s[142022]: I0313 19:30:07.077247  142022 controller.go:611] quota admission added evaluator for: addons.k3s.cattle.io
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.102381177-04:00" level=info msg="Starting batch/v1, Kind=Job controller"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.102357558-04:00" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChartConfig controller"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.102371739-04:00" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChart controller"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.102440985-04:00" level=info msg="Starting /v1, Kind=Pod controller"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.102441373-04:00" level=info msg="Starting /v1, Kind=Node controller"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.102448257-04:00" level=info msg="Starting /v1, Kind=Endpoints controller"
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.102422580-04:00" level=info msg="Starting /v1, Kind=Service controller"
Mar 13 19:30:07 newnas.local k3s[142022]: I0313 19:30:07.171022  142022 request.go:668] Waited for 1.048611205s due to client-side throttling, not priority and fairness, request: GET:https://127.0.0.1:6444/apis/apiextensions.k8s.io/v1beta1?timeout=32s
Mar 13 19:30:07 newnas.local k3s[142022]: time="2022-03-13T19:30:07.203627876-04:00" level=info msg="Updated coredns node hosts entry [192.168.1.147 ix-truenas]"
Mar 13 19:30:07 newnas.local k3s[142022]: I0313 19:30:07.378441  142022 controller.go:611] quota admission added evaluator for: statefulsets.apps
Mar 13 19:30:07 newnas.local k3s[142022]: I0313 19:30:07.400850  142022 controller.go:611] quota admission added evaluator for: daemonsets.apps
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.176502  142022 shared_informer.go:240] Waiting for caches to sync for tokens
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.195172  142022 controllermanager.go:574] Started "horizontalpodautoscaling"
Mar 13 19:30:08 newnas.local k3s[142022]: W0313 19:30:08.195192  142022 controllermanager.go:553] "bootstrapsigner" is disabled
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.195246  142022 horizontal.go:169] Starting HPA controller
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.195258  142022 shared_informer.go:240] Waiting for caches to sync for HPA
Mar 13 19:30:08 newnas.local k3s[142022]: E0313 19:30:08.203047  142022 core.go:91] Failed to start service controller: WARNING: no cloud provider provided, services of type LoadBalancer will fail
Mar 13 19:30:08 newnas.local k3s[142022]: W0313 19:30:08.203070  142022 controllermanager.go:566] Skipping "service"
Mar 13 19:30:08 newnas.local k3s[142022]: W0313 19:30:08.203134  142022 core.go:245] configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.
Mar 13 19:30:08 newnas.local k3s[142022]: W0313 19:30:08.203150  142022 controllermanager.go:566] Skipping "route"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.211967  142022 controllermanager.go:574] Started "ttl-after-finished"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.212132  142022 ttlafterfinished_controller.go:109] Starting TTL after finished controller
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.212147  142022 shared_informer.go:240] Waiting for caches to sync for TTL after finished
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.222149  142022 controllermanager.go:574] Started "csrapproving"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.222244  142022 certificate_controller.go:118] Starting certificate controller "csrapproving"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.222258  142022 shared_informer.go:240] Waiting for caches to sync for certificate-csrapproving
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.231895  142022 controllermanager.go:574] Started "pvc-protection"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.232030  142022 pvc_protection_controller.go:110] "Starting PVC protection controller"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.232044  142022 shared_informer.go:240] Waiting for caches to sync for PVC protection
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.241281  142022 controllermanager.go:574] Started "endpointslice"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.241379  142022 endpointslice_controller.go:256] Starting endpoint slice controller
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.241395  142022 shared_informer.go:240] Waiting for caches to sync for endpoint_slice
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.270132  142022 controllermanager.go:574] Started "namespace"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.270230  142022 namespace_controller.go:200] Starting namespace controller
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.270243  142022 shared_informer.go:240] Waiting for caches to sync for namespace
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.276602  142022 shared_informer.go:247] Caches are synced for tokens
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.281654  142022 controllermanager.go:574] Started "replicaset"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.281754  142022 replica_set.go:182] Starting replicaset controller
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.281768  142022 shared_informer.go:240] Waiting for caches to sync for ReplicaSet
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.285773  142022 certificate_controller.go:118] Starting certificate controller "csrsigning-kubelet-serving"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.285795  142022 shared_informer.go:240] Waiting for caches to sync for certificate-csrsigning-kubelet-serving
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.285820  142022 dynamic_serving_content.go:130] Starting csr-controller::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.crt::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.key
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.286423  142022 certificate_controller.go:118] Starting certificate controller "csrsigning-kubelet-client"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.286440  142022 shared_informer.go:240] Waiting for caches to sync for certificate-csrsigning-kubelet-client
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.286451  142022 dynamic_serving_content.go:130] Starting csr-controller::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.crt::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.key
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.286998  142022 certificate_controller.go:118] Starting certificate controller "csrsigning-kube-apiserver-client"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.287019  142022 shared_informer.go:240] Waiting for caches to sync for certificate-csrsigning-kube-apiserver-client
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.287032  142022 dynamic_serving_content.go:130] Starting csr-controller::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.crt::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.key
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.287444  142022 controllermanager.go:574] Started "csrsigning"
Mar 13 19:30:08 newnas.local k3s[142022]: W0313 19:30:08.287465  142022 controllermanager.go:553] "tokencleaner" is disabled
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.287542  142022 certificate_controller.go:118] Starting certificate controller "csrsigning-legacy-unknown"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.287559  142022 shared_informer.go:240] Waiting for caches to sync for certificate-csrsigning-legacy-unknown
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.287586  142022 dynamic_serving_content.go:130] Starting csr-controller::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.crt::/mnt/pool 0/ix-applications/k3s/server/tls/client-ca.key
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.332879  142022 controllermanager.go:574] Started "persistentvolume-expander"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.332952  142022 expand_controller.go:327] Starting expand controller
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.332962  142022 shared_informer.go:240] Waiting for caches to sync for expand
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.483098  142022 controllermanager.go:574] Started "clusterrole-aggregation"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.483164  142022 clusterroleaggregation_controller.go:194] Starting ClusterRoleAggregator
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.483177  142022 shared_informer.go:240] Waiting for caches to sync for ClusterRoleAggregator
Mar 13 19:30:08 newnas.local k3s[142022]: time="2022-03-13T19:30:08.503582675-04:00" level=info msg="Starting /v1, Kind=Secret controller"
Mar 13 19:30:08 newnas.local k3s[142022]: time="2022-03-13T19:30:08.602317043-04:00" level=info msg="Active TLS secret k3s-serving (ver=304) (count 8): map[listener.cattle.io/cn-0.0.0.0:0.0.0.0 listener.cattle.io/cn-127.0.0.1:127.0.0.1 listener.cattle.io/cn-172.17.0.1:172.17.0.1 listener.cattle.io/cn-192.168.1.147:192.168.1.147 listener.cattle.io/cn-kubernetes:kubernetes listener.cattle.io/cn-kubernetes.default:kubernetes.default listener.cattle.io/cn-kubernetes.default.svc.cluster.local:kubernetes.default.svc.cluster.local listener.cattle.io/cn-localhost:localhost listener.cattle.io/fingerprint:SHA1=AF9012260895B46BE7DF9AB8D37DA235D6520938]"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.631088  142022 controllermanager.go:574] Started "pv-protection"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.631137  142022 pv_protection_controller.go:83] Starting PV protection controller
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.631147  142022 shared_informer.go:240] Waiting for caches to sync for PV protection
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.783308  142022 controllermanager.go:574] Started "endpointslicemirroring"
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.783378  142022 endpointslicemirroring_controller.go:211] Starting EndpointSliceMirroring controller
Mar 13 19:30:08 newnas.local k3s[142022]: I0313 19:30:08.783392  142022 shared_informer.go:240] Waiting for caches to sync for endpoint_slice_mirroring
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090510  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for podtemplates
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090712  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for helmcharts.helm.cattle.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090761  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for controllerrevisions.apps
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090819  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for events.events.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090848  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for jobs.batch
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090882  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for helmchartconfigs.helm.cattle.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090917  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for network-attachment-definitions.k8s.cni.cncf.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090952  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for volumesnapshots.snapshot.storage.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.090984  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for limitranges
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091017  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for zfsrestores.zfs.openebs.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091067  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for cronjobs.batch
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091102  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for networkpolicies.networking.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091174  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for endpointslices.discovery.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091214  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for zfsbackups.zfs.openebs.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091259  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for deployments.apps
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091288  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for statefulsets.apps
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091322  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for roles.rbac.authorization.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091359  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for csistoragecapacities.storage.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091408  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for ingresses.networking.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091438  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for replicasets.apps
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091480  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for daemonsets.apps
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091541  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for ingresses.extensions
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091588  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for poddisruptionbudgets.policy
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091623  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for rolebindings.rbac.authorization.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091694  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for leases.coordination.k8s.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091731  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for zfssnapshots.zfs.openebs.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091826  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for zfsvolumes.zfs.openebs.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091865  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for zfsnodes.zfs.openebs.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091893  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for endpoints
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091958  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for serviceaccounts
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.091987  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for horizontalpodautoscalers.autoscaling
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.092033  142022 resource_quota_monitor.go:229] QuotaMonitor created object count evaluator for addons.k3s.cattle.io
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.092069  142022 controllermanager.go:574] Started "resourcequota"
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.092107  142022 resource_quota_controller.go:273] Starting resource quota controller
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.092124  142022 shared_informer.go:240] Waiting for caches to sync for resource quota
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.092185  142022 resource_quota_monitor.go:304] QuotaMonitor running
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.233877  142022 controllermanager.go:574] Started "cronjob"
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.233904  142022 cronjob_controllerv2.go:125] Starting cronjob controller v2
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.233918  142022 shared_informer.go:240] Waiting for caches to sync for cronjob
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.383984  142022 controllermanager.go:574] Started "ttl"
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.384047  142022 ttl_controller.go:121] Starting TTL controller
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.384064  142022 shared_informer.go:240] Waiting for caches to sync for TTL
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.533189  142022 node_lifecycle_controller.go:76] Sending events to api server
Mar 13 19:30:09 newnas.local k3s[142022]: E0313 19:30:09.533232  142022 core.go:231] failed to start cloud node lifecycle controller: no cloud provider provided
Mar 13 19:30:09 newnas.local k3s[142022]: W0313 19:30:09.533245  142022 controllermanager.go:566] Skipping "cloud-node-lifecycle"
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.682001  142022 controllermanager.go:574] Started "persistentvolume-binder"
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.682065  142022 pv_controller_base.go:308] Starting persistent volume controller
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.682076  142022 shared_informer.go:240] Waiting for caches to sync for persistent volume
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.830973  142022 controllermanager.go:574] Started "attachdetach"
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.831014  142022 attach_detach_controller.go:327] Starting attach detach controller
Mar 13 19:30:09 newnas.local k3s[142022]: I0313 19:30:09.831024  142022 shared_informer.go:240] Waiting for caches to sync for attach detach
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.029132  142022 controllermanager.go:574] Started "disruption"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.029197  142022 disruption.go:363] Starting disruption controller
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.029210  142022 shared_informer.go:240] Waiting for caches to sync for disruption
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.181761  142022 controllermanager.go:574] Started "statefulset"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.181810  142022 stateful_set.go:146] Starting stateful set controller
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.181821  142022 shared_informer.go:240] Waiting for caches to sync for stateful set
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.230833  142022 controllermanager.go:574] Started "csrcleaner"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.230904  142022 cleaner.go:82] Starting CSR cleaner controller
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.381436  142022 controllermanager.go:574] Started "ephemeral-volume"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.381499  142022 controller.go:170] Starting ephemeral volume controller
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.381513  142022 shared_informer.go:240] Waiting for caches to sync for ephemeral
Mar 13 19:30:10 newnas.local k3s[142022]: time="2022-03-13T19:30:10.453414448-04:00" level=info msg="Stopped tunnel to 127.0.0.1:6443"
Mar 13 19:30:10 newnas.local k3s[142022]: time="2022-03-13T19:30:10.453429694-04:00" level=info msg="Proxy done" err="context canceled" url="wss://127.0.0.1:6443/v1-k3s/connect"
Mar 13 19:30:10 newnas.local k3s[142022]: time="2022-03-13T19:30:10.453426281-04:00" level=info msg="Connecting to proxy" url="wss://192.168.1.147:6443/v1-k3s/connect"
Mar 13 19:30:10 newnas.local k3s[142022]: time="2022-03-13T19:30:10.453754686-04:00" level=info msg="error in remotedialer server [400]: websocket: close 1006 (abnormal closure): unexpected EOF"
Mar 13 19:30:10 newnas.local k3s[142022]: time="2022-03-13T19:30:10.456726567-04:00" level=info msg="Handling backend connection request [ix-truenas]"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.531502  142022 controllermanager.go:574] Started "serviceaccount"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.531571  142022 serviceaccounts_controller.go:117] Starting service account controller
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.531588  142022 shared_informer.go:240] Waiting for caches to sync for service account
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.661809  142022 cni.go:239] "Unable to update cni config" err="no networks found in /etc/cni/net.d"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.680938  142022 controllermanager.go:574] Started "job"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.680999  142022 job_controller.go:150] Starting job controller
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.681013  142022 shared_informer.go:240] Waiting for caches to sync for job
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.734156  142022 controller.go:611] quota admission added evaluator for: leases.coordination.k8s.io
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.832025  142022 controllermanager.go:574] Started "deployment"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.832207  142022 deployment_controller.go:153] "Starting controller" controller="deployment"
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.832221  142022 shared_informer.go:240] Waiting for caches to sync for deployment
Mar 13 19:30:10 newnas.local k3s[142022]: I0313 19:30:10.881392  142022 node_ipam_controller.go:91] Sending events to api server.
Mar 13 19:30:10 newnas.local k3s[142022]: E0313 19:30:10.944361  142022 kubelet.go:2218] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.904172  142022 range_allocator.go:82] Sending events to api server.
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.904371  142022 range_allocator.go:110] No Service CIDR provided. Skipping filtering out service addresses.
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.904385  142022 range_allocator.go:116] No Secondary Service CIDR provided. Skipping filtering out secondary service addresses.
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.904430  142022 controllermanager.go:574] Started "nodeipam"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.904561  142022 node_ipam_controller.go:154] Starting ipam controller
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.904571  142022 shared_informer.go:240] Waiting for caches to sync for node
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.913115  142022 controllermanager.go:574] Started "endpoint"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.913140  142022 endpoints_controller.go:189] Starting endpoint controller
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.913156  142022 shared_informer.go:240] Waiting for caches to sync for endpoint
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.920878  142022 controllermanager.go:574] Started "replicationcontroller"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.920970  142022 replica_set.go:182] Starting replicationcontroller controller
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.920986  142022 shared_informer.go:240] Waiting for caches to sync for ReplicationController
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.928295  142022 controllermanager.go:574] Started "podgc"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.928417  142022 gc_controller.go:89] Starting GC controller
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.928433  142022 shared_informer.go:240] Waiting for caches to sync for GC
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.941675  142022 garbagecollector.go:142] Starting garbage collector controller
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.941697  142022 shared_informer.go:240] Waiting for caches to sync for garbage collector
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.941733  142022 graph_builder.go:289] GraphBuilder running
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.941801  142022 controllermanager.go:574] Started "garbagecollector"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.950098  142022 controllermanager.go:574] Started "daemonset"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.950229  142022 daemon_controller.go:285] Starting daemon sets controller
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.950245  142022 shared_informer.go:240] Waiting for caches to sync for daemon sets
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.953808  142022 node_lifecycle_controller.go:377] Sending events to api server.
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.954383  142022 taint_manager.go:163] "Sending events to api server"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.954690  142022 node_lifecycle_controller.go:505] Controller will reconcile labels.
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.954736  142022 controllermanager.go:574] Started "nodelifecycle"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.954760  142022 node_lifecycle_controller.go:539] Starting node controller
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.954778  142022 shared_informer.go:240] Waiting for caches to sync for taint
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.964581  142022 controllermanager.go:574] Started "root-ca-cert-publisher"
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.965088  142022 shared_informer.go:240] Waiting for caches to sync for resource quota
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.965500  142022 publisher.go:102] Starting root CA certificate configmap publisher
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.965519  142022 shared_informer.go:240] Waiting for caches to sync for crt configmap
Mar 13 19:30:20 newnas.local k3s[142022]: W0313 19:30:20.975906  142022 actual_state_of_world.go:534] Failed to update statusUpdateNeeded field in actual state of world: Failed to set statusUpdateNeeded to needed true, because nodeName="ix-truenas" does not exist
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.981697  142022 shared_informer.go:247] Caches are synced for ephemeral
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.981825  142022 shared_informer.go:247] Caches are synced for ReplicaSet
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.982011  142022 shared_informer.go:247] Caches are synced for stateful set
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.982290  142022 shared_informer.go:247] Caches are synced for persistent volume
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.983713  142022 shared_informer.go:247] Caches are synced for endpoint_slice_mirroring
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.984106  142022 shared_informer.go:247] Caches are synced for TTL
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.988586  142022 shared_informer.go:240] Waiting for caches to sync for garbage collector
Mar 13 19:30:20 newnas.local k3s[142022]: I0313 19:30:20.996708  142022 controller.go:611] quota admission added evaluator for: controllerrevisions.apps
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.004718  142022 shared_informer.go:247] Caches are synced for node
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.004760  142022 range_allocator.go:172] Starting range CIDR allocator
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.004774  142022 shared_informer.go:240] Waiting for caches to sync for cidrallocator
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.004782  142022 shared_informer.go:247] Caches are synced for cidrallocator
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.008847  142022 event.go:291] "Event occurred" object="kube-system/openebs-zfs-controller" kind="StatefulSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="create Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.009390  142022 range_allocator.go:373] Set node ix-truenas PodCIDR to [172.16.0.0/16]
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.013243  142022 shared_informer.go:247] Caches are synced for endpoint
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.014356  142022 controller.go:611] quota admission added evaluator for: events.events.k8s.io
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.021120  142022 shared_informer.go:247] Caches are synced for ReplicationController
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.028491  142022 shared_informer.go:247] Caches are synced for GC
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.029311  142022 shared_informer.go:247] Caches are synced for disruption
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.029327  142022 disruption.go:371] Sending events to api server.
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.031064  142022 shared_informer.go:247] Caches are synced for attach detach
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.031258  142022 shared_informer.go:247] Caches are synced for PV protection
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.032134  142022 shared_informer.go:247] Caches are synced for service account
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.032144  142022 shared_informer.go:247] Caches are synced for PVC protection
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.032269  142022 shared_informer.go:247] Caches are synced for deployment
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.033509  142022 shared_informer.go:247] Caches are synced for expand
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.041723  142022 shared_informer.go:247] Caches are synced for endpoint_slice
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.051174  142022 shared_informer.go:247] Caches are synced for daemon sets
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.055454  142022 shared_informer.go:247] Caches are synced for taint
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.055630  142022 taint_manager.go:187] "Starting NoExecuteTaintManager"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.055669  142022 node_lifecycle_controller.go:1398] Initializing eviction metric for zone:
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.055747  142022 event.go:291] "Event occurred" object="ix-truenas" kind="Node" apiVersion="v1" type="Normal" reason="RegisteredNode" message="Node ix-truenas event: Registered Node ix-truenas in Controller"
Mar 13 19:30:21 newnas.local k3s[142022]: W0313 19:30:21.055758  142022 node_lifecycle_controller.go:1013] Missing timestamp for Node ix-truenas. Assuming now as a timestamp.
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.055823  142022 node_lifecycle_controller.go:1164] Controller detected that all Nodes are not-Ready. Entering master disruption mode.
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.065630  142022 shared_informer.go:247] Caches are synced for crt configmap
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.071088  142022 shared_informer.go:247] Caches are synced for namespace
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.071242  142022 kuberuntime_manager.go:1044] "Updating runtime config through cri with podcidr" CIDR="172.16.0.0/16"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.072079  142022 docker_service.go:359] "Docker cri received runtime config" runtimeConfig="&RuntimeConfig{NetworkConfig:&NetworkConfig{PodCidr:172.16.0.0/16,},}"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.072189  142022 kubelet_network.go:76] "Updating Pod CIDR" originalPodCIDR="" newPodCIDR="172.16.0.0/16"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.081070  142022 shared_informer.go:247] Caches are synced for job
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.092421  142022 controller.go:611] quota admission added evaluator for: replicasets.apps
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.094404  142022 event.go:291] "Event occurred" object="kube-system/coredns" kind="Deployment" apiVersion="apps/v1" type="Normal" reason="ScalingReplicaSet" message="Scaled up replica set coredns-7448499f4d to 1"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.112255  142022 shared_informer.go:247] Caches are synced for TTL after finished
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.122292  142022 shared_informer.go:247] Caches are synced for certificate-csrapproving
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.133967  142022 shared_informer.go:247] Caches are synced for cronjob
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.190905  142022 shared_informer.go:247] Caches are synced for certificate-csrsigning-kubelet-client
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.190918  142022 shared_informer.go:247] Caches are synced for certificate-csrsigning-kubelet-serving
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.190926  142022 shared_informer.go:247] Caches are synced for certificate-csrsigning-kube-apiserver-client
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.190894  142022 shared_informer.go:247] Caches are synced for certificate-csrsigning-legacy-unknown
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.190926  142022 shared_informer.go:247] Caches are synced for ClusterRoleAggregator
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.292186  142022 shared_informer.go:247] Caches are synced for resource quota
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.295599  142022 shared_informer.go:247] Caches are synced for HPA
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.365202  142022 shared_informer.go:247] Caches are synced for resource quota
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.504830  142022 event.go:291] "Event occurred" object="kube-system/openebs-zfs-node" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: openebs-zfs-node-l5t7c"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.510194  142022 topology_manager.go:187] "Topology Admit Handler"
Mar 13 19:30:21 newnas.local k3s[142022]: E0313 19:30:21.515691  142022 daemon_controller.go:320] kube-system/openebs-zfs-node failed with : error storing status for daemon set &v1.DaemonSet{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"openebs-zfs-node", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"7c7ff1b5-22eb-4a26-9451-c6b198492519", ResourceVersion:"298", Generation:1, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:63782811007, loc:(*time.Location)(0x8e40740)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{"objectset.rio.cattle.io/hash":"4b1efd47d7f37713706abe9f8b0de3f0adf02e2c", "openebs.io/component-name":"openebs-zfs-node", "openebs.io/version":"ci"}, Annotations:map[string]string{"deprecated.daemonset.template.generation":"1", "objectset.rio.cattle.io/applied":"H4sIAAAAAAAA/6xWUY/aOBD+K5XVk1opIVDabhtpHyjkrqveAoK2L9UKOfYk+HDsyHayy63y3092AgQI2z1p3xJ75puZbz6P/Yhwzn6C0kwKFCKc5zooB8hDGyYoCtEEQybFEgzyUAYGU2wwCh8RFkIabJgU2v7K+B8gRoPpKSZ7BBvDocdkwCwG8i7uy3sByk/LDQrRZqhbO+XAe/WNCXo9olSK30IInAEK0b+J9mUOChupnuWjc0ys46aIwddbbSBDlYc4joE/Wdka6zUK0ft4AAl9f0WvkuHV1WB41f+IY/icfIr7FIZJH9Ok/w7eEZtMDgJibZ2JzHIpQJhd3s2eb/MXksKxebnvD2E2u8tOT5SkcyC2IA0ciKUnfEQZNmT9975WnOddsFXlIQNZzrEB59XSAf+d84vV7SEl+YmPS21XGJHCYCZAaRT+ekRYpfYD+X55/QF5yPeJZj6mVIHW16/fjCaTRbRcvnVblisOxleQMm2UE7afY7O+fv1msrj5GS1Wi+iv1XI2/raaj75/fYvuPASidJGaWhpA5KES88KuBDkvUiYCollPS7JBlbe37kJtu5ZYBZzFQZNZA6UDyxSXBPO87MT99uNLtJrOJtFqOrqNdoh/KplZjhIGnC4g2X/PsbEythz2bAOmFqWqWoAOq862ld9RE+48xDKcOtF90r2UKNs9zVJfG6lwCjZT11+fKlaC2vOswvJdb9DrowZhXnA+l5yRLQrRTTKVZq5Ag7Djh7MEyJZwp8FcwdLI3H7Cw67/WYbt0PqFgpiJQK9ta+3JU9krXyWvgnZ320Re3rGJ1hTfWVJ2nFwsxzIkeZHBrSyEqYWY2c+G56aNaI9U//uUKdfEI9t2UgePI4k6vzvreZC7TczaXr9+M5tH0+jL8qCHWu0gaC6ZMC2L8fJmFU0n89nN9HttVGd2ClILoUv+Z7FeQHtd2bVEWAj2EAbBU8esI/kWAE6ttLrMR7fRcj4aR+eSP5J7s+YkU2shHPQ+94bP0XPHCNyrQwMpFDPbsRQGHoybr5zL+7liJeOQQqQJ5rUyQqMK8FC+26L1SvWSUqRQHgwplIxAt+FaZhBsYKsP5iCI8t3SmbW259SOkMPJWispjV+v6SJuLN2YOY+l3bPErSmZ47QhBH2V2nyX492FcIC3Lr6NYKcCYDoTfLuj6wT9dAB3R/rCKFNA7B/mLS4l1b4z3x3RO88Fn4K5l2rTapp0feZY62ntW1/Z9XwhihlGHLIGZWkfEWJhO+5OX+P9+KnbbQPW5Tyi/KiTZpu7S8jlLtW29ao46W4XRrvJp0gzNVZgHwrVBQEQKRKW3mI3uykkuODm1l794fDjh65TkZd+zEQLr6WRSxk+XeJBBZf8L9y+q2b4brsCdFR+Pqv/X7yj2/55IU9O8XOCPU3WuZQrDxU5xQaWtjRItxZcSc6ZSH+4jfpt+fBD4BIzjmP3bhv0+39Y3CbW4sihqqr/AgAA//9ZF4yTiwwAAA", "objectset.rio.cattle.io/id":"", "objectset.rio.cattle.io/owner-gvk":"k3s.cattle.io/v1, Kind=Addon", "objectset.rio.cattle.io/owner-name":"zfs-operator", "objectset.rio.cattle.io/owner-namespace":"kube-system"}, OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry{v1.ManagedFieldsEntry{Manager:"k3s", Operation:"Update", APIVersion:"apps/v1", Time:(*v1.Time)(0xc00be0ba80), FieldsType:"FieldsV1", FieldsV1:(*v1.FieldsV1)(0xc00be0baa0)}}}, Spec:v1.DaemonSetSpec{Selector:(*v1.LabelSelector)(0xc00be0bac0), Template:v1.PodTemplateSpec{ObjectMeta:v1.ObjectMeta{Name:"", GenerateName:"", Namespace:"", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{"app":"openebs-zfs-node", "openebs.io/component-name":"openebs-zfs-node", "openebs.io/version":"ci", "role":"openebs-zfs"}, Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.PodSpec{Volumes:[]v1.Volume{v1.Volume{Name:"device-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc00be0bae0), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"encr-keys", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc00be0bb00), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"chroot-zfs", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(nil), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(0xc009f064c0), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"host-root", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc00be0bb20), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"registration-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc00be0bb40), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"plugin-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc00be0bb60), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"pods-mount-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc00be0bb80), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}}, InitContainers:[]v1.Container(nil), Containers:[]v1.Container{v1.Container{Name:"csi-node-driver-registrar", Image:"k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0", Command:[]string(nil), Args:[]string{"--v=5", "--csi-address=$(ADDRESS)", "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"}, WorkingDir:"", Ports:[]v1.ContainerPort(nil), EnvFrom:[]v1.EnvFromSource(nil), Env:[]v1.EnvVar{v1.EnvVar{Name:"ADDRESS", Value:"/plugin/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"DRIVER_REG_SOCK_PATH", Value:"/var/lib/kubelet/plugins/zfs-localpv/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"KUBE_NODE_NAME", Value:"", ValueFrom:(*v1.EnvVarSource)(0xc00be0bbc0)}, v1.EnvVar{Name:"NODE_DRIVER", Value:"openebs-zfs", ValueFrom:(*v1.EnvVarSource)(nil)}}, Resources:v1.ResourceRequirements{Limits:v1.ResourceList(nil), Requests:v1.ResourceList(nil)}, VolumeMounts:[]v1.VolumeMount{v1.VolumeMount{Name:"plugin-dir", ReadOnly:false, MountPath:"/plugin", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"registration-dir", ReadOnly:false, MountPath:"/registration", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}}, VolumeDevices:[]v1.VolumeDevice(nil), LivenessProbe:(*v1.Probe)(nil), ReadinessProbe:(*v1.Probe)(nil), StartupProbe:(*v1.Probe)(nil), Lifecycle:(*v1.Lifecycle)(0xc012a88980), TerminationMessagePath:"/dev/termination-log", TerminationMessagePolicy:"File", ImagePullPolicy:"IfNotPresent", SecurityContext:(*v1.SecurityContext)(nil), Stdin:false, StdinOnce:false, TTY:false}, v1.Container{Name:"openebs-zfs-plugin", Image:"openebs/zfs-driver:1.9.3", Command:[]string(nil), Args:[]string{"--nodename=$(OPENEBS_NODE_NAME)", "--endpoint=$(OPENEBS_CSI_ENDPOINT)", "--plugin=$(OPENEBS_NODE_DRIVER)"}, WorkingDir:"", Ports:[]v1.ContainerPort(nil), EnvFrom:[]v1.EnvFromSource(nil), Env:[]v1.EnvVar{v1.EnvVar{Name:"OPENEBS_NODE_NAME", Value:"", ValueFrom:(*v1.EnvVarSource)(0xc00be0bca0)}, v1.EnvVar{Name:"OPENEBS_CSI_ENDPOINT", Value:"unix:///plugin/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"OPENEBS_NODE_DRIVER", Value:"agent", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"OPENEBS_NAMESPACE", Value:"openebs", ValueFrom:(*v1.EnvVarSource)(nil)}}, Resources:v1.ResourceRequirements{Limits:v1.ResourceList(nil), Requests:v1.ResourceList(nil)}, VolumeMounts:[]v1.VolumeMount{v1.VolumeMount{Name:"plugin-dir", ReadOnly:false, MountPath:"/plugin", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"device-dir", ReadOnly:false, MountPath:"/dev", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"encr-keys", ReadOnly:false, MountPath:"/home/keys", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"chroot-zfs", ReadOnly:false, MountPath:"/sbin/zfs", SubPath:"zfs", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"host-root", ReadOnly:true, MountPath:"/host", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(0xc012a889b0), SubPathExpr:""}, v1.VolumeMount{Name:"pods-mount-dir", ReadOnly:false, MountPath:"/var/lib/kubelet/", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(0xc012a889c0), SubPathExpr:""}}, VolumeDevices:[]v1.VolumeDevice(nil), LivenessProbe:(*v1.Probe)(nil), ReadinessProbe:(*v1.Probe)(nil), StartupProbe:(*v1.Probe)(nil), Lifecycle:(*v1.Lifecycle)(nil), TerminationMessagePath:"/dev/termination-log", TerminationMessagePolicy:"File", ImagePullPolicy:"IfNotPresent", SecurityContext:(*v1.SecurityContext)(0xc00b9eb920), Stdin:false, StdinOnce:false, TTY:false}}, EphemeralContainers:[]v1.EphemeralContainer(nil), RestartPolicy:"Always", TerminationGracePeriodSeconds:(*int64)(0xc00991dc30), ActiveDeadlineSeconds:(*int64)(nil), DNSPolicy:"ClusterFirst", NodeSelector:map[string]string(nil), ServiceAccountName:"openebs-zfs-node-sa", DeprecatedServiceAccount:"openebs-zfs-node-sa", AutomountServiceAccountToken:(*bool)(nil), NodeName:"", HostNetwork:true, HostPID:false, HostIPC:false, ShareProcessNamespace:(*bool)(nil), SecurityContext:(*v1.PodSecurityContext)(0xc008f018f0), ImagePullSecrets:[]v1.LocalObjectReference(nil), Hostname:"", Subdomain:"", Affinity:(*v1.Affinity)(nil), SchedulerName:"default-scheduler", Tolerations:[]v1.Toleration(nil), HostAliases:[]v1.HostAlias(nil), PriorityClassName:"system-node-critical", Priority:(*int32)(nil), DNSConfig:(*v1.PodDNSConfig)(nil), ReadinessGates:[]v1.PodReadinessGate(nil), RuntimeClassName:(*string)(nil), EnableServiceLinks:(*bool)(nil), PreemptionPolicy:(*v1.PreemptionPolicy)(nil), Overhead:v1.ResourceList(nil), TopologySpreadConstraints:[]v1.TopologySpreadConstraint(nil), SetHostnameAsFQDN:(*bool)(nil)}}, UpdateStrategy:v1.DaemonSetUpdateStrategy{Type:"RollingUpdate", RollingUpdate:(*v1.RollingUpdateDaemonSet)(0xc012a889d0)}, MinReadySeconds:0, RevisionHistoryLimit:(*int32)(0xc00991dc3c)}, Status:v1.DaemonSetStatus{CurrentNumberScheduled:0, NumberMisscheduled:0, DesiredNumberScheduled:0, NumberReady:0, ObservedGeneration:0, UpdatedNumberScheduled:0, NumberAvailable:0, NumberUnavailable:0, CollisionCount:(*int32)(nil), Conditions:[]v1.DaemonSetCondition(nil)}}: Operation cannot be fulfilled on daemonsets.apps "openebs-zfs-node": the object has been modified; please apply your changes to the latest version and try again
Mar 13 19:30:21 newnas.local k3s[142022]: E0313 19:30:21.532407  142022 daemon_controller.go:320] kube-system/openebs-zfs-node failed with : error storing status for daemon set &v1.DaemonSet{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"openebs-zfs-node", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"7c7ff1b5-22eb-4a26-9451-c6b198492519", ResourceVersion:"444", Generation:1, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:63782811007, loc:(*time.Location)(0x8e40740)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{"objectset.rio.cattle.io/hash":"4b1efd47d7f37713706abe9f8b0de3f0adf02e2c", "openebs.io/component-name":"openebs-zfs-node", "openebs.io/version":"ci"}, Annotations:map[string]string{"deprecated.daemonset.template.generation":"1", "objectset.rio.cattle.io/applied":"H4sIAAAAAAAA/6xWUY/aOBD+K5XVk1opIVDabhtpHyjkrqveAoK2L9UKOfYk+HDsyHayy63y3092AgQI2z1p3xJ75puZbz6P/Yhwzn6C0kwKFCKc5zooB8hDGyYoCtEEQybFEgzyUAYGU2wwCh8RFkIabJgU2v7K+B8gRoPpKSZ7BBvDocdkwCwG8i7uy3sByk/LDQrRZqhbO+XAe/WNCXo9olSK30IInAEK0b+J9mUOChupnuWjc0ys46aIwddbbSBDlYc4joE/Wdka6zUK0ft4AAl9f0WvkuHV1WB41f+IY/icfIr7FIZJH9Ok/w7eEZtMDgJibZ2JzHIpQJhd3s2eb/MXksKxebnvD2E2u8tOT5SkcyC2IA0ciKUnfEQZNmT9975WnOddsFXlIQNZzrEB59XSAf+d84vV7SEl+YmPS21XGJHCYCZAaRT+ekRYpfYD+X55/QF5yPeJZj6mVIHW16/fjCaTRbRcvnVblisOxleQMm2UE7afY7O+fv1msrj5GS1Wi+iv1XI2/raaj75/fYvuPASidJGaWhpA5KES88KuBDkvUiYCollPS7JBlbe37kJtu5ZYBZzFQZNZA6UDyxSXBPO87MT99uNLtJrOJtFqOrqNdoh/KplZjhIGnC4g2X/PsbEythz2bAOmFqWqWoAOq862ld9RE+48xDKcOtF90r2UKNs9zVJfG6lwCjZT11+fKlaC2vOswvJdb9DrowZhXnA+l5yRLQrRTTKVZq5Ag7Djh7MEyJZwp8FcwdLI3H7Cw67/WYbt0PqFgpiJQK9ta+3JU9krXyWvgnZ320Re3rGJ1hTfWVJ2nFwsxzIkeZHBrSyEqYWY2c+G56aNaI9U//uUKdfEI9t2UgePI4k6vzvreZC7TczaXr9+M5tH0+jL8qCHWu0gaC6ZMC2L8fJmFU0n89nN9HttVGd2ClILoUv+Z7FeQHtd2bVEWAj2EAbBU8esI/kWAE6ttLrMR7fRcj4aR+eSP5J7s+YkU2shHPQ+94bP0XPHCNyrQwMpFDPbsRQGHoybr5zL+7liJeOQQqQJ5rUyQqMK8FC+26L1SvWSUqRQHgwplIxAt+FaZhBsYKsP5iCI8t3SmbW259SOkMPJWispjV+v6SJuLN2YOY+l3bPErSmZ47QhBH2V2nyX492FcIC3Lr6NYKcCYDoTfLuj6wT9dAB3R/rCKFNA7B/mLS4l1b4z3x3RO88Fn4K5l2rTapp0feZY62ntW1/Z9XwhihlGHLIGZWkfEWJhO+5OX+P9+KnbbQPW5Tyi/KiTZpu7S8jlLtW29ao46W4XRrvJp0gzNVZgHwrVBQEQKRKW3mI3uykkuODm1l794fDjh65TkZd+zEQLr6WRSxk+XeJBBZf8L9y+q2b4brsCdFR+Pqv/X7yj2/55IU9O8XOCPU3WuZQrDxU5xQaWtjRItxZcSc6ZSH+4jfpt+fBD4BIzjmP3bhv0+39Y3CbW4sihqqr/AgAA//9ZF4yTiwwAAA", "objectset.rio.cattle.io/id":"", "objectset.rio.cattle.io/owner-gvk":"k3s.cattle.io/v1, Kind=Addon", "objectset.rio.cattle.io/owner-name":"zfs-operator", "objectset.rio.cattle.io/owner-namespace":"kube-system"}, OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry{v1.ManagedFieldsEntry{Manager:"k3s", Operation:"Update", APIVersion:"apps/v1", Time:(*v1.Time)(0xc001775340), FieldsType:"FieldsV1", FieldsV1:(*v1.FieldsV1)(0xc001775360)}}}, Spec:v1.DaemonSetSpec{Selector:(*v1.LabelSelector)(0xc001775380), Template:v1.PodTemplateSpec{ObjectMeta:v1.ObjectMeta{Name:"", GenerateName:"", Namespace:"", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{"app":"openebs-zfs-node", "openebs.io/component-name":"openebs-zfs-node", "openebs.io/version":"ci", "role":"openebs-zfs"}, Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.PodSpec{Volumes:[]v1.Volume{v1.Volume{Name:"device-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc001775400), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"encr-keys", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc001775420), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"chroot-zfs", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(nil), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(0xc00dfcf580), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"host-root", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc001775440), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"registration-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc001775460), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"plugin-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc0017754a0), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}, v1.Volume{Name:"pods-mount-dir", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0xc0017754c0), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil)}}}, InitContainers:[]v1.Container(nil), Containers:[]v1.Container{v1.Container{Name:"csi-node-driver-registrar", Image:"k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0", Command:[]string(nil), Args:[]string{"--v=5", "--csi-address=$(ADDRESS)", "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"}, WorkingDir:"", Ports:[]v1.ContainerPort(nil), EnvFrom:[]v1.EnvFromSource(nil), Env:[]v1.EnvVar{v1.EnvVar{Name:"ADDRESS", Value:"/plugin/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"DRIVER_REG_SOCK_PATH", Value:"/var/lib/kubelet/plugins/zfs-localpv/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"KUBE_NODE_NAME", Value:"", ValueFrom:(*v1.EnvVarSource)(0xc001775540)}, v1.EnvVar{Name:"NODE_DRIVER", Value:"openebs-zfs", ValueFrom:(*v1.EnvVarSource)(nil)}}, Resources:v1.ResourceRequirements{Limits:v1.ResourceList(nil), Requests:v1.ResourceList(nil)}, VolumeMounts:[]v1.VolumeMount{v1.VolumeMount{Name:"plugin-dir", ReadOnly:false, MountPath:"/plugin", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"registration-dir", ReadOnly:false, MountPath:"/registration", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}}, VolumeDevices:[]v1.VolumeDevice(nil), LivenessProbe:(*v1.Probe)(nil), ReadinessProbe:(*v1.Probe)(nil), StartupProbe:(*v1.Probe)(nil), Lifecycle:(*v1.Lifecycle)(0xc01213a920), TerminationMessagePath:"/dev/termination-log", TerminationMessagePolicy:"File", ImagePullPolicy:"IfNotPresent", SecurityContext:(*v1.SecurityContext)(nil), Stdin:false, StdinOnce:false, TTY:false}, v1.Container{Name:"openebs-zfs-plugin", Image:"openebs/zfs-driver:1.9.3", Command:[]string(nil), Args:[]string{"--nodename=$(OPENEBS_NODE_NAME)", "--endpoint=$(OPENEBS_CSI_ENDPOINT)", "--plugin=$(OPENEBS_NODE_DRIVER)"}, WorkingDir:"", Ports:[]v1.ContainerPort(nil), EnvFrom:[]v1.EnvFromSource(nil), Env:[]v1.EnvVar{v1.EnvVar{Name:"OPENEBS_NODE_NAME", Value:"", ValueFrom:(*v1.EnvVarSource)(0xc001775840)}, v1.EnvVar{Name:"OPENEBS_CSI_ENDPOINT", Value:"unix:///plugin/csi.sock", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"OPENEBS_NODE_DRIVER", Value:"agent", ValueFrom:(*v1.EnvVarSource)(nil)}, v1.EnvVar{Name:"OPENEBS_NAMESPACE", Value:"openebs", ValueFrom:(*v1.EnvVarSource)(nil)}}, Resources:v1.ResourceRequirements{Limits:v1.ResourceList(nil), Requests:v1.ResourceList(nil)}, VolumeMounts:[]v1.VolumeMount{v1.VolumeMount{Name:"plugin-dir", ReadOnly:false, MountPath:"/plugin", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"device-dir", ReadOnly:false, MountPath:"/dev", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"encr-keys", ReadOnly:false, MountPath:"/home/keys", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"chroot-zfs", ReadOnly:false, MountPath:"/sbin/zfs", SubPath:"zfs", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:""}, v1.VolumeMount{Name:"host-root", ReadOnly:true, MountPath:"/host", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(0xc01213a950), SubPathExpr:""}, v1.VolumeMount{Name:"pods-mount-dir", ReadOnly:false, MountPath:"/var/lib/kubelet/", SubPath:"", MountPropagation:(*v1.MountPropagationMode)(0xc01213a960), SubPathExpr:""}}, VolumeDevices:[]v1.VolumeDevice(nil), LivenessProbe:(*v1.Probe)(nil), ReadinessProbe:(*v1.Probe)(nil), StartupProbe:(*v1.Probe)(nil), Lifecycle:(*v1.Lifecycle)(nil), TerminationMessagePath:"/dev/termination-log", TerminationMessagePolicy:"File", ImagePullPolicy:"IfNotPresent", SecurityContext:(*v1.SecurityContext)(0xc011a8a960), Stdin:false, StdinOnce:false, TTY:false}}, EphemeralContainers:[]v1.EphemeralContainer(nil), RestartPolicy:"Always", TerminationGracePeriodSeconds:(*int64)(0xc00b6bdf00), ActiveDeadlineSeconds:(*int64)(nil), DNSPolicy:"ClusterFirst", NodeSelector:map[string]string(nil), ServiceAccountName:"openebs-zfs-node-sa", DeprecatedServiceAccount:"openebs-zfs-node-sa", AutomountServiceAccountToken:(*bool)(nil), NodeName:"", HostNetwork:true, HostPID:false, HostIPC:false, ShareProcessNamespace:(*bool)(nil), SecurityContext:(*v1.PodSecurityContext)(0xc009e0ecb0), ImagePullSecrets:[]v1.LocalObjectReference(nil), Hostname:"", Subdomain:"", Affinity:(*v1.Affinity)(nil), SchedulerName:"default-scheduler", Tolerations:[]v1.Toleration(nil), HostAliases:[]v1.HostAlias(nil), PriorityClassName:"system-node-critical", Priority:(*int32)(nil), DNSConfig:(*v1.PodDNSConfig)(nil), ReadinessGates:[]v1.PodReadinessGate(nil), RuntimeClassName:(*string)(nil), EnableServiceLinks:(*bool)(nil), PreemptionPolicy:(*v1.PreemptionPolicy)(nil), Overhead:v1.ResourceList(nil), TopologySpreadConstraints:[]v1.TopologySpreadConstraint(nil), SetHostnameAsFQDN:(*bool)(nil)}}, UpdateStrategy:v1.DaemonSetUpdateStrategy{Type:"RollingUpdate", RollingUpdate:(*v1.RollingUpdateDaemonSet)(0xc01213a970)}, MinReadySeconds:0, RevisionHistoryLimit:(*int32)(0xc00b6bdf0c)}, Status:v1.DaemonSetStatus{CurrentNumberScheduled:0, NumberMisscheduled:0, DesiredNumberScheduled:0, NumberReady:0, ObservedGeneration:1, UpdatedNumberScheduled:0, NumberAvailable:0, NumberUnavailable:0, CollisionCount:(*int32)(nil), Conditions:[]v1.DaemonSetCondition(nil)}}: Operation cannot be fulfilled on daemonsets.apps "openebs-zfs-node": the object has been modified; please apply your changes to the latest version and try again
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.591842  142022 event.go:291] "Event occurred" object="kube-system/coredns-7448499f4d" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: coredns-7448499f4d-qzbr4"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.595785  142022 topology_manager.go:187] "Topology Admit Handler"
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.673058  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/f5fad81a-b088-466e-9c3d-4e116edeb641-chroot-zfs\") pod \"openebs-zfs-node-l5t7c\" (UID: \"f5fad81a-b088-466e-9c3d-4e116edeb641\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.673102  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/f5fad81a-b088-466e-9c3d-4e116edeb641-host-root\") pod \"openebs-zfs-node-l5t7c\" (UID: \"f5fad81a-b088-466e-9c3d-4e116edeb641\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.673146  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/f5fad81a-b088-466e-9c3d-4e116edeb641-registration-dir\") pod \"openebs-zfs-node-l5t7c\" (UID: \"f5fad81a-b088-466e-9c3d-4e116edeb641\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.673179  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/f5fad81a-b088-466e-9c3d-4e116edeb641-plugin-dir\") pod \"openebs-zfs-node-l5t7c\" (UID: \"f5fad81a-b088-466e-9c3d-4e116edeb641\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.673211  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/f5fad81a-b088-466e-9c3d-4e116edeb641-pods-mount-dir\") pod \"openebs-zfs-node-l5t7c\" (UID: \"f5fad81a-b088-466e-9c3d-4e116edeb641\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.673248  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-snzf2\" (UniqueName: \"kubernetes.io/projected/f5fad81a-b088-466e-9c3d-4e116edeb641-kube-api-access-snzf2\") pod \"openebs-zfs-node-l5t7c\" (UID: \"f5fad81a-b088-466e-9c3d-4e116edeb641\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.673280  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/f5fad81a-b088-466e-9c3d-4e116edeb641-device-dir\") pod \"openebs-zfs-node-l5t7c\" (UID: \"f5fad81a-b088-466e-9c3d-4e116edeb641\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.673318  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/f5fad81a-b088-466e-9c3d-4e116edeb641-encr-keys\") pod \"openebs-zfs-node-l5t7c\" (UID: \"f5fad81a-b088-466e-9c3d-4e116edeb641\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.741873  142022 shared_informer.go:247] Caches are synced for garbage collector
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.741889  142022 garbagecollector.go:151] Garbage collector: all resource monitors have synced. Proceeding to collect garbage
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.773739  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/e150fb2e-ded4-43fa-a37f-75b61c350a30-config-volume\") pod \"coredns-7448499f4d-qzbr4\" (UID: \"e150fb2e-ded4-43fa-a37f-75b61c350a30\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.773781  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-9qzsv\" (UniqueName: \"kubernetes.io/projected/e150fb2e-ded4-43fa-a37f-75b61c350a30-kube-api-access-9qzsv\") pod \"coredns-7448499f4d-qzbr4\" (UID: \"e150fb2e-ded4-43fa-a37f-75b61c350a30\") "
Mar 13 19:30:21 newnas.local k3s[142022]: I0313 19:30:21.788806  142022 shared_informer.go:247] Caches are synced for garbage collector
Mar 13 19:30:22 newnas.local k3s[142022]: I0313 19:30:22.476207  142022 topology_manager.go:187] "Topology Admit Handler"
Mar 13 19:30:22 newnas.local k3s[142022]: I0313 19:30:22.675101  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/4e5509b0-fcae-4486-b3ae-0278682ecd79-socket-dir\") pod \"openebs-zfs-controller-0\" (UID: \"4e5509b0-fcae-4486-b3ae-0278682ecd79\") "
Mar 13 19:30:22 newnas.local k3s[142022]: I0313 19:30:22.675171  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-mc6r8\" (UniqueName: \"kubernetes.io/projected/4e5509b0-fcae-4486-b3ae-0278682ecd79-kube-api-access-mc6r8\") pod \"openebs-zfs-controller-0\" (UID: \"4e5509b0-fcae-4486-b3ae-0278682ecd79\") "
Mar 13 19:30:26 newnas.local k3s[142022]: I0313 19:30:26.056272  142022 node_lifecycle_controller.go:1191] Controller detected that some Nodes are Ready. Exiting master disruption mode.
Mar 13 19:30:38 newnas.local k3s[142022]: E0313 19:30:38.023053  142022 kuberuntime_manager.go:1003] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 93dcfe8865d1e497b2a6aca044f48925bfda6387e487a27f14058705a10ffe76" podSandboxID="93dcfe8865d1e497b2a6aca044f48925bfda6387e487a27f14058705a10ffe76" pod="kube-system/openebs-zfs-node-l5t7c"
Mar 13 19:30:38 newnas.local k3s[142022]: E0313 19:30:38.024444  142022 kuberuntime_manager.go:1003] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 3aa01d013da82947f09a1c9d71c31a275e082b1571daaeefbf14719c41c3c16e" podSandboxID="3aa01d013da82947f09a1c9d71c31a275e082b1571daaeefbf14719c41c3c16e" pod="kube-system/coredns-7448499f4d-qzbr4"
Mar 13 19:30:39 newnas.local k3s[142022]: E0313 19:30:39.030071  142022 kuberuntime_manager.go:1003] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 3aa01d013da82947f09a1c9d71c31a275e082b1571daaeefbf14719c41c3c16e" podSandboxID="3aa01d013da82947f09a1c9d71c31a275e082b1571daaeefbf14719c41c3c16e" pod="kube-system/coredns-7448499f4d-qzbr4"
Mar 13 19:30:39 newnas.local k3s[142022]: I0313 19:30:39.556220  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="93dcfe8865d1e497b2a6aca044f48925bfda6387e487a27f14058705a10ffe76"
Mar 13 19:30:39 newnas.local k3s[142022]: E0313 19:30:39.557688  142022 kuberuntime_manager.go:1003] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 2c6a0b546948383cc9525f3a57daf81dd281d5afa17c8c2f314bb81e6008b06f" podSandboxID="2c6a0b546948383cc9525f3a57daf81dd281d5afa17c8c2f314bb81e6008b06f" pod="kube-system/openebs-zfs-controller-0"
Mar 13 19:30:40 newnas.local k3s[142022]: I0313 19:30:40.959388  148264 event.go:278] Event(v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"coredns-7448499f4d-qzbr4", UID:"e150fb2e-ded4-43fa-a37f-75b61c350a30", APIVersion:"v1", ResourceVersion:"464", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.20/16] from ix-net
Mar 13 19:30:43 newnas.local k3s[142022]: I0313 19:30:43.352854  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="2c6a0b546948383cc9525f3a57daf81dd281d5afa17c8c2f314bb81e6008b06f"
Mar 13 19:30:43 newnas.local k3s[142022]: I0313 19:30:43.722492  148680 event.go:278] Event(v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"openebs-zfs-controller-0", UID:"4e5509b0-fcae-4486-b3ae-0278682ecd79", APIVersion:"v1", ResourceVersion:"473", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.21/16] from ix-net
Mar 13 19:31:00 newnas.local k3s[142022]: I0313 19:31:00.718049  142022 scope.go:111] "RemoveContainer" containerID="cba659f34c960e3fbe7c5fe347364c0a246d2d2c437cceb394ce8c2153d18643"
Mar 13 19:31:02 newnas.local k3s[142022]: I0313 19:31:02.471668  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:03 newnas.local k3s[142022]: I0313 19:31:03.374162  142022 scope.go:111] "RemoveContainer" containerID="ed105bd53ae2fe847fc392ac5ce5de08cfb17cd4cff2c6d85023fea6dcd8566e"
Mar 13 19:31:03 newnas.local k3s[142022]: I0313 19:31:03.487372  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:03 newnas.local k3s[142022]: I0313 19:31:03.494038  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="cba659f34c960e3fbe7c5fe347364c0a246d2d2c437cceb394ce8c2153d18643"
Mar 13 19:31:05 newnas.local k3s[142022]: I0313 19:31:05.501177  142022 scope.go:111] "RemoveContainer" containerID="661797458f4f57e77ac337052f05c7a2b645744124839c17eeba5ac1dab6b0c5"
Mar 13 19:31:05 newnas.local k3s[142022]: E0313 19:31:05.506627  142022 remote_runtime.go:334] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 559f44e3d962eeeb348c5d8f1a8cb92e1a40ca5a88a6b4fd250a1f50d8746cd7" containerID="559f44e3d962eeeb348c5d8f1a8cb92e1a40ca5a88a6b4fd250a1f50d8746cd7"
Mar 13 19:31:05 newnas.local k3s[142022]: E0313 19:31:05.506693  142022 kuberuntime_manager.go:1018] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 559f44e3d962eeeb348c5d8f1a8cb92e1a40ca5a88a6b4fd250a1f50d8746cd7" pod="kube-system/openebs-zfs-node-l5t7c"
Mar 13 19:31:05 newnas.local k3s[142022]: I0313 19:31:05.508652  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:05 newnas.local k3s[142022]: I0313 19:31:05.590691  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="ed105bd53ae2fe847fc392ac5ce5de08cfb17cd4cff2c6d85023fea6dcd8566e"
Mar 13 19:31:05 newnas.local k3s[142022]: E0313 19:31:05.595480  142022 remote_runtime.go:334] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: cceb279ff6c8faf57edfdeb245085aa9ea1461b3a703a54b967da1589ae3a100" containerID="cceb279ff6c8faf57edfdeb245085aa9ea1461b3a703a54b967da1589ae3a100"
Mar 13 19:31:05 newnas.local k3s[142022]: E0313 19:31:05.595527  142022 kuberuntime_manager.go:1018] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: cceb279ff6c8faf57edfdeb245085aa9ea1461b3a703a54b967da1589ae3a100" pod="kube-system/coredns-7448499f4d-qzbr4"
Mar 13 19:31:06 newnas.local k3s[142022]: E0313 19:31:06.605588  142022 remote_runtime.go:334] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 37f085bbc2a71b46fb3bcf501889c5f53702b3eded449249698cb44fbf3908f1" containerID="37f085bbc2a71b46fb3bcf501889c5f53702b3eded449249698cb44fbf3908f1"
Mar 13 19:31:06 newnas.local k3s[142022]: E0313 19:31:06.605657  142022 kuberuntime_manager.go:1018] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 37f085bbc2a71b46fb3bcf501889c5f53702b3eded449249698cb44fbf3908f1" pod="kube-system/openebs-zfs-controller-0"
Mar 13 19:31:06 newnas.local k3s[142022]: I0313 19:31:06.607737  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:13 newnas.local k3s[142022]: I0313 19:31:13.190040  142022 scope.go:111] "RemoveContainer" containerID="b80e68dad001ae10a1e65727463d7a031f9cdd4b91f456f05b2d59e90601d8e2"
Mar 13 19:31:14 newnas.local k3s[142022]: I0313 19:31:14.480825  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:14 newnas.local k3s[142022]: I0313 19:31:14.487182  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="661797458f4f57e77ac337052f05c7a2b645744124839c17eeba5ac1dab6b0c5"
Mar 13 19:31:30 newnas.local k3s[142022]: I0313 19:31:30.223057  142022 scope.go:111] "RemoveContainer" containerID="ac4b1631aa0987a819ab3ece976026508d88643ab1b638a778afc14ebd18cd9e"
Mar 13 19:31:30 newnas.local k3s[142022]: I0313 19:31:30.578914  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="b80e68dad001ae10a1e65727463d7a031f9cdd4b91f456f05b2d59e90601d8e2"
Mar 13 19:31:30 newnas.local k3s[142022]: I0313 19:31:30.580863  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:33 newnas.local k3s[142022]: I0313 19:31:33.413084  142022 scope.go:111] "RemoveContainer" containerID="f1c83259b8336fc0bd9e02dc88013cac555d43ff92a8a19e8b61ccc06042f3e5"
Mar 13 19:31:33 newnas.local k3s[142022]: I0313 19:31:33.600482  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:33 newnas.local k3s[142022]: I0313 19:31:33.602685  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="ac4b1631aa0987a819ab3ece976026508d88643ab1b638a778afc14ebd18cd9e"
Mar 13 19:31:35 newnas.local k3s[142022]: I0313 19:31:35.446795  142022 scope.go:111] "RemoveContainer" containerID="3c5acf974a3c8c2c01e924ae63454d54b47cd615de450c6c37af3bd6b26e2e72"
Mar 13 19:31:35 newnas.local k3s[142022]: I0313 19:31:35.616378  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="f1c83259b8336fc0bd9e02dc88013cac555d43ff92a8a19e8b61ccc06042f3e5"
Mar 13 19:31:35 newnas.local k3s[142022]: I0313 19:31:35.618177  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:37 newnas.local k3s[142022]: I0313 19:31:37.491955  142022 cni.go:333] "CNI failed to retrieve network namespace path" err="cannot find network namespace for the terminated container \"44fe819055da9eefe1b7b40e1fc66bb21e6a41fb4810d9e2a467171c2e628085\""
Mar 13 19:31:37 newnas.local k3s[142022]: 2022-03-13T19:31:37-04:00 [error] Multus: failed to get the cached delegates file: open /var/lib/cni/multus/44fe819055da9eefe1b7b40e1fc66bb21e6a41fb4810d9e2a467171c2e628085: no such file or directory, cannot properly delete
Mar 13 19:31:37 newnas.local k3s[142022]: I0313 19:31:37.629942  142022 docker_sandbox.go:401] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:31:37 newnas.local k3s[142022]: I0313 19:31:37.630544  142022 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="3c5acf974a3c8c2c01e924ae63454d54b47cd615de450c6c37af3bd6b26e2e72"
Mar 13 19:31:57 newnas.local k3s[142022]: E0313 19:31:57.711015  142022 remote_runtime.go:334] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 53f501e667a4612be3ca6c7ce2ea42fd04d5d1782361f9153a54569c70917743" containerID="53f501e667a4612be3ca6c7ce2ea42fd04d5d1782361f9153a54569c70917743"
Mar 13 19:31:57 newnas.local k3s[142022]: E0313 19:31:57.711091  142022 kuberuntime_manager.go:1018] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 53f501e667a4612be3ca6c7ce2ea42fd04d5d1782361f9153a54569c70917743" pod="kube-system/openebs-zfs-node-l5t7c"
Mar 13 19:31:57 newnas.local k3s[142022]: E0313 19:31:57.715644  142022 remote_runtime.go:334] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 3b13637b34206f65ab2ac394dab19dcafca6bfcc7699131db3ac7834eb0412d7" containerID="3b13637b34206f65ab2ac394dab19dcafca6bfcc7699131db3ac7834eb0412d7"
Mar 13 19:31:57 newnas.local k3s[142022]: E0313 19:31:57.715683  142022 kuberuntime_manager.go:1018] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 3b13637b34206f65ab2ac394dab19dcafca6bfcc7699131db3ac7834eb0412d7" pod="kube-system/openebs-zfs-controller-0"
Mar 13 19:32:01 newnas.local k3s[142022]: I0313 19:32:01.869336  142022 controller.go:611] quota admission added evaluator for: zfsnodes.zfs.openebs.io
Mar 13 19:32:02 newnas.local k3s[142022]: I0313 19:32:02.298148  142022 cni.go:333] "CNI failed to retrieve network namespace path" err="cannot find network namespace for the terminated container \"617261a2bd6e6253a9291b3927c7eff6f37711a0bcc7e2049a7ff49466df3f9c\""
Mar 13 19:32:02 newnas.local k3s[142022]: I0313 19:32:02.948686  142022 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: zfs.csi.openebs.io endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock versions: 1.0.0
Mar 13 19:32:02 newnas.local k3s[142022]: I0313 19:32:02.948896  142022 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: zfs.csi.openebs.io at endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock
Mar 13 19:32:02 newnas.local k3s[142022]: E0313 19:32:02.972570  142022 nodeinfomanager.go:566] Invalid attach limit value 0 cannot be added to CSINode object for "zfs.csi.openebs.io"
Mar 13 19:32:07 newnas.local k3s[142022]: E0313 19:32:07.218121  142022 remote_runtime.go:334] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 2396cd1c27bf502b8515b3b695808587b2b6836065e8acb2070f7e241c81e61b" containerID="2396cd1c27bf502b8515b3b695808587b2b6836065e8acb2070f7e241c81e61b"
Mar 13 19:32:07 newnas.local k3s[142022]: E0313 19:32:07.218185  142022 kuberuntime_manager.go:1018] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 2396cd1c27bf502b8515b3b695808587b2b6836065e8acb2070f7e241c81e61b" pod="kube-system/openebs-zfs-controller-0"
Mar 13 19:32:25 newnas.local k3s[142022]: E0313 19:32:25.296961  142022 remote_runtime.go:334] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 45e54ca4ff9ac9bc23b06ed238f05e0ca425ff65aa49ce57a334d73d8a4c43cb" containerID="45e54ca4ff9ac9bc23b06ed238f05e0ca425ff65aa49ce57a334d73d8a4c43cb"
Mar 13 19:32:25 newnas.local k3s[142022]: E0313 19:32:25.297015  142022 kuberuntime_manager.go:1018] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 45e54ca4ff9ac9bc23b06ed238f05e0ca425ff65aa49ce57a334d73d8a4c43cb" pod="kube-system/openebs-zfs-controller-0"
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.472394  142022 event.go:291] "Event occurred" object="ix-plex-test/plex-test" kind="Deployment" apiVersion="apps/v1" type="Normal" reason="ScalingReplicaSet" message="Scaled up replica set plex-test-8574fd5c48 to 1"
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.479667  142022 event.go:291] "Event occurred" object="ix-plex-test/plex-test-8574fd5c48" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: plex-test-8574fd5c48-96c29"
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.488041  142022 topology_manager.go:187] "Topology Admit Handler"
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.602853  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/390f57ac-8a2a-4385-a396-b4f703c5c108-shared\") pod \"plex-test-8574fd5c48-96c29\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.602901  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared-logs\" (UniqueName: \"kubernetes.io/empty-dir/390f57ac-8a2a-4385-a396-b4f703c5c108-shared-logs\") pod \"plex-test-8574fd5c48-96c29\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.602935  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-p6b2c\" (UniqueName: \"kubernetes.io/projected/390f57ac-8a2a-4385-a396-b4f703c5c108-kube-api-access-p6b2c\") pod \"plex-test-8574fd5c48-96c29\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.602971  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-config\") pod \"plex-test-8574fd5c48-96c29\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.603226  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plex-probe-check\" (UniqueName: \"kubernetes.io/configmap/390f57ac-8a2a-4385-a396-b4f703c5c108-plex-probe-check\") pod \"plex-test-8574fd5c48-96c29\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.603269  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"data\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-data\") pod \"plex-test-8574fd5c48-96c29\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 19:52:25 newnas.local k3s[142022]: I0313 19:52:25.603329  142022 reconciler.go:224] "operationExecutor.VerifyControllerAttachedVolume started for volume \"transcode\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-transcode\") pod \"plex-test-8574fd5c48-96c29\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 19:52:48 newnas.local k3s[142022]: E0313 19:52:48.068156  142022 kuberuntime_manager.go:1003] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 0117c7a69fb4abb724a92e47634563db3aa725eced91b5d38fbaf5dcbbca1907" podSandboxID="0117c7a69fb4abb724a92e47634563db3aa725eced91b5d38fbaf5dcbbca1907" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:52:48 newnas.local k3s[142022]: I0313 19:52:48.704096  204069 event.go:278] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ix-plex-test", Name:"plex-test-8574fd5c48-96c29", UID:"390f57ac-8a2a-4385-a396-b4f703c5c108", APIVersion:"v1", ResourceVersion:"1809", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.22/16] from ix-net
Mar 13 19:52:55 newnas.local k3s[142022]: E0313 19:52:55.035416  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:53:05 newnas.local k3s[142022]: E0313 19:53:05.061367  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:53:15 newnas.local k3s[142022]: E0313 19:53:15.084973  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:53:25 newnas.local k3s[142022]: E0313 19:53:25.111369  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:53:35 newnas.local k3s[142022]: E0313 19:53:35.136118  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:53:45 newnas.local k3s[142022]: E0313 19:53:45.163542  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:53:55 newnas.local k3s[142022]: E0313 19:53:55.187425  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:54:05 newnas.local k3s[142022]: E0313 19:54:05.209540  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:54:15 newnas.local k3s[142022]: E0313 19:54:15.237235  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:54:19 newnas.local k3s[142022]: E0313 19:54:19.290013  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="9c281a74223f: Download complete "
Mar 13 19:54:19 newnas.local k3s[142022]: E0313 19:54:19.290358  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 19:54:19 newnas.local k3s[142022]: E0313 19:54:19.290404  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 19:54:19 newnas.local k3s[142022]: E0313 19:54:19.291235  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 19:54:19 newnas.local k3s[142022]: E0313 19:54:19.291281  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 19:54:19 newnas.local k3s[142022]: E0313 19:54:19.459040  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 19:54:25 newnas.local k3s[142022]: E0313 19:54:25.263231  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:54:35 newnas.local k3s[142022]: E0313 19:54:35.286348  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:54:45 newnas.local k3s[142022]: E0313 19:54:45.312095  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:54:55 newnas.local k3s[142022]: E0313 19:54:55.338160  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:55:05 newnas.local k3s[142022]: E0313 19:55:05.360367  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:55:15 newnas.local k3s[142022]: E0313 19:55:15.386688  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:55:25 newnas.local k3s[142022]: E0313 19:55:25.410272  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:55:35 newnas.local k3s[142022]: E0313 19:55:35.436158  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:55:45 newnas.local k3s[142022]: E0313 19:55:45.464282  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:55:55 newnas.local k3s[142022]: E0313 19:55:55.489389  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:56:05 newnas.local k3s[142022]: E0313 19:56:05.515225  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:56:15 newnas.local k3s[142022]: E0313 19:56:15.537793  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:56:25 newnas.local k3s[142022]: E0313 19:56:25.562001  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:56:35 newnas.local k3s[142022]: E0313 19:56:35.584509  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:56:45 newnas.local k3s[142022]: E0313 19:56:45.610610  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:56:55 newnas.local k3s[142022]: E0313 19:56:55.636109  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:56:58 newnas.local k3s[142022]: E0313 19:56:58.546794  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="9c281a74223f: Download complete "
Mar 13 19:56:58 newnas.local k3s[142022]: E0313 19:56:58.547015  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 19:56:58 newnas.local k3s[142022]: E0313 19:56:58.547051  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 19:56:58 newnas.local k3s[142022]: E0313 19:56:58.547258  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 19:56:58 newnas.local k3s[142022]: E0313 19:56:58.547311  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 19:57:05 newnas.local k3s[142022]: E0313 19:57:05.658174  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:57:11 newnas.local k3s[142022]: E0313 19:57:11.848644  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 19:57:15 newnas.local k3s[142022]: E0313 19:57:15.684659  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:57:25 newnas.local k3s[142022]: E0313 19:57:25.706156  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:57:35 newnas.local k3s[142022]: E0313 19:57:35.733612  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:57:45 newnas.local k3s[142022]: E0313 19:57:45.764723  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:57:55 newnas.local k3s[142022]: E0313 19:57:55.790862  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:58:05 newnas.local k3s[142022]: E0313 19:58:05.817716  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:58:15 newnas.local k3s[142022]: E0313 19:58:15.842826  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:58:25 newnas.local k3s[142022]: E0313 19:58:25.867864  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:58:35 newnas.local k3s[142022]: E0313 19:58:35.895604  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:58:45 newnas.local k3s[142022]: E0313 19:58:45.922445  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:58:55 newnas.local k3s[142022]: E0313 19:58:55.944169  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:59:05 newnas.local k3s[142022]: E0313 19:59:05.971233  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:59:15 newnas.local k3s[142022]: E0313 19:59:15.996376  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:59:26 newnas.local k3s[142022]: E0313 19:59:26.017266  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:59:36 newnas.local k3s[142022]: E0313 19:59:36.043849  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:59:46 newnas.local k3s[142022]: E0313 19:59:46.070641  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:59:56 newnas.local k3s[142022]: E0313 19:59:56.094983  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 19:59:58 newnas.local k3s[142022]: E0313 19:59:58.395754  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="9c281a74223f: Download complete "
Mar 13 19:59:58 newnas.local k3s[142022]: E0313 19:59:58.396436  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 19:59:58 newnas.local k3s[142022]: E0313 19:59:58.396473  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 19:59:58 newnas.local k3s[142022]: E0313 19:59:58.396718  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 19:59:58 newnas.local k3s[142022]: E0313 19:59:58.396774  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:00:06 newnas.local k3s[142022]: E0313 20:00:06.116946  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:00:11 newnas.local k3s[142022]: E0313 20:00:11.848445  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:00:16 newnas.local k3s[142022]: E0313 20:00:16.141663  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:00:22 newnas.local k3s[142022]: E0313 20:00:22.848862  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:00:26 newnas.local k3s[142022]: E0313 20:00:26.169464  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:00:36 newnas.local k3s[142022]: E0313 20:00:36.194947  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:00:36 newnas.local k3s[142022]: E0313 20:00:36.848695  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:00:46 newnas.local k3s[142022]: E0313 20:00:46.220989  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:00:56 newnas.local k3s[142022]: E0313 20:00:56.245261  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:01:06 newnas.local k3s[142022]: E0313 20:01:06.271091  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:01:16 newnas.local k3s[142022]: E0313 20:01:16.306234  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:01:26 newnas.local k3s[142022]: E0313 20:01:26.330323  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:01:36 newnas.local k3s[142022]: E0313 20:01:36.355412  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:01:46 newnas.local k3s[142022]: E0313 20:01:46.380457  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:01:56 newnas.local k3s[142022]: E0313 20:01:56.404812  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:02:06 newnas.local k3s[142022]: E0313 20:02:06.428334  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:02:16 newnas.local k3s[142022]: E0313 20:02:16.454370  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:02:26 newnas.local k3s[142022]: E0313 20:02:26.477200  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:02:36 newnas.local k3s[142022]: E0313 20:02:36.500107  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:02:46 newnas.local k3s[142022]: E0313 20:02:46.524484  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:02:56 newnas.local k3s[142022]: E0313 20:02:56.551522  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:03:02 newnas.local k3s[142022]: E0313 20:03:02.699304  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="9c281a74223f: Download complete "
Mar 13 20:03:02 newnas.local k3s[142022]: E0313 20:03:02.699511  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:03:02 newnas.local k3s[142022]: E0313 20:03:02.699543  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:03:02 newnas.local k3s[142022]: E0313 20:03:02.699741  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 20:03:02 newnas.local k3s[142022]: E0313 20:03:02.699780  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:03:06 newnas.local k3s[142022]: E0313 20:03:06.576759  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:03:16 newnas.local k3s[142022]: E0313 20:03:16.600841  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:03:16 newnas.local k3s[142022]: E0313 20:03:16.848660  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:03:26 newnas.local k3s[142022]: E0313 20:03:26.626311  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:03:30 newnas.local k3s[142022]: E0313 20:03:30.848654  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:03:36 newnas.local k3s[142022]: E0313 20:03:36.649550  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:03:42 newnas.local k3s[142022]: E0313 20:03:42.848452  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:03:46 newnas.local k3s[142022]: E0313 20:03:46.673225  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:03:56 newnas.local k3s[142022]: E0313 20:03:56.699434  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:03:57 newnas.local k3s[142022]: E0313 20:03:57.848749  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:04:06 newnas.local k3s[142022]: E0313 20:04:06.723052  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:04:11 newnas.local k3s[142022]: E0313 20:04:11.848703  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:04:16 newnas.local k3s[142022]: E0313 20:04:16.744441  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:04:26 newnas.local k3s[142022]: E0313 20:04:26.770990  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:04:36 newnas.local k3s[142022]: E0313 20:04:36.794641  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:04:46 newnas.local k3s[142022]: E0313 20:04:46.820568  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:04:56 newnas.local k3s[142022]: E0313 20:04:56.845418  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:05:06 newnas.local k3s[142022]: E0313 20:05:06.868446  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:05:16 newnas.local k3s[142022]: E0313 20:05:16.891319  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:05:26 newnas.local k3s[142022]: E0313 20:05:26.931371  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:05:36 newnas.local k3s[142022]: E0313 20:05:36.956075  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:05:46 newnas.local k3s[142022]: E0313 20:05:46.978997  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:05:55 newnas.local k3s[142022]: E0313 20:05:55.321189  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="9c281a74223f: Download complete "
Mar 13 20:05:55 newnas.local k3s[142022]: E0313 20:05:55.321354  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:05:55 newnas.local k3s[142022]: E0313 20:05:55.321383  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:05:55 newnas.local k3s[142022]: E0313 20:05:55.321566  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 20:05:55 newnas.local k3s[142022]: E0313 20:05:55.321607  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:05:57 newnas.local k3s[142022]: E0313 20:05:57.003208  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:06:06 newnas.local k3s[142022]: E0313 20:06:06.857241  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:06:07 newnas.local k3s[142022]: E0313 20:06:07.035824  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:06:17 newnas.local k3s[142022]: E0313 20:06:17.062668  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:06:19 newnas.local k3s[142022]: E0313 20:06:19.848856  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:06:27 newnas.local k3s[142022]: E0313 20:06:27.085491  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:06:33 newnas.local k3s[142022]: E0313 20:06:33.848508  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:06:37 newnas.local k3s[142022]: E0313 20:06:37.113276  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:06:47 newnas.local k3s[142022]: E0313 20:06:47.138124  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:06:48 newnas.local k3s[142022]: E0313 20:06:48.848719  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:06:57 newnas.local k3s[142022]: E0313 20:06:57.162007  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:06:59 newnas.local k3s[142022]: E0313 20:06:59.848615  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:07:07 newnas.local k3s[142022]: E0313 20:07:07.187111  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:07:11 newnas.local k3s[142022]: E0313 20:07:11.848726  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:07:17 newnas.local k3s[142022]: E0313 20:07:17.216179  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:07:23 newnas.local k3s[142022]: E0313 20:07:23.848574  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:07:27 newnas.local k3s[142022]: E0313 20:07:27.238647  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:07:37 newnas.local k3s[142022]: E0313 20:07:37.263678  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:07:38 newnas.local k3s[142022]: E0313 20:07:38.848464  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:07:47 newnas.local k3s[142022]: E0313 20:07:47.290880  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:07:51 newnas.local k3s[142022]: E0313 20:07:51.848482  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:07:57 newnas.local k3s[142022]: E0313 20:07:57.315140  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:08:05 newnas.local k3s[142022]: E0313 20:08:05.848748  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:08:07 newnas.local k3s[142022]: E0313 20:08:07.336479  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:08:16 newnas.local k3s[142022]: E0313 20:08:16.848862  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:08:17 newnas.local k3s[142022]: E0313 20:08:17.363093  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:08:27 newnas.local k3s[142022]: E0313 20:08:27.387049  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:08:28 newnas.local k3s[142022]: E0313 20:08:28.848590  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:08:37 newnas.local k3s[142022]: E0313 20:08:37.409700  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:08:47 newnas.local k3s[142022]: E0313 20:08:47.435958  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:08:57 newnas.local k3s[142022]: E0313 20:08:57.462626  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:09:07 newnas.local k3s[142022]: E0313 20:09:07.486178  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:09:17 newnas.local k3s[142022]: E0313 20:09:17.510458  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:09:27 newnas.local k3s[142022]: E0313 20:09:27.537671  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:09:37 newnas.local k3s[142022]: E0313 20:09:37.562391  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:09:47 newnas.local k3s[142022]: E0313 20:09:47.588465  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:09:57 newnas.local k3s[142022]: E0313 20:09:57.610942  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:10:07 newnas.local k3s[142022]: E0313 20:10:07.634694  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:10:17 newnas.local k3s[142022]: E0313 20:10:17.660412  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:10:22 newnas.local k3s[142022]: E0313 20:10:22.331432  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="fc7d2d39953a: Download complete "
Mar 13 20:10:22 newnas.local k3s[142022]: E0313 20:10:22.331621  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:10:22 newnas.local k3s[142022]: E0313 20:10:22.331656  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:10:22 newnas.local k3s[142022]: E0313 20:10:22.331845  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 20:10:22 newnas.local k3s[142022]: E0313 20:10:22.331896  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:10:27 newnas.local k3s[142022]: E0313 20:10:27.686842  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:10:33 newnas.local k3s[142022]: E0313 20:10:33.848968  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:10:37 newnas.local k3s[142022]: E0313 20:10:37.710155  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:10:45 newnas.local k3s[142022]: E0313 20:10:45.849029  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:10:47 newnas.local k3s[142022]: E0313 20:10:47.745348  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:10:57 newnas.local k3s[142022]: E0313 20:10:57.776939  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:10:57 newnas.local k3s[142022]: E0313 20:10:57.849171  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:11:07 newnas.local k3s[142022]: E0313 20:11:07.813162  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:11:12 newnas.local k3s[142022]: E0313 20:11:12.848866  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:11:17 newnas.local k3s[142022]: E0313 20:11:17.836039  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:11:25 newnas.local k3s[142022]: E0313 20:11:25.848698  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:11:27 newnas.local k3s[142022]: E0313 20:11:27.862019  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:11:37 newnas.local k3s[142022]: E0313 20:11:37.848771  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:11:37 newnas.local k3s[142022]: E0313 20:11:37.896878  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:11:47 newnas.local k3s[142022]: E0313 20:11:47.922541  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:11:50 newnas.local k3s[142022]: E0313 20:11:50.848980  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:11:57 newnas.local k3s[142022]: E0313 20:11:57.949789  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:12:02 newnas.local k3s[142022]: E0313 20:12:02.849102  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:12:07 newnas.local k3s[142022]: E0313 20:12:07.977351  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:12:16 newnas.local k3s[142022]: E0313 20:12:16.848985  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:12:18 newnas.local k3s[142022]: E0313 20:12:18.007541  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:12:28 newnas.local k3s[142022]: E0313 20:12:28.037319  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:12:29 newnas.local k3s[142022]: E0313 20:12:29.848984  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:12:38 newnas.local k3s[142022]: E0313 20:12:38.068975  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:12:43 newnas.local k3s[142022]: E0313 20:12:43.848997  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:12:48 newnas.local k3s[142022]: E0313 20:12:48.095797  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:12:55 newnas.local k3s[142022]: E0313 20:12:55.849002  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:12:58 newnas.local k3s[142022]: E0313 20:12:58.125397  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:13:07 newnas.local k3s[142022]: E0313 20:13:07.849052  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:13:08 newnas.local k3s[142022]: E0313 20:13:08.152287  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:13:18 newnas.local k3s[142022]: E0313 20:13:18.179089  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:13:19 newnas.local k3s[142022]: E0313 20:13:19.848738  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:13:28 newnas.local k3s[142022]: E0313 20:13:28.202118  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:13:34 newnas.local k3s[142022]: E0313 20:13:34.848554  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:13:38 newnas.local k3s[142022]: E0313 20:13:38.228799  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:13:46 newnas.local k3s[142022]: E0313 20:13:46.848807  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:13:48 newnas.local k3s[142022]: E0313 20:13:48.249768  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:13:58 newnas.local k3s[142022]: E0313 20:13:58.281933  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:14:01 newnas.local k3s[142022]: E0313 20:14:01.848734  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:14:08 newnas.local k3s[142022]: E0313 20:14:08.307002  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:14:14 newnas.local k3s[142022]: E0313 20:14:14.848813  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:14:18 newnas.local k3s[142022]: E0313 20:14:18.331541  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:14:25 newnas.local k3s[142022]: E0313 20:14:25.848989  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:14:28 newnas.local k3s[142022]: E0313 20:14:28.359299  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:14:37 newnas.local k3s[142022]: E0313 20:14:37.848813  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:14:38 newnas.local k3s[142022]: E0313 20:14:38.389291  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:14:48 newnas.local k3s[142022]: E0313 20:14:48.460557  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:14:48 newnas.local k3s[142022]: E0313 20:14:48.848831  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:14:58 newnas.local k3s[142022]: E0313 20:14:58.485140  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:14:59 newnas.local k3s[142022]: E0313 20:14:59.848638  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:15:08 newnas.local k3s[142022]: E0313 20:15:08.514906  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:15:12 newnas.local k3s[142022]: E0313 20:15:12.849368  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:15:18 newnas.local k3s[142022]: E0313 20:15:18.549465  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:15:28 newnas.local k3s[142022]: E0313 20:15:28.576397  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:15:38 newnas.local k3s[142022]: E0313 20:15:38.612148  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:15:48 newnas.local k3s[142022]: E0313 20:15:48.643139  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:15:58 newnas.local k3s[142022]: E0313 20:15:58.667456  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:16:08 newnas.local k3s[142022]: E0313 20:16:08.698421  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:16:18 newnas.local k3s[142022]: E0313 20:16:18.722558  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:16:28 newnas.local k3s[142022]: E0313 20:16:28.748242  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:16:38 newnas.local k3s[142022]: E0313 20:16:38.769754  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:16:48 newnas.local k3s[142022]: E0313 20:16:48.797363  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:16:58 newnas.local k3s[142022]: E0313 20:16:58.430255  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="9c281a74223f: Download complete "
Mar 13 20:16:58 newnas.local k3s[142022]: E0313 20:16:58.430548  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:16:58 newnas.local k3s[142022]: E0313 20:16:58.430591  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:16:58 newnas.local k3s[142022]: E0313 20:16:58.430822  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 20:16:58 newnas.local k3s[142022]: E0313 20:16:58.430865  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:16:58 newnas.local k3s[142022]: E0313 20:16:58.822034  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:17:08 newnas.local k3s[142022]: E0313 20:17:08.851528  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:17:09 newnas.local k3s[142022]: E0313 20:17:09.849454  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:17:18 newnas.local k3s[142022]: E0313 20:17:18.883803  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:17:24 newnas.local k3s[142022]: E0313 20:17:24.848910  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:17:28 newnas.local k3s[142022]: E0313 20:17:28.915899  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:17:38 newnas.local k3s[142022]: E0313 20:17:38.848775  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:17:38 newnas.local k3s[142022]: E0313 20:17:38.947822  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:17:48 newnas.local k3s[142022]: E0313 20:17:48.982836  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:17:50 newnas.local k3s[142022]: E0313 20:17:50.849594  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:17:59 newnas.local k3s[142022]: E0313 20:17:59.012047  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:18:03 newnas.local k3s[142022]: E0313 20:18:03.848693  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:18:09 newnas.local k3s[142022]: E0313 20:18:09.041426  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:18:16 newnas.local k3s[142022]: E0313 20:18:16.848662  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:18:19 newnas.local k3s[142022]: E0313 20:18:19.069958  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:18:28 newnas.local k3s[142022]: E0313 20:18:28.849191  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:18:29 newnas.local k3s[142022]: E0313 20:18:29.098038  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:18:39 newnas.local k3s[142022]: E0313 20:18:39.130422  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:18:42 newnas.local k3s[142022]: E0313 20:18:42.848609  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:18:49 newnas.local k3s[142022]: E0313 20:18:49.164559  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:18:57 newnas.local k3s[142022]: E0313 20:18:57.848888  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:18:59 newnas.local k3s[142022]: E0313 20:18:59.195157  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:19:09 newnas.local k3s[142022]: E0313 20:19:09.217329  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:19:10 newnas.local k3s[142022]: E0313 20:19:10.848810  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:19:19 newnas.local k3s[142022]: E0313 20:19:19.244045  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:19:24 newnas.local k3s[142022]: E0313 20:19:24.849129  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:19:29 newnas.local k3s[142022]: E0313 20:19:29.291723  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:19:36 newnas.local k3s[142022]: E0313 20:19:36.849362  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:19:39 newnas.local k3s[142022]: E0313 20:19:39.323638  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:19:48 newnas.local k3s[142022]: E0313 20:19:48.848978  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:19:49 newnas.local k3s[142022]: E0313 20:19:49.352759  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:19:59 newnas.local k3s[142022]: E0313 20:19:59.378390  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:20:03 newnas.local k3s[142022]: E0313 20:20:03.849207  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:20:09 newnas.local k3s[142022]: E0313 20:20:09.403451  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:20:16 newnas.local k3s[142022]: E0313 20:20:16.848941  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:20:19 newnas.local k3s[142022]: E0313 20:20:19.428108  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:20:29 newnas.local k3s[142022]: E0313 20:20:29.454093  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:20:31 newnas.local k3s[142022]: E0313 20:20:31.848762  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:20:39 newnas.local k3s[142022]: E0313 20:20:39.475353  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:20:46 newnas.local k3s[142022]: E0313 20:20:46.848754  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:20:49 newnas.local k3s[142022]: E0313 20:20:49.502054  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:20:57 newnas.local k3s[142022]: E0313 20:20:57.848496  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:20:59 newnas.local k3s[142022]: E0313 20:20:59.524123  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:21:09 newnas.local k3s[142022]: E0313 20:21:09.551179  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:21:10 newnas.local k3s[142022]: E0313 20:21:10.848978  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:21:19 newnas.local k3s[142022]: E0313 20:21:19.573972  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:21:21 newnas.local k3s[142022]: E0313 20:21:21.848788  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:21:29 newnas.local k3s[142022]: E0313 20:21:29.599130  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:21:34 newnas.local k3s[142022]: E0313 20:21:34.849191  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:21:39 newnas.local k3s[142022]: E0313 20:21:39.626136  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:21:48 newnas.local k3s[142022]: E0313 20:21:48.848777  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:21:49 newnas.local k3s[142022]: E0313 20:21:49.654222  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:21:59 newnas.local k3s[142022]: E0313 20:21:59.696169  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:22:09 newnas.local k3s[142022]: E0313 20:22:09.721265  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:22:19 newnas.local k3s[142022]: E0313 20:22:19.744690  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:22:29 newnas.local k3s[142022]: E0313 20:22:29.768707  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:22:39 newnas.local k3s[142022]: E0313 20:22:39.794372  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:22:49 newnas.local k3s[142022]: E0313 20:22:49.820957  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:22:59 newnas.local k3s[142022]: E0313 20:22:59.845032  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:23:09 newnas.local k3s[142022]: E0313 20:23:09.867441  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:23:19 newnas.local k3s[142022]: E0313 20:23:19.894406  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:23:29 newnas.local k3s[142022]: E0313 20:23:29.918457  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:23:34 newnas.local k3s[142022]: E0313 20:23:34.410125  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="9c281a74223f: Download complete "
Mar 13 20:23:34 newnas.local k3s[142022]: E0313 20:23:34.410331  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:23:34 newnas.local k3s[142022]: E0313 20:23:34.410364  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:23:34 newnas.local k3s[142022]: E0313 20:23:34.410570  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 20:23:34 newnas.local k3s[142022]: E0313 20:23:34.410625  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:23:39 newnas.local k3s[142022]: E0313 20:23:39.943038  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:23:49 newnas.local k3s[142022]: E0313 20:23:49.848834  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:23:49 newnas.local k3s[142022]: E0313 20:23:49.969310  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:23:59 newnas.local k3s[142022]: E0313 20:23:59.995547  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:24:01 newnas.local k3s[142022]: E0313 20:24:01.848496  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:24:10 newnas.local k3s[142022]: E0313 20:24:10.021601  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:24:16 newnas.local k3s[142022]: E0313 20:24:16.848835  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:24:20 newnas.local k3s[142022]: E0313 20:24:20.046205  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:24:30 newnas.local k3s[142022]: E0313 20:24:30.070819  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:24:31 newnas.local k3s[142022]: E0313 20:24:31.848647  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:24:40 newnas.local k3s[142022]: E0313 20:24:40.094467  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:24:45 newnas.local k3s[142022]: E0313 20:24:45.848918  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:24:50 newnas.local k3s[142022]: E0313 20:24:50.117557  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:24:58 newnas.local k3s[142022]: E0313 20:24:58.848685  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:25:00 newnas.local k3s[142022]: E0313 20:25:00.141583  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:25:10 newnas.local k3s[142022]: E0313 20:25:10.168212  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:25:11 newnas.local k3s[142022]: E0313 20:25:11.849091  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:25:20 newnas.local k3s[142022]: E0313 20:25:20.192516  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:25:24 newnas.local k3s[142022]: E0313 20:25:24.848838  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:25:30 newnas.local k3s[142022]: E0313 20:25:30.217006  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:25:37 newnas.local k3s[142022]: E0313 20:25:37.848868  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:25:40 newnas.local k3s[142022]: E0313 20:25:40.240456  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:25:48 newnas.local k3s[142022]: E0313 20:25:48.848583  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:25:50 newnas.local k3s[142022]: E0313 20:25:50.265711  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:26:00 newnas.local k3s[142022]: E0313 20:26:00.290825  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:26:03 newnas.local k3s[142022]: E0313 20:26:03.848633  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:26:10 newnas.local k3s[142022]: E0313 20:26:10.311804  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:26:16 newnas.local k3s[142022]: E0313 20:26:16.848505  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:26:20 newnas.local k3s[142022]: E0313 20:26:20.340845  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:26:27 newnas.local k3s[142022]: E0313 20:26:27.848582  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:26:30 newnas.local k3s[142022]: E0313 20:26:30.364789  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:26:40 newnas.local k3s[142022]: E0313 20:26:40.389317  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:26:42 newnas.local k3s[142022]: E0313 20:26:42.848597  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:26:50 newnas.local k3s[142022]: E0313 20:26:50.412483  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:26:53 newnas.local k3s[142022]: E0313 20:26:53.849307  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:27:00 newnas.local k3s[142022]: E0313 20:27:00.438133  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:27:04 newnas.local k3s[142022]: E0313 20:27:04.848682  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:27:10 newnas.local k3s[142022]: E0313 20:27:10.463437  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:27:18 newnas.local k3s[142022]: E0313 20:27:18.848903  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:27:20 newnas.local k3s[142022]: E0313 20:27:20.486198  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:27:30 newnas.local k3s[142022]: E0313 20:27:30.511563  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:27:30 newnas.local k3s[142022]: E0313 20:27:30.848743  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:27:40 newnas.local k3s[142022]: E0313 20:27:40.534892  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:27:43 newnas.local k3s[142022]: E0313 20:27:43.848571  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:27:50 newnas.local k3s[142022]: E0313 20:27:50.556799  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:27:56 newnas.local k3s[142022]: E0313 20:27:56.848678  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:28:00 newnas.local k3s[142022]: E0313 20:28:00.583692  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:28:09 newnas.local k3s[142022]: E0313 20:28:09.848952  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:28:10 newnas.local k3s[142022]: E0313 20:28:10.610313  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:28:20 newnas.local k3s[142022]: E0313 20:28:20.633804  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:28:23 newnas.local k3s[142022]: E0313 20:28:23.848893  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:28:30 newnas.local k3s[142022]: E0313 20:28:30.656131  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:28:40 newnas.local k3s[142022]: E0313 20:28:40.683159  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:28:50 newnas.local k3s[142022]: E0313 20:28:50.707232  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:29:00 newnas.local k3s[142022]: E0313 20:29:00.734229  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:29:10 newnas.local k3s[142022]: E0313 20:29:10.755663  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:29:20 newnas.local k3s[142022]: E0313 20:29:20.784524  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:29:30 newnas.local k3s[142022]: E0313 20:29:30.805792  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:29:40 newnas.local k3s[142022]: E0313 20:29:40.833495  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:29:50 newnas.local k3s[142022]: E0313 20:29:50.858776  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:30:00 newnas.local k3s[142022]: E0313 20:30:00.884625  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:30:06 newnas.local k3s[142022]: E0313 20:30:06.377871  142022 kube_docker_client.go:340] "Cancel pulling image because of exceed image pull deadline, record latest progress" image="plexinc/pms-docker:1.25.6.5577-c8bd13540" deadline="1m0s" progress="9c281a74223f: Download complete "
Mar 13 20:30:06 newnas.local k3s[142022]: E0313 20:30:06.378097  142022 remote_image.go:114] "PullImage from image service failed" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:30:06 newnas.local k3s[142022]: E0313 20:30:06.378132  142022 kuberuntime_image.go:51] "Failed to pull image" err="rpc error: code = Unknown desc = context canceled" image="plexinc/pms-docker:1.25.6.5577-c8bd13540"
Mar 13 20:30:06 newnas.local k3s[142022]: E0313 20:30:06.378342  142022 kuberuntime_manager.go:864] container &Container{Name:plex,Image:plexinc/pms-docker:1.25.6.5577-c8bd13540,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:pms,HostPort:0,ContainerPort:32400,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna,HostPort:0,ContainerPort:32469,Protocol:TCP,HostIP:,},ContainerPort{Name:plex-dlna-udp,HostPort:0,ContainerPort:1900,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm1,HostPort:0,ContainerPort:32410,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm2,HostPort:0,ContainerPort:32412,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm3,HostPort:0,ContainerPort:32413,Protocol:UDP,HostIP:,},ContainerPort{Name:plex-gdm4,HostPort:0,ContainerPort:32414,Protocol:UDP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:KUBE_NAMESPACE,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},EnvVar{Name:TZ,Value:America/New_York,ValueFrom:nil,},EnvVar{Name:PLEX_CLAIM,Value:,ValueFrom:nil,},EnvVar{Name:PMS_INTERNAL_ADDRESS,Value:http://plex-test:32400,ValueFrom:nil,},EnvVar{Name:PMS_IMAGE,Value:plexinc/pms-docker:1.25.6.5577-c8bd13540,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:config,ReadOnly:false,MountPath:/config,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:data,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared,ReadOnly:false,MountPath:/shared,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:shared-logs,ReadOnly:false,MountPath:/config/Library/Application Support/Plex Media Server/Logs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:transcode,ReadOnly:false,MountPath:/transcode,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:plex-probe-check,ReadOnly:true,MountPath:/bin/plex_probe_check.sh,SubPath:entrypoint.sh,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-p6b2c,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:5,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{Handler:Handler{Exec:&ExecAction{Command:[/bin/plex_probe_check.sh],},HTTPGet:nil,TCPSocket:nil,},InitialDelaySeconds:5,TimeoutSeconds:1,PeriodSeconds:15,SuccessThreshold:1,FailureThreshold:40,TerminationGracePeriodSeconds:nil,},} start failed in pod plex-test-8574fd5c48-96c29_ix-plex-test(390f57ac-8a2a-4385-a396-b4f703c5c108): ErrImagePull: rpc error: code = Unknown desc = context canceled
Mar 13 20:30:06 newnas.local k3s[142022]: E0313 20:30:06.378392  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ErrImagePull: \"rpc error: code = Unknown desc = context canceled\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:30:10 newnas.local k3s[142022]: E0313 20:30:10.908060  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:30:18 newnas.local k3s[142022]: E0313 20:30:18.848692  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:30:20 newnas.local k3s[142022]: E0313 20:30:20.932140  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:30:30 newnas.local k3s[142022]: E0313 20:30:30.848960  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:30:30 newnas.local k3s[142022]: E0313 20:30:30.957982  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:30:40 newnas.local k3s[142022]: E0313 20:30:40.984848  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:30:43 newnas.local k3s[142022]: E0313 20:30:43.848633  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:30:51 newnas.local k3s[142022]: E0313 20:30:51.009823  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:30:56 newnas.local k3s[142022]: E0313 20:30:56.849004  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:31:01 newnas.local k3s[142022]: E0313 20:31:01.033983  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:31:08 newnas.local k3s[142022]: E0313 20:31:08.848804  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:31:11 newnas.local k3s[142022]: E0313 20:31:11.056529  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:31:20 newnas.local k3s[142022]: E0313 20:31:20.848847  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:31:21 newnas.local k3s[142022]: E0313 20:31:21.098614  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:31:31 newnas.local k3s[142022]: E0313 20:31:31.120447  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:31:34 newnas.local k3s[142022]: E0313 20:31:34.848540  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:31:41 newnas.local k3s[142022]: E0313 20:31:41.147698  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:31:47 newnas.local k3s[142022]: E0313 20:31:47.848833  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:31:51 newnas.local k3s[142022]: E0313 20:31:51.171317  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:32:01 newnas.local k3s[142022]: E0313 20:32:01.197235  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:32:02 newnas.local k3s[142022]: E0313 20:32:02.848732  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:32:11 newnas.local k3s[142022]: E0313 20:32:11.221376  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:32:16 newnas.local k3s[142022]: E0313 20:32:16.848743  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:32:21 newnas.local k3s[142022]: E0313 20:32:21.244790  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:32:28 newnas.local k3s[142022]: E0313 20:32:28.848455  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:32:31 newnas.local k3s[142022]: E0313 20:32:31.267700  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:32:41 newnas.local k3s[142022]: E0313 20:32:41.295791  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:32:42 newnas.local k3s[142022]: E0313 20:32:42.848616  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:32:51 newnas.local k3s[142022]: E0313 20:32:51.320323  142022 cadvisor_stats_provider.go:151] "Unable to fetch pod etc hosts stats" err="failed to get stats failed command 'du' ($ nice -n 19 du -x -s -B 1) on path /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/etc-hosts with error exit status 1" pod="ix-plex-test/plex-test-8574fd5c48-96c29"
Mar 13 20:32:54 newnas.local k3s[142022]: E0313 20:32:54.848582  142022 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"plex\" with ImagePullBackOff: \"Back-off pulling image \\\"plexinc/pms-docker:1.25.6.5577-c8bd13540\\\"\"" pod="ix-plex-test/plex-test-8574fd5c48-96c29" podUID=390f57ac-8a2a-4385-a396-b4f703c5c108
Mar 13 20:32:55 newnas.local k3s[142022]: I0313 20:32:55.495177  142022 event.go:291] "Event occurred" object="ix-plex-test/plex-test" kind="Deployment" apiVersion="apps/v1" type="Normal" reason="ScalingReplicaSet" message="Scaled down replica set plex-test-8574fd5c48 to 0"
Mar 13 20:32:55 newnas.local k3s[142022]: I0313 20:32:55.510655  142022 event.go:291] "Event occurred" object="ix-plex-test/plex-test-8574fd5c48" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulDelete" message="Deleted pod: plex-test-8574fd5c48-96c29"
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672594  142022 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-config\") pod \"390f57ac-8a2a-4385-a396-b4f703c5c108\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672647  142022 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"transcode\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-transcode\") pod \"390f57ac-8a2a-4385-a396-b4f703c5c108\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672704  142022 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"plex-probe-check\" (UniqueName: \"kubernetes.io/configmap/390f57ac-8a2a-4385-a396-b4f703c5c108-plex-probe-check\") pod \"390f57ac-8a2a-4385-a396-b4f703c5c108\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672755  142022 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/390f57ac-8a2a-4385-a396-b4f703c5c108-shared\") pod \"390f57ac-8a2a-4385-a396-b4f703c5c108\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672807  142022 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"shared-logs\" (UniqueName: \"kubernetes.io/empty-dir/390f57ac-8a2a-4385-a396-b4f703c5c108-shared-logs\") pod \"390f57ac-8a2a-4385-a396-b4f703c5c108\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672867  142022 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"kube-api-access-p6b2c\" (UniqueName: \"kubernetes.io/projected/390f57ac-8a2a-4385-a396-b4f703c5c108-kube-api-access-p6b2c\") pod \"390f57ac-8a2a-4385-a396-b4f703c5c108\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672917  142022 reconciler.go:196] "operationExecutor.UnmountVolume started for volume \"data\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-data\") pod \"390f57ac-8a2a-4385-a396-b4f703c5c108\" (UID: \"390f57ac-8a2a-4385-a396-b4f703c5c108\") "
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672961  142022 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-transcode" (OuterVolumeSpecName: "transcode") pod "390f57ac-8a2a-4385-a396-b4f703c5c108" (UID: "390f57ac-8a2a-4385-a396-b4f703c5c108"). InnerVolumeSpecName "transcode". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.673009  142022 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-config" (OuterVolumeSpecName: "config") pod "390f57ac-8a2a-4385-a396-b4f703c5c108" (UID: "390f57ac-8a2a-4385-a396-b4f703c5c108"). InnerVolumeSpecName "config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.672972  142022 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-data" (OuterVolumeSpecName: "data") pod "390f57ac-8a2a-4385-a396-b4f703c5c108" (UID: "390f57ac-8a2a-4385-a396-b4f703c5c108"). InnerVolumeSpecName "data". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Mar 13 20:32:59 newnas.local k3s[142022]: W0313 20:32:59.673430  142022 empty_dir.go:520] Warning: Failed to clear quota on /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/volumes/kubernetes.io~empty-dir/shared: clearQuota called, but quotas disabled
Mar 13 20:32:59 newnas.local k3s[142022]: W0313 20:32:59.673508  142022 empty_dir.go:520] Warning: Failed to clear quota on /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/volumes/kubernetes.io~empty-dir/shared-logs: clearQuota called, but quotas disabled
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.673526  142022 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/390f57ac-8a2a-4385-a396-b4f703c5c108-shared" (OuterVolumeSpecName: "shared") pod "390f57ac-8a2a-4385-a396-b4f703c5c108" (UID: "390f57ac-8a2a-4385-a396-b4f703c5c108"). InnerVolumeSpecName "shared". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.673583  142022 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/390f57ac-8a2a-4385-a396-b4f703c5c108-shared-logs" (OuterVolumeSpecName: "shared-logs") pod "390f57ac-8a2a-4385-a396-b4f703c5c108" (UID: "390f57ac-8a2a-4385-a396-b4f703c5c108"). InnerVolumeSpecName "shared-logs". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Mar 13 20:32:59 newnas.local k3s[142022]: W0313 20:32:59.673388  142022 empty_dir.go:520] Warning: Failed to clear quota on /var/lib/kubelet/pods/390f57ac-8a2a-4385-a396-b4f703c5c108/volumes/kubernetes.io~configmap/plex-probe-check: clearQuota called, but quotas disabled
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.674041  142022 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/configmap/390f57ac-8a2a-4385-a396-b4f703c5c108-plex-probe-check" (OuterVolumeSpecName: "plex-probe-check") pod "390f57ac-8a2a-4385-a396-b4f703c5c108" (UID: "390f57ac-8a2a-4385-a396-b4f703c5c108"). InnerVolumeSpecName "plex-probe-check". PluginName "kubernetes.io/configmap", VolumeGidValue ""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.675678  142022 operation_generator.go:829] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/390f57ac-8a2a-4385-a396-b4f703c5c108-kube-api-access-p6b2c" (OuterVolumeSpecName: "kube-api-access-p6b2c") pod "390f57ac-8a2a-4385-a396-b4f703c5c108" (UID: "390f57ac-8a2a-4385-a396-b4f703c5c108"). InnerVolumeSpecName "kube-api-access-p6b2c". PluginName "kubernetes.io/projected", VolumeGidValue ""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.773264  142022 reconciler.go:319] "Volume detached for volume \"plex-probe-check\" (UniqueName: \"kubernetes.io/configmap/390f57ac-8a2a-4385-a396-b4f703c5c108-plex-probe-check\") on node \"ix-truenas\" DevicePath \"\""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.773353  142022 reconciler.go:319] "Volume detached for volume \"kube-api-access-p6b2c\" (UniqueName: \"kubernetes.io/projected/390f57ac-8a2a-4385-a396-b4f703c5c108-kube-api-access-p6b2c\") on node \"ix-truenas\" DevicePath \"\""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.773404  142022 reconciler.go:319] "Volume detached for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/390f57ac-8a2a-4385-a396-b4f703c5c108-shared\") on node \"ix-truenas\" DevicePath \"\""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.773442  142022 reconciler.go:319] "Volume detached for volume \"shared-logs\" (UniqueName: \"kubernetes.io/empty-dir/390f57ac-8a2a-4385-a396-b4f703c5c108-shared-logs\") on node \"ix-truenas\" DevicePath \"\""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.773475  142022 reconciler.go:319] "Volume detached for volume \"data\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-data\") on node \"ix-truenas\" DevicePath \"\""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.773491  142022 reconciler.go:319] "Volume detached for volume \"config\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-config\") on node \"ix-truenas\" DevicePath \"\""
Mar 13 20:32:59 newnas.local k3s[142022]: I0313 20:32:59.773507  142022 reconciler.go:319] "Volume detached for volume \"transcode\" (UniqueName: \"kubernetes.io/host-path/390f57ac-8a2a-4385-a396-b4f703c5c108-transcode\") on node \"ix-truenas\" DevicePath \"\""
debug finished in 0 seconds for K3s Logs (last 1000 lines)


