+--------------------------------------------------------------------------------+
+                      Kubernetes Configuration @1674050909                      +
+--------------------------------------------------------------------------------+
{
  "id": 1,
  "pool": "VMs",
  "cluster_cidr": "172.16.0.0/16",
  "service_cidr": "172.17.0.0/16",
  "cluster_dns_ip": "172.17.0.10",
  "route_v4_interface": "br0",
  "route_v4_gateway": "10.5.5.1",
  "route_v6_interface": null,
  "route_v6_gateway": null,
  "node_ip": "0.0.0.0",
  "configure_gpus": true,
  "servicelb": false,
  "validate_host_path": true,
  "dataset": "VMs/ix-applications"
}
debug finished in 0 seconds for Kubernetes Configuration


+--------------------------------------------------------------------------------+
+                     k3s kubectl describe nodes @1674050910                     +
+--------------------------------------------------------------------------------+
Name:               ix-truenas
Roles:              control-plane,master
Labels:             beta.kubernetes.io/arch=amd64
                    beta.kubernetes.io/os=linux
                    egress.k3s.io/cluster=true
                    kubernetes.io/arch=amd64
                    kubernetes.io/hostname=ix-truenas
                    kubernetes.io/os=linux
                    node-role.kubernetes.io/control-plane=true
                    node-role.kubernetes.io/master=true
                    openebs.io/nodeid=ix-truenas
                    openebs.io/nodename=ix-truenas
Annotations:        csi.volume.kubernetes.io/nodeid: {"zfs.csi.openebs.io":"ix-truenas"}
                    k3s.io/node-args:
                      ["server","--cluster-cidr","172.16.0.0/16","--cluster-dns","172.17.0.10","--data-dir","/mnt/VMs/ix-applications/k3s","--disable","servicel...
                    k3s.io/node-config-hash: M5VCJ3QDTATOI7I7U55BD37HSH74EF6A5PV4PZTMFBN6EB4PMFOA====
                    k3s.io/node-env: {"K3S_DATA_DIR":"/mnt/VMs/ix-applications/k3s/data/3622b68605bf9a49243ca44f328acbb9e681a95150de9d9f94baaba1d5511603"}
                    node.alpha.kubernetes.io/ttl: 0
                    volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp:  Sat, 05 Mar 2022 12:59:43 -0800
Taints:             <none>
Unschedulable:      false
Lease:
  HolderIdentity:  ix-truenas
  AcquireTime:     <unset>
  RenewTime:       Wed, 18 Jan 2023 06:08:27 -0800
Conditions:
  Type             Status  LastHeartbeatTime                 LastTransitionTime                Reason                       Message
  ----             ------  -----------------                 ------------------                ------                       -------
  MemoryPressure   False   Wed, 18 Jan 2023 06:03:55 -0800   Wed, 14 Sep 2022 05:03:27 -0700   KubeletHasSufficientMemory   kubelet has sufficient memory available
  DiskPressure     False   Wed, 18 Jan 2023 06:03:55 -0800   Wed, 14 Sep 2022 05:03:27 -0700   KubeletHasNoDiskPressure     kubelet has no disk pressure
  PIDPressure      False   Wed, 18 Jan 2023 06:03:55 -0800   Wed, 14 Sep 2022 05:03:27 -0700   KubeletHasSufficientPID      kubelet has sufficient PID available
  Ready            True    Wed, 18 Jan 2023 06:03:55 -0800   Wed, 18 Jan 2023 04:01:09 -0800   KubeletReady                 kubelet is posting ready status. AppArmor enabled
Addresses:
  InternalIP:  10.5.5.10
  Hostname:    ix-truenas
Capacity:
  cpu:                20
  ephemeral-storage:  417975552Ki
  hugepages-1Gi:      0
  hugepages-2Mi:      0
  memory:             32804224Ki
  nvidia.com/gpu:     0
  pods:               250
Allocatable:
  cpu:                20
  ephemeral-storage:  406606616667
  hugepages-1Gi:      0
  hugepages-2Mi:      0
  memory:             32804224Ki
  nvidia.com/gpu:     0
  pods:               250
System Info:
  Machine ID:                 f71c57914e094dfb8451f71f6feb246c
  System UUID:                edd1db80-7796-11e3-9e3c-f0921cde3e20
  Boot ID:                    5837e5c6-5b85-4820-a44e-e2af47a700c6
  Kernel Version:             5.15.79+truenas
  OS Image:                   Debian GNU/Linux 11 (bullseye)
  Operating System:           linux
  Architecture:               amd64
  Container Runtime Version:  docker://Unknown
  Kubelet Version:            v1.25.3+k3s-9afcd6b9-dirty
  Kube-Proxy Version:         v1.25.3+k3s-9afcd6b9-dirty
PodCIDR:                      172.16.0.0/16
PodCIDRs:                     172.16.0.0/16
Non-terminated Pods:          (12 in total)
  Namespace                   Name                                        CPU Requests  CPU Limits  Memory Requests  Memory Limits  Age
  ---------                   ----                                        ------------  ----------  ---------------  -------------  ---
  kube-system                 openebs-zfs-node-wjj5j                      0 (0%)        0 (0%)      0 (0%)           0 (0%)         127m
  kube-system                 coredns-d76bd69b-plvk8                      100m (0%)     0 (0%)      70Mi (0%)        170Mi (0%)     127m
  cnpg-system                 cnpg-controller-manager-854876b995-ccsfw    100m (0%)     100m (0%)   100Mi (0%)       200Mi (0%)     127m
  prometheus-operator         prometheus-operator-5c7445d877-npglz        100m (0%)     200m (1%)   100Mi (0%)       200Mi (0%)     127m
  metallb-system              speaker-rlx98                               0 (0%)        0 (0%)      0 (0%)           0 (0%)         127m
  kube-system                 openebs-zfs-controller-0                    0 (0%)        0 (0%)      0 (0%)           0 (0%)         126m
  metallb-system              controller-7597dd4f7b-6dxpb                 0 (0%)        0 (0%)      0 (0%)           0 (0%)         127m
  ix-nextcloud                nextcloud-postgresql-0                      10m (0%)      4 (20%)     50Mi (0%)        8Gi (25%)      127m
  ix-ubiquiti                 ubiquiti-unifi-68fbf8dd6d-g5b4t             10m (0%)      4 (20%)     50Mi (0%)        8Gi (25%)      127m
  ix-nextcloud                nextcloud-redis-0                           10m (0%)      4 (20%)     50Mi (0%)        8Gi (25%)      126m
  ix-nextcloud                nextcloud-785ff76589-q8zfn                  10m (0%)      4 (20%)     50Mi (0%)        8Gi (25%)      127m
  kube-system                 nvidia-device-plugin-daemonset-55rxn        0 (0%)        0 (0%)      0 (0%)           0 (0%)         127m
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  Resource           Requests    Limits
  --------           --------    ------
  cpu                340m (1%)   16300m (81%)
  memory             470Mi (1%)  33338Mi (104%)
  ephemeral-storage  0 (0%)      0 (0%)
  hugepages-1Gi      0 (0%)      0 (0%)
  hugepages-2Mi      0 (0%)      0 (0%)
  nvidia.com/gpu     0           0
Events:              <none>
debug finished in 0 seconds for k3s kubectl describe nodes


+--------------------------------------------------------------------------------+
+k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide @1674050910+
+--------------------------------------------------------------------------------+
NAMESPACE             NAME                                           READY   STATUS             RESTARTS      AGE     IP             NODE         NOMINATED NODE   READINESS GATES
cnpg-system           pod/cnpg-controller-manager-854876b995-wxd8f   0/1     Error              1             19h     <none>         ix-truenas   <none>           <none>
kube-system           pod/coredns-d76bd69b-6zwqv                     0/1     Error              1             19h     <none>         ix-truenas   <none>           <none>
prometheus-operator   pod/prometheus-operator-5c7445d877-t9kml       0/1     Error              1             19h     <none>         ix-truenas   <none>           <none>
kube-system           pod/openebs-zfs-node-wjj5j                     2/2     Running            0             127m    10.5.5.10      ix-truenas   <none>           <none>
kube-system           pod/coredns-d76bd69b-plvk8                     1/1     Running            0             127m    172.16.5.113   ix-truenas   <none>           <none>
cnpg-system           pod/cnpg-controller-manager-854876b995-ccsfw   1/1     Running            0             127m    172.16.5.112   ix-truenas   <none>           <none>
prometheus-operator   pod/prometheus-operator-5c7445d877-npglz       1/1     Running            0             127m    172.16.5.118   ix-truenas   <none>           <none>
metallb-system        pod/speaker-rlx98                              1/1     Running            0             127m    10.5.5.10      ix-truenas   <none>           <none>
kube-system           pod/openebs-zfs-controller-0                   5/5     Running            0             126m    172.16.5.119   ix-truenas   <none>           <none>
metallb-system        pod/controller-7597dd4f7b-6dxpb                1/1     Running            0             127m    172.16.5.111   ix-truenas   <none>           <none>
ix-nextcloud          pod/nextcloud-postgresql-0                     1/1     Running            0             127m    172.16.5.115   ix-truenas   <none>           <none>
ix-ubiquiti           pod/ubiquiti-unifi-68fbf8dd6d-g5b4t            1/1     Running            0             127m    172.16.5.117   ix-truenas   <none>           <none>
ix-nextcloud          pod/nextcloud-redis-0                          1/1     Running            0             126m    172.16.5.120   ix-truenas   <none>           <none>
ix-nextcloud          pod/nextcloud-785ff76589-q8zfn                 4/4     Running            0             127m    172.16.5.116   ix-truenas   <none>           <none>
ix-nextcloud          pod/nextcloud-cronjob-27900840-9n8ws           0/1     Completed          0             8m30s   172.16.5.144   ix-truenas   <none>           <none>
kube-system           pod/nvidia-device-plugin-daemonset-55rxn       0/1     CrashLoopBackOff   29 (4m ago)   127m    172.16.5.110   ix-truenas   <none>           <none>
ix-nextcloud          pod/nextcloud-cronjob-27900845-w579j           0/1     Completed          0             3m30s   172.16.5.145   ix-truenas   <none>           <none>

NAMESPACE             NAME                           TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                               AGE    SELECTOR
default               service/kubernetes             ClusterIP   172.17.0.1       <none>        443/TCP                               318d   <none>
kube-system           service/kube-dns               ClusterIP   172.17.0.10      <none>        53/UDP,53/TCP,9153/TCP                318d   k8s-app=kube-dns
cnpg-system           service/cnpg-webhook-service   ClusterIP   172.17.130.213   <none>        443/TCP                               65d    app.kubernetes.io/name=cloudnative-pg
kube-system           service/kubelet                ClusterIP   None             <none>        10250/TCP,10255/TCP,4194/TCP          67d    <none>
ix-nextcloud          service/nextcloud-postgresql   ClusterIP   172.17.25.24     <none>        5432/TCP                              65d    app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=postgresql
ix-nextcloud          service/nextcloud-redis        ClusterIP   172.17.149.206   <none>        6379/TCP                              65d    app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=redis
ix-nextcloud          service/nextcloud-backend      ClusterIP   172.17.236.123   <none>        9000/TCP,7867/TCP,7868/TCP,9090/TCP   65d    app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=nextcloud
metallb-system        service/webhook-service        ClusterIP   172.17.80.175    <none>        443/TCP                               69d    component=controller
prometheus-operator   service/prometheus-operator    ClusterIP   None             <none>        8080/TCP                              67d    app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator

NAMESPACE        NAME                                            DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR            AGE    CONTAINERS                                     IMAGES                                                                             SELECTOR
kube-system      daemonset.apps/nvidia-device-plugin-daemonset   1         1         0       1            0           <none>                   318d   nvidia-device-plugin-ctr                       nvidia/k8s-device-plugin:v0.10.0                                                   name=nvidia-device-plugin-ds
kube-system      daemonset.apps/openebs-zfs-node                 1         1         1       1            1           <none>                   318d   csi-node-driver-registrar,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0,openebs/zfs-driver:2.0.0   app=openebs-zfs-node
metallb-system   daemonset.apps/speaker                          1         1         1       1            1           kubernetes.io/os=linux   69d    speaker                                        quay.io/metallb/speaker:v0.13.7                                                    app=metallb,component=speaker

NAMESPACE             NAME                                      READY   UP-TO-DATE   AVAILABLE   AGE    CONTAINERS                      IMAGES                                                                                                                                                                                                                                                                                                                                                                                                                                               SELECTOR
ix-plex               deployment.apps/plex                      0/0     0            0           193d   plex                            tccr.io/truecharts/plex:1.29.2.6364@sha256:16eac4e90b6873107332cd15bdd2e84eea4479adafbcb8032585c180b22b28c9                                                                                                                                                                                                                                                                                                                                          app.kubernetes.io/instance=plex,app.kubernetes.io/name=plex
kube-system           deployment.apps/coredns                   1/1     1            1           318d   coredns                         rancher/mirrored-coredns-coredns:1.9.1                                                                                                                                                                                                                                                                                                                                                                                                               k8s-app=kube-dns
cnpg-system           deployment.apps/cnpg-controller-manager   1/1     1            1           65d    manager                         ghcr.io/cloudnative-pg/cloudnative-pg:1.18.0                                                                                                                                                                                                                                                                                                                                                                                                         app.kubernetes.io/name=cloudnative-pg
prometheus-operator   deployment.apps/prometheus-operator       1/1     1            1           67d    prometheus-operator             quay.io/prometheus-operator/prometheus-operator:v0.60.1                                                                                                                                                                                                                                                                                                                                                                                              app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
metallb-system        deployment.apps/controller                1/1     1            1           69d    controller                      quay.io/metallb/controller:v0.13.7                                                                                                                                                                                                                                                                                                                                                                                                                   app=metallb,component=controller
ix-ubiquiti           deployment.apps/ubiquiti-unifi            1/1     1            1           84d    ubiquiti-unifi                  tccr.io/truecharts/unifi:7.2.95@sha256:1d8afe976c4f081274295f0ba06521162341375647883add244db6a6508df0ec                                                                                                                                                                                                                                                                                                                                              app.kubernetes.io/instance=ubiquiti,app.kubernetes.io/name=unifi
ix-nextcloud          deployment.apps/nextcloud                 1/1     1            1           65d    nextcloud,hpb,imaginary,nginx   tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f,tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f,h2non/imaginary:1.2.4@sha256:7facb4221047a5e79b9e902f380247f4e5bf4376400d0badbeb738d3e1c2f654,tccr.io/truecharts/nginx-unprivileged:1.23.2@sha256:86064448a2764926ca215714b36a80561bcfed064c435245cee641f4ce14768b   app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=nextcloud

NAMESPACE      NAME                                      READY   AGE    CONTAINERS                                                                           IMAGES
kube-system    statefulset.apps/openebs-zfs-controller   1/1     318d   csi-resizer,csi-snapshotter,snapshot-controller,csi-provisioner,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-resizer:v1.2.0,k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0,k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0,k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0,openebs/zfs-driver:2.0.0
ix-nextcloud   statefulset.apps/nextcloud-postgresql     1/1     65d    nextcloud-postgresql                                                                 tccr.io/truecharts/postgresql:14.5.0@sha256:1d5c95be8ddb4182509773cfc4d75101adcbd1ae97a47011cd52123dff59c691
ix-nextcloud   statefulset.apps/nextcloud-redis          1/1     65d    nextcloud-redis                                                                      tccr.io/truecharts/redis:7.0.5@sha256:a0de36bdb830474aa89a387be8297edac22fc668cd01ad6831e1d3fa70b6b8df

NAMESPACE   NAME                                                          PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
            storageclass.storage.k8s.io/ix-storage-class-plex             zfs.csi.openebs.io   Retain          Immediate           true                   193d
            storageclass.storage.k8s.io/ix-storage-class-ubiquiti         zfs.csi.openebs.io   Retain          Immediate           true                   84d
            storageclass.storage.k8s.io/ix-storage-class-metallb          zfs.csi.openebs.io   Retain          Immediate           true                   65d
            storageclass.storage.k8s.io/ix-storage-class-nextcloud        zfs.csi.openebs.io   Retain          Immediate           true                   65d
            storageclass.storage.k8s.io/openebs-zfspv-default (default)   zfs.csi.openebs.io   Delete          Immediate           true                   127m

NAMESPACE      NAME                                              STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS                 AGE   VOLUMEMODE
ix-nextcloud   persistentvolumeclaim/db-nextcloud-postgresql-0   Bound    pvc-40eb9fc7-af7c-431d-8bd5-6396b8141470   999Gi      RWO            ix-storage-class-nextcloud   65d   Filesystem
ix-nextcloud   persistentvolumeclaim/data-nextcloud-redis-0      Bound    pvc-b0400a3d-0295-47fb-87e2-4e21540bbb6f   999Gi      RWO            ix-storage-class-nextcloud   65d   Filesystem
ix-nextcloud   persistentvolumeclaim/nextcloud-data              Bound    pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586   256Gi      RWO            ix-storage-class-nextcloud   65d   Filesystem
ix-nextcloud   persistentvolumeclaim/nextcloud-html              Bound    pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68   256Gi      RWO            ix-storage-class-nextcloud   65d   Filesystem
ix-ubiquiti    persistentvolumeclaim/ubiquiti-unifi-config       Bound    pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96   999Gi      RWO            ix-storage-class-ubiquiti    84d   Filesystem

NAMESPACE   NAME                            STATUS   AGE
            namespace/default               Active   318d
            namespace/kube-system           Active   318d
            namespace/kube-public           Active   318d
            namespace/kube-node-lease       Active   318d
            namespace/openebs               Active   318d
            namespace/ix-plex               Active   193d
            namespace/ix-ubiquiti           Active   84d
            namespace/cnpg-system           Active   65d
            namespace/ix-metallb            Active   65d
            namespace/ix-nextcloud          Active   65d
            namespace/metallb-system        Active   69d
            namespace/prometheus-operator   Active   69d
            namespace/tc-system             Active   69d

NAMESPACE      NAME                                   COMPLETIONS   DURATION   AGE     CONTAINERS                 IMAGES                                                                                                             SELECTOR
ix-metallb     job.batch/metallb-manifests            1/1           4m39s      4d19h   metallb-manifests          tccr.io/truecharts/kubectl:v1.26.0@sha256:6d6e0e50f28b961ed1c1c6a9c140553238641591fbdc9ac7c1a348636f78c552         controller-uid=14a05d27-94f7-4ac8-80ad-80552718e0fd
ix-ubiquiti    job.batch/ubiquiti-unifi-manifests     1/1           6m32s      4d19h   ubiquiti-unifi-manifests   tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f   controller-uid=764e03cc-34bf-429d-b885-2a6d0c4a36ae
ix-nextcloud   job.batch/nextcloud-cronjob-27900840   1/1           6s         8m30s   nextcloud                  tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f    controller-uid=895eb55e-5658-45c8-86ec-e5a38f2bbc39
ix-nextcloud   job.batch/nextcloud-cronjob-27900845   1/1           6s         3m30s   nextcloud                  tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f    controller-uid=9fba7890-87f0-49dc-af2b-02676bf350f8
debug finished in 0 seconds for k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide


+--------------------------------------------------------------------------------+
+         k3s kubectl describe deployments --all-namespaces @1674050910          +
+--------------------------------------------------------------------------------+
Name:               plex
Namespace:          ix-plex
CreationTimestamp:  Fri, 08 Jul 2022 08:02:10 -0700
Labels:             app.kubernetes.io/instance=plex
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=plex
                    app.kubernetes.io/version=1.29.2.636
                    helm-revision=38
                    helm.sh/chart=plex-11.0.3
Annotations:        deployment.kubernetes.io/revision: 26
                    meta.helm.sh/release-name: plex
                    meta.helm.sh/release-namespace: ix-plex
                    rollme: ffroR
Selector:           app.kubernetes.io/instance=plex,app.kubernetes.io/name=plex
Replicas:           0 desired | 0 updated | 0 total | 0 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:           app.kubernetes.io/instance=plex
                    app.kubernetes.io/name=plex
  Annotations:      k8s.v1.cni.cncf.io/networks: ix-plex-0
  Service Account:  default
  Init Containers:
   prepare:
    Image:      tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      echo "Automatically correcting permissions for /config..."
      /usr/bin/nfs4xdr_winacl -a chown -G 568 -r -c '/config' -p '/config' || echo "Failed setting permissions..."
      
      EOF
      
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /config from config (rw)
  Containers:
   plex:
    Image:      tccr.io/truecharts/plex:1.29.2.6364@sha256:16eac4e90b6873107332cd15bdd2e84eea4479adafbcb8032585c180b22b28c9
    Port:       32400/TCP
    Host Port:  0/TCP
    Limits:
      cpu:             4
      memory:          8Gi
      nvidia.com/gpu:  1
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   tcp-socket :32400 delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  tcp-socket :32400 delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    tcp-socket :32400 delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                       2
      UMASK_SET:                   2
      S6_READ_ONLY_ROOT:           1
      NVIDIA_DRIVER_CAPABILITIES:  all
      TZ:                          America/New_York
      ADVERTISE_IP:                http://10.5.5.46:32400
      ALLOWED_NETWORKS:            172.16.0.0/12,10.0.0.0/8,192.168.0.0/16,10.5.5.0/24
      PLEX_CLAIM:                  claim-7FTxsz14R9ywgUFmPHns
      PLEX_PREFERENCE_GDM:         GdmEnabled=0
    Mounts:
      /config from config (rw)
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
  Volumes:
   config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/PLEX_Storage
    HostPathType:  
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   plex-794f5465f6 (0/0 replicas created)
Events:          <none>


Name:                   coredns
Namespace:              kube-system
CreationTimestamp:      Sat, 05 Mar 2022 12:59:47 -0800
Labels:                 k8s-app=kube-dns
                        kubernetes.io/name=CoreDNS
                        objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:            deployment.kubernetes.io/revision: 2
                        objectset.rio.cattle.io/applied:
                          H4sIAAAAAAAA/6xVUW/bNhD+K8M9S7GVtI0rYA+dna1FV8+ok74URkFTJ4szxePIkxMj0H8fTrIdu02TdtiTZfK743cfvzveg/LmE4ZoyEEOyvs42GSQwNq4AnKYoLe0rdExJFAjq0...
                        objectset.rio.cattle.io/id: 
                        objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                        objectset.rio.cattle.io/owner-name: coredns
                        objectset.rio.cattle.io/owner-namespace: kube-system
Selector:               k8s-app=kube-dns
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  1 max unavailable, 25% max surge
Pod Template:
  Labels:           k8s-app=kube-dns
  Service Account:  coredns
  Containers:
   coredns:
    Image:       rancher/mirrored-coredns-coredns:1.9.1
    Ports:       53/UDP, 53/TCP, 9153/TCP
    Host Ports:  0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
  Volumes:
   config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
   custom-config-volume:
    Type:                       ConfigMap (a volume populated by a ConfigMap)
    Name:                       coredns-custom
    Optional:                   true
  Topology Spread Constraints:  kubernetes.io/hostname:DoNotSchedule when max skew 1 is exceeded for selector k8s-app=kube-dns
  Priority Class Name:          system-cluster-critical
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   coredns-d76bd69b (1/1 replicas created)
Events:          <none>


Name:                   cnpg-controller-manager
Namespace:              cnpg-system
CreationTimestamp:      Sun, 13 Nov 2022 11:09:28 -0800
Labels:                 app.kubernetes.io/name=cloudnative-pg
Annotations:            deployment.kubernetes.io/revision: 1
Selector:               app.kubernetes.io/name=cloudnative-pg
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app.kubernetes.io/name=cloudnative-pg
  Service Account:  cnpg-manager
  Containers:
   manager:
    Image:       ghcr.io/cloudnative-pg/cloudnative-pg:1.18.0
    Ports:       8080/TCP, 9443/TCP
    Host Ports:  0/TCP, 0/TCP
    Command:
      /manager
    Args:
      controller
      --leader-elect
      --config-map-name=cnpg-controller-manager-config
      --secret-name=cnpg-controller-manager-config
      --webhook-port=9443
    Limits:
      cpu:     100m
      memory:  200Mi
    Requests:
      cpu:      100m
      memory:   100Mi
    Liveness:   http-get https://:9443/readyz delay=0s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get https://:9443/readyz delay=0s timeout=1s period=10s #success=1 #failure=3
    Environment:
      OPERATOR_IMAGE_NAME:           ghcr.io/cloudnative-pg/cloudnative-pg:1.18.0
      OPERATOR_NAMESPACE:             (v1:metadata.namespace)
      MONITORING_QUERIES_CONFIGMAP:  cnpg-default-monitoring
    Mounts:
      /controller from scratch-data (rw)
      /run/secrets/cnpg.io/webhook from webhook-certificates (rw)
  Volumes:
   scratch-data:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   webhook-certificates:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  cnpg-webhook-cert
    Optional:    true
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   cnpg-controller-manager-854876b995 (1/1 replicas created)
Events:          <none>


Name:                   prometheus-operator
Namespace:              prometheus-operator
CreationTimestamp:      Fri, 11 Nov 2022 14:18:22 -0800
Labels:                 app.kubernetes.io/component=controller
                        app.kubernetes.io/name=prometheus-operator
                        app.kubernetes.io/version=0.60.1
Annotations:            deployment.kubernetes.io/revision: 1
Selector:               app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app.kubernetes.io/component=controller
                    app.kubernetes.io/name=prometheus-operator
                    app.kubernetes.io/version=0.60.1
  Annotations:      kubectl.kubernetes.io/default-container: prometheus-operator
  Service Account:  prometheus-operator
  Containers:
   prometheus-operator:
    Image:      quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Port:       8080/TCP
    Host Port:  0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:       <none>
  Volumes:        <none>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   prometheus-operator-5c7445d877 (1/1 replicas created)
Events:          <none>


Name:                   controller
Namespace:              metallb-system
CreationTimestamp:      Wed, 09 Nov 2022 12:11:04 -0800
Labels:                 app=metallb
                        component=controller
Annotations:            deployment.kubernetes.io/revision: 1
Selector:               app=metallb,component=controller
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app=metallb
                    component=controller
  Annotations:      prometheus.io/port: 7472
                    prometheus.io/scrape: true
  Service Account:  controller
  Containers:
   controller:
    Image:       quay.io/metallb/controller:v0.13.7
    Ports:       7472/TCP, 9443/TCP
    Host Ports:  0/TCP, 0/TCP
    Args:
      --port=7472
      --log-level=info
    Liveness:   http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_ML_SECRET_NAME:  memberlist
      METALLB_DEPLOYMENT:      controller
    Mounts:
      /tmp/k8s-webhook-server/serving-certs from cert (ro)
  Volumes:
   cert:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  webhook-server-cert
    Optional:    false
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   controller-7597dd4f7b (1/1 replicas created)
Events:          <none>


Name:               ubiquiti-unifi
Namespace:          ix-ubiquiti
CreationTimestamp:  Wed, 26 Oct 2022 05:53:23 -0700
Labels:             app.kubernetes.io/instance=ubiquiti
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=unifi
                    app.kubernetes.io/version=7.2.95
                    helm-revision=20
                    helm.sh/chart=unifi-11.0.49
Annotations:        deployment.kubernetes.io/revision: 2
                    meta.helm.sh/release-name: ubiquiti
                    meta.helm.sh/release-namespace: ix-ubiquiti
                    rollme: hL9JG
Selector:           app.kubernetes.io/instance=ubiquiti,app.kubernetes.io/name=unifi
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:           app.kubernetes.io/instance=ubiquiti
                    app.kubernetes.io/name=unifi
  Service Account:  default
  Init Containers:
   prepare:
    Image:      tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      
      EOF
      
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:       <none>
  Containers:
   ubiquiti-unifi:
    Image:       tccr.io/truecharts/unifi:7.2.95@sha256:1d8afe976c4f081274295f0ba06521162341375647883add244db6a6508df0ec
    Ports:       8080/TCP, 8880/TCP, 8843/TCP, 8443/TCP, 6789/TCP, 3478/UDP
    Host Ports:  0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/UDP
    Limits:
      cpu:             4
      memory:          8Gi
      nvidia.com/gpu:  0
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   tcp-socket :8443 delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  tcp-socket :8443 delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    tcp-socket :8443 delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                       2
      UMASK_SET:                   2
      S6_READ_ONLY_ROOT:           1
      NVIDIA_DRIVER_CAPABILITIES:  all
      TZ:                          America/New_York
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /unifi from config (rw)
      /var/logs from varlogs (rw)
  Volumes:
   config:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  ubiquiti-unifi-config
    ReadOnly:   false
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   ubiquiti-unifi-68fbf8dd6d (1/1 replicas created)
Events:          <none>


Name:               nextcloud
Namespace:          ix-nextcloud
CreationTimestamp:  Sun, 13 Nov 2022 17:54:14 -0800
Labels:             app.kubernetes.io/instance=nextcloud
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=nextcloud
                    app.kubernetes.io/version=25.0.1
                    helm-revision=10
                    helm.sh/chart=nextcloud-19.0.4
Annotations:        deployment.kubernetes.io/revision: 3
                    meta.helm.sh/release-name: nextcloud
                    meta.helm.sh/release-namespace: ix-nextcloud
                    rollme: NtVTW
Selector:           app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=nextcloud
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:           app.kubernetes.io/instance=nextcloud
                    app.kubernetes.io/name=nextcloud
  Service Account:  default
  Init Containers:
   prepare:
    Image:      tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      until
        pg_isready -U nextcloud -h nextcloud-postgresql
        do sleep 2
      done
      [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD";
      export LIVE=false;
      until "$LIVE";
      do
        response=$(
            timeout -s 3 2 \
            redis-cli \
              -h "$REDIS_HOST" \
              -p "$REDIS_PORT" \
              ping
          )
        if [ "$response" == "PONG" ] || [ "$response" == "LOADING Redis is loading the dataset in memory" ]; then
          LIVE=true
          echo "$response"
          echo "Redis Responded, ending initcontainer and starting main container(s)..."
        else
          echo "$response"
          echo "Redis not responding... Sleeping for 10 sec..."
          sleep 10
        fi;
      done
      
      EOF
      
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:     10m
      memory:  50Mi
    Environment:
      REDIS_HOST:      <set to the key 'plainhost' in secret 'rediscreds'>       Optional: false
      REDIS_PASSWORD:  <set to the key 'redis-password' in secret 'rediscreds'>  Optional: false
      REDIS_PORT:      6379
    Mounts:            <none>
   prestart:
    Image:      tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Forcing permissions on userdata folder..."
      if nfs4xdr_getfacl && nfs4xdr_getfacl | grep -qv "Failed to get NFSv4 ACL"; then
        echo "NFSv4 ACLs detected, Trying to override permissions using nfs4_setfacl..."
        nfs4_setfacl -R -a A:g:33:RWX "/var/www/html/data"
      else
        echo "No NFSv4 ACLs detected, trying to override permissions using chown/chmod..."
        echo "checking ownership..."
        if [ $(stat -c %g .) -eq 33 ]; then
          echo "Ownership already set to 33, skipping..."
        else
          echo "Changing ownership to group 33..."
          chown -R :33 "/var/www/html/data"
        fi
        chmod 770 /var/www/html/data
      fi
      EOF
      
    Environment:  <none>
    Mounts:
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
  Containers:
   nextcloud:
    Image:       tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Ports:       9000/TCP, 7867/TCP, 7868/TCP, 9090/TCP, 8080/TCP
    Host Ports:  0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP
    Limits:
      cpu:             4
      memory:          8Gi
      nvidia.com/gpu:  0
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   http-get http://:8080/status.php delay=25s timeout=5s period=10s #success=1 #failure=5
    Readiness:  http-get http://:8080/status.php delay=25s timeout=5s period=10s #success=1 #failure=5
    Startup:    http-get http://:8080/status.php delay=25s timeout=2s period=5s #success=1 #failure=60
    Environment Variables from:
      nextcloud        Secret     Optional: false
      nextcloudconfig  ConfigMap  Optional: false
    Environment:
      PUID:                                 568
      USER_ID:                              568
      UID:                                  568
      UMASK:                                2
      UMASK_SET:                            2
      PGID:                                 33
      GROUP_ID:                             33
      GID:                                  33
      NVIDIA_DRIVER_CAPABILITIES:           all
      TZ:                                   America/Los_Angeles
      AccessIP:                             10.5.5.10
      NC_check_data_directory_permissions:  true
      NEXTCLOUD_CHUNKSIZE:                  31457280
      NEXTCLOUD_INIT_HTACCESS:              true
      PHP_MEMORY_LIMIT:                     1G
      PHP_UPLOAD_LIMIT:                     10G
      POSTGRES_DB:                          nextcloud
      POSTGRES_HOST:                        <set to the key 'plainporthost' in secret 'dbcreds'>        Optional: false
      POSTGRES_PASSWORD:                    <set to the key 'postgresql-password' in secret 'dbcreds'>  Optional: false
      POSTGRES_USER:                        nextcloud
      REDIS_HOST:                           <set to the key 'plainhost' in secret 'rediscreds'>       Optional: false
      REDIS_HOST_PASSWORD:                  <set to the key 'redis-password' in secret 'rediscreds'>  Optional: false
      TRUSTED_PROXIES:                      172.16.0.0/16 127.0.0.1
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
   hpb:
    Image:      tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:       7867/TCP
    Host Port:  0/TCP
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      set -m
      echo "Waiting for notify_push file to be available..."
      until [ -f /var/www/html/custom_apps/notify_push/bin/x86_64/notify_push ]
      do
           sleep 10
           echo "Notify_push app not found... waiting..."
      done
      echo "Waiting for Nextcloud to start..."
      until $(curl --output /dev/null --silent --head --fail -H "Host: test.fakedomain.dns" http://127.0.0.1:8080/status.php); do
          echo "Nextcloud not responding... waiting..."
          sleep 10
      done
      until $(curl --silent --fail -H "Host: test.fakedomain.dns" http://127.0.0.1:8080/status.php | jq --raw-output '.installed' | grep "true"); do
          echo "Nextcloud not installed... waiting..."
          sleep 10
      done
      echo "Nextcloud instance with Notify_push found... Launching High Performance Backend..."
      /var/www/html/custom_apps/notify_push/bin/x86_64/notify_push /var/www/html/config/config.php &
      
      echo  "Configuring CLI url..."
      php /var/www/html/occ config:system:set overwrite.cli.url --value='http://10.5.5.10:10020/'
      echo  "Imaginary High Performance Previews enabled, enabling it on Nextcloud..."
      php /var/www/html/occ config:system:set preview_imaginary_url --value='http://127.0.0.1:9090'
      php /var/www/html/occ config:system:set preview_max_x --value='2048'
      php /var/www/html/occ config:system:set preview_max_y --value='2048'
      php /var/www/html/occ config:system:set preview_max_memory --value='512'
      php /var/www/html/occ config:system:set preview_max_filesize_image --value='150'
      # Remove all preview providers and re-add only selected
      php /var/www/html/occ config:system:delete enabledPreviewProviders
      # Add imaginary always
       # Initialize counter
      php /var/www/html/occ config:system:set enabledPreviewProviders 0 --value='OC\Preview\Imaginary'
      php /var/www/html/occ config:system:set enabledPreviewProviders 1 --value='OC\Preview\PNG'
      php /var/www/html/occ config:system:set enabledPreviewProviders 2 --value='OC\Preview\JPEG'
      php /var/www/html/occ config:system:set enabledPreviewProviders 3 --value='OC\Preview\GIF'
      php /var/www/html/occ config:system:set enabledPreviewProviders 4 --value='OC\Preview\BMP'
      php /var/www/html/occ config:system:set enabledPreviewProviders 5 --value='OC\Preview\XBitmap'
      php /var/www/html/occ config:system:set enabledPreviewProviders 6 --value='OC\Preview\MP3'
      php /var/www/html/occ config:system:set enabledPreviewProviders 7 --value='OC\Preview\MarkDown'
      php /var/www/html/occ config:system:set enabledPreviewProviders 8 --value='OC\Preview\OpenDocument'
      php /var/www/html/occ config:system:set enabledPreviewProviders 9 --value='OC\Preview\TXT'
      php /var/www/html/occ config:system:set enabledPreviewProviders 10 --value='OC\Preview\Krita'
      php /var/www/html/occ config:system:set enabledPreviewProviders 11 --value='OC\Preview\Illustrator'
      php /var/www/html/occ config:system:set enabledPreviewProviders 12 --value='OC\Preview\HEIC'
      php /var/www/html/occ config:system:set enabledPreviewProviders 13 --value='OC\Preview\Movie'
      
      
      
      php /var/www/html/occ config:system:set enabledPreviewProviders 14 --value='OC\Preview\PDF'
      php /var/www/html/occ config:system:set enabledPreviewProviders 15 --value='OC\Preview\Photoshop'
      
      
      
      php /var/www/html/occ config:system:set enabledPreviewProviders 16 --value='OC\Preview\TIFF'
      
      
      # Set default phone region
      php /var/www/html/occ config:system:set default_phone_region --value='US'
      
      echo  "Configuring High Performance Backend for url: http://10.5.5.10:10020"
      php /var/www/html/occ config:app:set notify_push base_endpoint --value='http://10.5.5.10:10020/push'
      fg
      EOF
      
    Liveness:   http-get http://:7867/push/test/cookie delay=25s timeout=5s period=10s #success=1 #failure=5
    Readiness:  http-get http://:7867/push/test/cookie delay=25s timeout=5s period=10s #success=1 #failure=5
    Startup:    http-get http://:7867/push/test/cookie delay=25s timeout=2s period=5s #success=1 #failure=60
    Environment Variables from:
      nextcloudconfig  ConfigMap  Optional: false
    Environment:
      NEXTCLOUD_URL:        http://127.0.0.1:8080
      METRICS_PORT:         7868
      TRUSTED_PROXIES:      172.16.0.0/16 127.0.0.1
      POSTGRES_DB:          nextcloud
      POSTGRES_USER:        nextcloud
      POSTGRES_PASSWORD:    <set to the key 'postgresql-password' in secret 'dbcreds'>  Optional: false
      POSTGRES_HOST:        <set to the key 'plainporthost' in secret 'dbcreds'>        Optional: false
      REDIS_HOST:           <set to the key 'plainhost' in secret 'rediscreds'>         Optional: false
      REDIS_HOST_PASSWORD:  <set to the key 'redis-password' in secret 'rediscreds'>    Optional: false
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
   imaginary:
    Image:      h2non/imaginary:1.2.4@sha256:7facb4221047a5e79b9e902f380247f4e5bf4376400d0badbeb738d3e1c2f654
    Port:       9090/TCP
    Host Port:  0/TCP
    Args:
      -enable-url-source
    Liveness:   http-get http://:9090/ delay=25s timeout=5s period=10s #success=1 #failure=5
    Readiness:  http-get http://:9090/ delay=25s timeout=5s period=10s #success=1 #failure=5
    Startup:    http-get http://:9090/ delay=25s timeout=2s period=5s #success=1 #failure=60
    Environment:
      PORT:  9090
    Mounts:  <none>
   nginx:
    Image:        tccr.io/truecharts/nginx-unprivileged:1.23.2@sha256:86064448a2764926ca215714b36a80561bcfed064c435245cee641f4ce14768b
    Port:         8080/TCP
    Host Port:    0/TCP
    Liveness:     http-get http://:8080/robots.txt delay=25s timeout=5s period=10s #success=1 #failure=5
    Readiness:    http-get http://:8080/robots.txt delay=25s timeout=5s period=10s #success=1 #failure=5
    Startup:      http-get http://:8080/robots.txt delay=25s timeout=2s period=5s #success=1 #failure=60
    Environment:  <none>
    Mounts:
      /dev/shm from shm (rw)
      /etc/nginx/nginx.conf from nginx (ro,path="nginx.conf")
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
  Volumes:
   cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-data
    ReadOnly:   false
   html:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-html
    ReadOnly:   false
   nginx:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-nginx
    Optional:  false
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varrun:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   nextcloud-785ff76589 (1/1 replicas created)
Events:          <none>
debug finished in 0 seconds for k3s kubectl describe deployments --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe pods --all-namespaces @1674050910             +
+--------------------------------------------------------------------------------+
Name:             cnpg-controller-manager-854876b995-wxd8f
Namespace:        cnpg-system
Priority:         0
Service Account:  cnpg-manager
Node:             ix-truenas/10.5.5.10
Start Time:       Tue, 17 Jan 2023 10:58:17 -0800
Labels:           app.kubernetes.io/name=cloudnative-pg
                  pod-template-hash=854876b995
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.4.156"
                        ],
                        "mac": "0a:82:0e:39:16:1a",
                        "default": true,
                        "dns": {}
                    }]
Status:           Failed
Reason:           TaintToleration
Message:          Pod Predicate TaintToleration failed
IP:               
IPs:              <none>
Controlled By:    ReplicaSet/cnpg-controller-manager-854876b995
Containers:
  manager:
    Container ID:  docker://1a4f1a2b55884c80a1b14f96e079f8ec3b0f2f9bb17e11fa798b48b5cc63e50b
    Image:         ghcr.io/cloudnative-pg/cloudnative-pg:1.18.0
    Image ID:      docker-pullable://ghcr.io/cloudnative-pg/cloudnative-pg@sha256:0a11453b711c3d0a072a997df922af1ed9735709b3fce070227b0f957c0b5c6b
    Ports:         8080/TCP, 9443/TCP
    Host Ports:    0/TCP, 0/TCP
    Command:
      /manager
    Args:
      controller
      --leader-elect
      --config-map-name=cnpg-controller-manager-config
      --secret-name=cnpg-controller-manager-config
      --webhook-port=9443
    State:          Terminated
      Reason:       Error
      Exit Code:    255
      Started:      Tue, 17 Jan 2023 11:07:15 -0800
      Finished:     Wed, 18 Jan 2023 04:00:51 -0800
    Ready:          False
    Restart Count:  1
    Limits:
      cpu:     100m
      memory:  200Mi
    Requests:
      cpu:      100m
      memory:   100Mi
    Liveness:   http-get https://:9443/readyz delay=0s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get https://:9443/readyz delay=0s timeout=1s period=10s #success=1 #failure=3
    Environment:
      OPERATOR_IMAGE_NAME:           ghcr.io/cloudnative-pg/cloudnative-pg:1.18.0
      OPERATOR_NAMESPACE:            cnpg-system (v1:metadata.namespace)
      MONITORING_QUERIES_CONFIGMAP:  cnpg-default-monitoring
    Mounts:
      /controller from scratch-data (rw)
      /run/secrets/cnpg.io/webhook from webhook-certificates (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-lmk66 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  scratch-data:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  webhook-certificates:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  cnpg-webhook-cert
    Optional:    true
  kube-api-access-lmk66:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:                 coredns-d76bd69b-6zwqv
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      coredns
Node:                 ix-truenas/10.5.5.10
Start Time:           Tue, 17 Jan 2023 10:58:23 -0800
Labels:               k8s-app=kube-dns
                      pod-template-hash=d76bd69b
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.4.161"
                            ],
                            "mac": "0a:d3:f4:8d:b0:f3",
                            "default": true,
                            "dns": {}
                        }]
Status:               Failed
Reason:               TaintToleration
Message:              Pod Predicate TaintToleration failed
IP:                   
IPs:                  <none>
Controlled By:        ReplicaSet/coredns-d76bd69b
Containers:
  coredns:
    Container ID:  docker://6fa473975653f865a956462bc151397f7fb6de319befd52cf1fba1635cb30033
    Image:         rancher/mirrored-coredns-coredns:1.9.1
    Image ID:      docker://sha256:99376d8f35e0abb6ff9d66b50a7c81df6e6dfdb649becc5df84a691a7b4beca4
    Ports:         53/UDP, 53/TCP, 9153/TCP
    Host Ports:    0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    State:          Terminated
      Reason:       Error
      Exit Code:    255
      Started:      Tue, 17 Jan 2023 11:07:21 -0800
      Finished:     Wed, 18 Jan 2023 04:00:51 -0800
    Ready:          False
    Restart Count:  1
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5skhd (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
  custom-config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns-custom
    Optional:  true
  kube-api-access-5skhd:
    Type:                     Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:   3607
    ConfigMapName:            kube-root-ca.crt
    ConfigMapOptional:        <nil>
    DownwardAPI:              true
QoS Class:                    Burstable
Node-Selectors:               beta.kubernetes.io/os=linux
Tolerations:                  CriticalAddonsOnly op=Exists
                              node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                              node-role.kubernetes.io/master:NoSchedule op=Exists
                              node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                              node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Topology Spread Constraints:  kubernetes.io/hostname:DoNotSchedule when max skew 1 is exceeded for selector k8s-app=kube-dns
Events:                       <none>


Name:             prometheus-operator-5c7445d877-t9kml
Namespace:        prometheus-operator
Priority:         0
Service Account:  prometheus-operator
Node:             ix-truenas/10.5.5.10
Start Time:       Tue, 17 Jan 2023 10:58:27 -0800
Labels:           app.kubernetes.io/component=controller
                  app.kubernetes.io/name=prometheus-operator
                  app.kubernetes.io/version=0.60.1
                  pod-template-hash=5c7445d877
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.4.160"
                        ],
                        "mac": "8a:13:59:ab:e8:a4",
                        "default": true,
                        "dns": {}
                    }]
                  kubectl.kubernetes.io/default-container: prometheus-operator
Status:           Failed
Reason:           TaintToleration
Message:          Pod Predicate TaintToleration failed
IP:               
IPs:              <none>
Controlled By:    ReplicaSet/prometheus-operator-5c7445d877
Containers:
  prometheus-operator:
    Container ID:  docker://c21a3c4c5847475ddbd90d1f5bbfd9c34a7810ed0eb69356c75da1e357131cbc
    Image:         quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Image ID:      docker-pullable://quay.io/prometheus-operator/prometheus-operator@sha256:63d4d0f8fb817313ec13c2960b72d1b4341bea269eb97ec7bf44c585a04b1e2e
    Port:          8080/TCP
    Host Port:     0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    State:          Terminated
      Reason:       Error
      Exit Code:    255
      Started:      Tue, 17 Jan 2023 11:07:20 -0800
      Finished:     Wed, 18 Jan 2023 04:00:51 -0800
    Ready:          False
    Restart Count:  1
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-55jrh (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  kube-api-access-55jrh:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:                 openebs-zfs-node-wjj5j
Namespace:            kube-system
Priority:             900001000
Priority Class Name:  openebs-zfs-csi-node-critical
Service Account:      openebs-zfs-node-sa
Node:                 ix-truenas/10.5.5.10
Start Time:           Wed, 18 Jan 2023 04:01:12 -0800
Labels:               app=openebs-zfs-node
                      controller-revision-hash=57f5455f6b
                      openebs.io/component-name=openebs-zfs-node
                      openebs.io/version=ci
                      pod-template-generation=2
                      role=openebs-zfs
Annotations:          <none>
Status:               Running
IP:                   10.5.5.10
IPs:
  IP:           10.5.5.10
Controlled By:  DaemonSet/openebs-zfs-node
Containers:
  csi-node-driver-registrar:
    Container ID:  docker://7f88253c0fe0bf6a1247d3f338412384ae1d615fb2ea04631b74896acc9fbdba
    Image:         k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:f9bcee63734b7b01555ee8fc8fb01ac2922478b2c8934bf8d468dd2916edc405
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:14 -0800
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-knjlg (ro)
  openebs-zfs-plugin:
    Container ID:  docker://423f1252f14221ee6344625aa425159ab783d0a3d8cc4dcb5a6168b116daa395
    Image:         openebs/zfs-driver:2.0.0
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:472dc7ca43f9780246cc6d2d478ff4b08ead904d757dbb263f796fb613dbe04d
    Port:          <none>
    Host Port:     <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:15 -0800
    Ready:          True
    Restart Count:  0
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-knjlg (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
  encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
  chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
  host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
  plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
  pods-mount-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/
    HostPathType:  Directory
  kube-api-access-knjlg:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:                      <none>


Name:                 coredns-d76bd69b-plvk8
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      coredns
Node:                 ix-truenas/10.5.5.10
Start Time:           Wed, 18 Jan 2023 04:01:19 -0800
Labels:               k8s-app=kube-dns
                      pod-template-hash=d76bd69b
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.5.113"
                            ],
                            "mac": "3e:57:5b:d3:53:a4",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.5.113
IPs:
  IP:           172.16.5.113
Controlled By:  ReplicaSet/coredns-d76bd69b
Containers:
  coredns:
    Container ID:  docker://0234ca1bc77c8582ec4be91fa1cdb9b409d0c4a421666ccf32b0f85c7daf7677
    Image:         rancher/mirrored-coredns-coredns:1.9.1
    Image ID:      docker://sha256:99376d8f35e0abb6ff9d66b50a7c81df6e6dfdb649becc5df84a691a7b4beca4
    Ports:         53/UDP, 53/TCP, 9153/TCP
    Host Ports:    0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:21 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-sxdk7 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
  custom-config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns-custom
    Optional:  true
  kube-api-access-sxdk7:
    Type:                     Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:   3607
    ConfigMapName:            kube-root-ca.crt
    ConfigMapOptional:        <nil>
    DownwardAPI:              true
QoS Class:                    Burstable
Node-Selectors:               beta.kubernetes.io/os=linux
Tolerations:                  CriticalAddonsOnly op=Exists
                              node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                              node-role.kubernetes.io/master:NoSchedule op=Exists
                              node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                              node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Topology Spread Constraints:  kubernetes.io/hostname:DoNotSchedule when max skew 1 is exceeded for selector k8s-app=kube-dns
Events:                       <none>


Name:             cnpg-controller-manager-854876b995-ccsfw
Namespace:        cnpg-system
Priority:         0
Service Account:  cnpg-manager
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 04:01:18 -0800
Labels:           app.kubernetes.io/name=cloudnative-pg
                  pod-template-hash=854876b995
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.112"
                        ],
                        "mac": "3a:d6:0f:61:ce:5b",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.5.112
IPs:
  IP:           172.16.5.112
Controlled By:  ReplicaSet/cnpg-controller-manager-854876b995
Containers:
  manager:
    Container ID:  docker://1c852f226b2ab0b30620bd367cba301098970251845176c7383f5ac7f0fd02b8
    Image:         ghcr.io/cloudnative-pg/cloudnative-pg:1.18.0
    Image ID:      docker-pullable://ghcr.io/cloudnative-pg/cloudnative-pg@sha256:0a11453b711c3d0a072a997df922af1ed9735709b3fce070227b0f957c0b5c6b
    Ports:         8080/TCP, 9443/TCP
    Host Ports:    0/TCP, 0/TCP
    Command:
      /manager
    Args:
      controller
      --leader-elect
      --config-map-name=cnpg-controller-manager-config
      --secret-name=cnpg-controller-manager-config
      --webhook-port=9443
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:21 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     100m
      memory:  200Mi
    Requests:
      cpu:      100m
      memory:   100Mi
    Liveness:   http-get https://:9443/readyz delay=0s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get https://:9443/readyz delay=0s timeout=1s period=10s #success=1 #failure=3
    Environment:
      OPERATOR_IMAGE_NAME:           ghcr.io/cloudnative-pg/cloudnative-pg:1.18.0
      OPERATOR_NAMESPACE:            cnpg-system (v1:metadata.namespace)
      MONITORING_QUERIES_CONFIGMAP:  cnpg-default-monitoring
    Mounts:
      /controller from scratch-data (rw)
      /run/secrets/cnpg.io/webhook from webhook-certificates (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-89wvb (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  scratch-data:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  webhook-certificates:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  cnpg-webhook-cert
    Optional:    true
  kube-api-access-89wvb:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             prometheus-operator-5c7445d877-npglz
Namespace:        prometheus-operator
Priority:         0
Service Account:  prometheus-operator
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 04:01:24 -0800
Labels:           app.kubernetes.io/component=controller
                  app.kubernetes.io/name=prometheus-operator
                  app.kubernetes.io/version=0.60.1
                  pod-template-hash=5c7445d877
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.118"
                        ],
                        "mac": "c2:75:6f:f4:2b:fa",
                        "default": true,
                        "dns": {}
                    }]
                  kubectl.kubernetes.io/default-container: prometheus-operator
Status:           Running
IP:               172.16.5.118
IPs:
  IP:           172.16.5.118
Controlled By:  ReplicaSet/prometheus-operator-5c7445d877
Containers:
  prometheus-operator:
    Container ID:  docker://04951e5db1863c5909b8d1a482128910b01a4b8e87fafd6206f5f70908dc3421
    Image:         quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Image ID:      docker-pullable://quay.io/prometheus-operator/prometheus-operator@sha256:63d4d0f8fb817313ec13c2960b72d1b4341bea269eb97ec7bf44c585a04b1e2e
    Port:          8080/TCP
    Host Port:     0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:32 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-bs2xp (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  kube-api-access-bs2xp:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             speaker-rlx98
Namespace:        metallb-system
Priority:         0
Service Account:  speaker
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 04:01:12 -0800
Labels:           app=metallb
                  component=speaker
                  controller-revision-hash=5c6774c4f9
                  pod-template-generation=1
Annotations:      prometheus.io/port: 7472
                  prometheus.io/scrape: true
Status:           Running
IP:               10.5.5.10
IPs:
  IP:           10.5.5.10
Controlled By:  DaemonSet/speaker
Containers:
  speaker:
    Container ID:  docker://8905ce8bce6acf0c255895b483bc8f054ac76d89807f052b5f63e669b2e723a7
    Image:         quay.io/metallb/speaker:v0.13.7
    Image ID:      docker-pullable://quay.io/metallb/speaker@sha256:aeab5aeeb25c7ca31a1c468a35b1db4513cd6199bf376bdf19ff5689d2ebad28
    Ports:         7472/TCP, 7946/TCP, 7946/UDP
    Host Ports:    7472/TCP, 7946/TCP, 7946/UDP
    Args:
      --port=7472
      --log-level=info
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:15 -0800
    Ready:          True
    Restart Count:  0
    Liveness:       http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:      http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_NODE_NAME:       (v1:spec.nodeName)
      METALLB_HOST:            (v1:status.hostIP)
      METALLB_ML_BIND_ADDR:    (v1:status.podIP)
      METALLB_ML_LABELS:      app=metallb,component=speaker
      METALLB_ML_SECRET_KEY:  <set to the key 'secretkey' in secret 'memberlist'>  Optional: false
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-zmxt4 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  kube-api-access-zmxt4:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                             node-role.kubernetes.io/master:NoSchedule op=Exists
                             node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:                      <none>


Name:                 openebs-zfs-controller-0
Namespace:            kube-system
Priority:             900000000
Priority Class Name:  openebs-zfs-csi-controller-critical
Service Account:      openebs-zfs-controller-sa
Node:                 ix-truenas/10.5.5.10
Start Time:           Wed, 18 Jan 2023 04:01:31 -0800
Labels:               app=openebs-zfs-controller
                      controller-revision-hash=openebs-zfs-controller-698698d48b
                      openebs.io/component-name=openebs-zfs-controller
                      openebs.io/version=ci
                      role=openebs-zfs
                      statefulset.kubernetes.io/pod-name=openebs-zfs-controller-0
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.5.119"
                            ],
                            "mac": "fa:4a:b2:91:a8:94",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.5.119
IPs:
  IP:           172.16.5.119
Controlled By:  StatefulSet/openebs-zfs-controller
Containers:
  csi-resizer:
    Container ID:  docker://f0be549fa30a1d562bef5d08885ae4d5639272b9c2d4470970687882f588bb20
    Image:         k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:33 -0800
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mhg85 (ro)
  csi-snapshotter:
    Container ID:  docker://44d16be0523fc6f0e8e822469a04b5b1dec184626c8ee24b48e147ef6172ccca
    Image:         k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-snapshotter@sha256:51f2dfde5bccac7854b3704689506aeecfb793328427b91115ba253a93e60782
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:33 -0800
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mhg85 (ro)
  snapshot-controller:
    Container ID:  docker://86b3c7d895ad459a29d9b3b3720e382a68d459a27959c79a1018d7918ea56883
    Image:         k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/snapshot-controller@sha256:00fcc441ea9f72899c25eed61d602272a2a58c5f0014332bdcb5ac24acef08e4
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --leader-election=true
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:33 -0800
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mhg85 (ro)
  csi-provisioner:
    Container ID:  docker://aea6ab6cf1324976e64a70cd1ad455b93ad5f6b3bffd0b87a3db514dce568640
    Image:         k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-provisioner@sha256:6477988532358148d2e98f7c747db4e9250bbc7ad2664bf666348abf9ee1f5aa
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:33 -0800
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:  kube-system (v1:metadata.namespace)
      POD_NAME:   openebs-zfs-controller-0 (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mhg85 (ro)
  openebs-zfs-plugin:
    Container ID:  docker://4c5c27ec8d2cb9a3bd46049f09a86dae3cb9a5de8d73273a3f5ee04432d4805d
    Image:         openebs/zfs-driver:2.0.0
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:472dc7ca43f9780246cc6d2d478ff4b08ead904d757dbb263f796fb613dbe04d
    Port:          <none>
    Host Port:     <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:33 -0800
    Ready:          True
    Restart Count:  0
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-mhg85 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-mhg85:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             controller-7597dd4f7b-6dxpb
Namespace:        metallb-system
Priority:         0
Service Account:  controller
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 04:01:15 -0800
Labels:           app=metallb
                  component=controller
                  pod-template-hash=7597dd4f7b
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.111"
                        ],
                        "mac": "9e:78:01:96:ea:1d",
                        "default": true,
                        "dns": {}
                    }]
                  prometheus.io/port: 7472
                  prometheus.io/scrape: true
Status:           Running
IP:               172.16.5.111
IPs:
  IP:           172.16.5.111
Controlled By:  ReplicaSet/controller-7597dd4f7b
Containers:
  controller:
    Container ID:  docker://7b967504fa52e637a9359ddd9aad0dad885b9ccbd3e7c69181abbf028fbf112a
    Image:         quay.io/metallb/controller:v0.13.7
    Image ID:      docker-pullable://quay.io/metallb/controller@sha256:0e486995b27d62433e728f7b266fb8b7b7f4fd65e1b751bef17fe56e76653bf1
    Ports:         7472/TCP, 9443/TCP
    Host Ports:    0/TCP, 0/TCP
    Args:
      --port=7472
      --log-level=info
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:18 -0800
    Ready:          True
    Restart Count:  0
    Liveness:       http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:      http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_ML_SECRET_NAME:  memberlist
      METALLB_DEPLOYMENT:      controller
    Mounts:
      /tmp/k8s-webhook-server/serving-certs from cert (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-gtjrg (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  cert:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  webhook-server-cert
    Optional:    false
  kube-api-access-gtjrg:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             nextcloud-postgresql-0
Namespace:        ix-nextcloud
Priority:         0
Service Account:  default
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 04:01:21 -0800
Labels:           app.kubernetes.io/instance=nextcloud
                  app.kubernetes.io/name=postgresql
                  controller-revision-hash=nextcloud-postgresql-8948ff749
                  statefulset.kubernetes.io/pod-name=nextcloud-postgresql-0
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.115"
                        ],
                        "mac": "62:fb:6b:d4:84:8c",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.5.115
IPs:
  IP:           172.16.5.115
Controlled By:  StatefulSet/nextcloud-postgresql
Init Containers:
  prepare:
    Container ID:  docker://1eafa10871368e5799527a10e0a4f7faf7f05433f1d197c63d4556bc29a2aaad
    Image:         tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Image ID:      docker-pullable://tccr.io/truecharts/ubuntu@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      
      EOF
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Wed, 18 Jan 2023 04:01:24 -0800
      Finished:     Wed, 18 Jan 2023 04:01:24 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-2vn2x (ro)
Containers:
  nextcloud-postgresql:
    Container ID:   docker://73d7b3e9bea63b4628b064f18acd1784b2878541c544068865241ee9a99f2f47
    Image:          tccr.io/truecharts/postgresql:14.5.0@sha256:1d5c95be8ddb4182509773cfc4d75101adcbd1ae97a47011cd52123dff59c691
    Image ID:       docker-pullable://tccr.io/truecharts/postgresql@sha256:1d5c95be8ddb4182509773cfc4d75101adcbd1ae97a47011cd52123dff59c691
    Port:           5432/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:25 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   exec [sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2 ; done] delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  exec [sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2 ; done] delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    exec [sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2 ; done] delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                         2
      UMASK_SET:                     2
      S6_READ_ONLY_ROOT:             1
      NVIDIA_VISIBLE_DEVICES:        void
      TZ:                            UTC
      POSTGRESQL_POSTGRES_PASSWORD:  <set to the key 'postgresql-postgres-password' in secret 'dbcreds'>  Optional: false
      POSTGRES_DB:                   nextcloud
      POSTGRES_PASSWORD:             <set to the key 'postgresql-password' in secret 'dbcreds'>  Optional: false
      POSTGRES_USER:                 nextcloud
    Mounts:
      /bitnami/postgresql from db (rw)
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-2vn2x (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  db:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  db-nextcloud-postgresql-0
    ReadOnly:   false
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-2vn2x:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             ubiquiti-unifi-68fbf8dd6d-g5b4t
Namespace:        ix-ubiquiti
Priority:         0
Service Account:  default
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 04:01:26 -0800
Labels:           app.kubernetes.io/instance=ubiquiti
                  app.kubernetes.io/name=unifi
                  pod-template-hash=68fbf8dd6d
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.117"
                        ],
                        "mac": "2a:61:7f:dc:c7:0f",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.5.117
IPs:
  IP:           172.16.5.117
Controlled By:  ReplicaSet/ubiquiti-unifi-68fbf8dd6d
Init Containers:
  prepare:
    Container ID:  docker://dc8c8e55d9085eee93f18068aa3b3ce7f9defb9ee0572af01855c426fa07976d
    Image:         tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Image ID:      docker-pullable://tccr.io/truecharts/ubuntu@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      
      EOF
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Wed, 18 Jan 2023 04:01:30 -0800
      Finished:     Wed, 18 Jan 2023 04:01:30 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-229gj (ro)
Containers:
  ubiquiti-unifi:
    Container ID:   docker://14974236b05f9ed8be748e7f9c6fa3f47ecc7aa80e29df09f90aad8bc902d3db
    Image:          tccr.io/truecharts/unifi:7.2.95@sha256:1d8afe976c4f081274295f0ba06521162341375647883add244db6a6508df0ec
    Image ID:       docker-pullable://tccr.io/truecharts/unifi@sha256:1d8afe976c4f081274295f0ba06521162341375647883add244db6a6508df0ec
    Ports:          8080/TCP, 8880/TCP, 8843/TCP, 8443/TCP, 6789/TCP, 3478/UDP
    Host Ports:     0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/UDP
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:31 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:             4
      memory:          8Gi
      nvidia.com/gpu:  0
    Requests:
      cpu:             10m
      memory:          50Mi
      nvidia.com/gpu:  0
    Liveness:          tcp-socket :8443 delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:         tcp-socket :8443 delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:           tcp-socket :8443 delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                       2
      UMASK_SET:                   2
      S6_READ_ONLY_ROOT:           1
      NVIDIA_DRIVER_CAPABILITIES:  all
      TZ:                          America/New_York
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /unifi from config (rw)
      /var/logs from varlogs (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-229gj (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  ubiquiti-unifi-config
    ReadOnly:   false
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-229gj:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             nextcloud-redis-0
Namespace:        ix-nextcloud
Priority:         0
Service Account:  default
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 04:01:32 -0800
Labels:           app.kubernetes.io/instance=nextcloud
                  app.kubernetes.io/name=redis
                  controller-revision-hash=nextcloud-redis-5c659c888d
                  statefulset.kubernetes.io/pod-name=nextcloud-redis-0
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.120"
                        ],
                        "mac": "06:ca:32:07:18:4c",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.5.120
IPs:
  IP:           172.16.5.120
Controlled By:  StatefulSet/nextcloud-redis
Init Containers:
  prepare:
    Container ID:  docker://13a5059388d76b78a73829c995d0e7971693d8271c62712435f11f42c0de1ff1
    Image:         tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Image ID:      docker-pullable://tccr.io/truecharts/ubuntu@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      
      EOF
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Wed, 18 Jan 2023 04:01:35 -0800
      Finished:     Wed, 18 Jan 2023 04:01:35 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-w4jgs (ro)
Containers:
  nextcloud-redis:
    Container ID:   docker://e13ab6d3c5884b55fa16bd7cdac877c3e7dceb3e74cf00738647b7acf78f050c
    Image:          tccr.io/truecharts/redis:7.0.5@sha256:a0de36bdb830474aa89a387be8297edac22fc668cd01ad6831e1d3fa70b6b8df
    Image ID:       docker-pullable://tccr.io/truecharts/redis@sha256:a0de36bdb830474aa89a387be8297edac22fc668cd01ad6831e1d3fa70b6b8df
    Port:           6379/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:36 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   exec [sh -c /health/ping_liveness_local.sh 2] delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  exec [sh -c /health/ping_readiness_local.sh 2] delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    exec [sh -c /health/ping_readiness_local.sh 2] delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                   2
      UMASK_SET:               2
      S6_READ_ONLY_ROOT:       1
      NVIDIA_VISIBLE_DEVICES:  void
      TZ:                      UTC
      ALLOW_EMPTY_PASSWORD:    yes
      REDIS_PASSWORD:          <set to the key 'redis-password' in secret 'rediscreds'>  Optional: false
      REDIS_PORT:              6379
      REDIS_REPLICATION_MODE:  master
    Mounts:
      /bitnami/redis from data (rw)
      /dev/shm from shm (rw)
      /health from redis-health (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-w4jgs (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  data-nextcloud-redis-0
    ReadOnly:   false
  redis-health:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-redis-health
    Optional:  false
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-w4jgs:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             nextcloud-785ff76589-q8zfn
Namespace:        ix-nextcloud
Priority:         0
Service Account:  default
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 04:01:21 -0800
Labels:           app.kubernetes.io/instance=nextcloud
                  app.kubernetes.io/name=nextcloud
                  pod-template-hash=785ff76589
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.116"
                        ],
                        "mac": "42:b4:83:98:5c:d9",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.5.116
IPs:
  IP:           172.16.5.116
Controlled By:  ReplicaSet/nextcloud-785ff76589
Init Containers:
  prepare:
    Container ID:  docker://2122f696b0cd509eef3a1200ceec39161c9685dfca9ddcbd852fd186181fad23
    Image:         tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Image ID:      docker-pullable://tccr.io/truecharts/ubuntu@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      until
        pg_isready -U nextcloud -h nextcloud-postgresql
        do sleep 2
      done
      [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD";
      export LIVE=false;
      until "$LIVE";
      do
        response=$(
            timeout -s 3 2 \
            redis-cli \
              -h "$REDIS_HOST" \
              -p "$REDIS_PORT" \
              ping
          )
        if [ "$response" == "PONG" ] || [ "$response" == "LOADING Redis is loading the dataset in memory" ]; then
          LIVE=true
          echo "$response"
          echo "Redis Responded, ending initcontainer and starting main container(s)..."
        else
          echo "$response"
          echo "Redis not responding... Sleeping for 10 sec..."
          sleep 10
        fi;
      done
      
      EOF
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Wed, 18 Jan 2023 04:01:27 -0800
      Finished:     Wed, 18 Jan 2023 04:01:55 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:     10m
      memory:  50Mi
    Environment:
      REDIS_HOST:      <set to the key 'plainhost' in secret 'rediscreds'>       Optional: false
      REDIS_PASSWORD:  <set to the key 'redis-password' in secret 'rediscreds'>  Optional: false
      REDIS_PORT:      6379
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6g6lf (ro)
  prestart:
    Container ID:  docker://410efb6ac878942c8811d6869e62b2bd358e236f08bd91a898ebfa1fc7268bae
    Image:         tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Image ID:      docker-pullable://tccr.io/truecharts/nextcloud-fpm@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Forcing permissions on userdata folder..."
      if nfs4xdr_getfacl && nfs4xdr_getfacl | grep -qv "Failed to get NFSv4 ACL"; then
        echo "NFSv4 ACLs detected, Trying to override permissions using nfs4_setfacl..."
        nfs4_setfacl -R -a A:g:33:RWX "/var/www/html/data"
      else
        echo "No NFSv4 ACLs detected, trying to override permissions using chown/chmod..."
        echo "checking ownership..."
        if [ $(stat -c %g .) -eq 33 ]; then
          echo "Ownership already set to 33, skipping..."
        else
          echo "Changing ownership to group 33..."
          chown -R :33 "/var/www/html/data"
        fi
        chmod 770 /var/www/html/data
      fi
      EOF
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Wed, 18 Jan 2023 04:01:55 -0800
      Finished:     Wed, 18 Jan 2023 04:01:55 -0800
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6g6lf (ro)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
Containers:
  nextcloud:
    Container ID:   docker://4d83ce42e8b3ff64bbf00466c4d2d2c19075794b70c53f427a5342f55d72e6b3
    Image:          tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Image ID:       docker-pullable://tccr.io/truecharts/nextcloud-fpm@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Ports:          9000/TCP, 7867/TCP, 7868/TCP, 9090/TCP, 8080/TCP
    Host Ports:     0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:56 -0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:             4
      memory:          8Gi
      nvidia.com/gpu:  0
    Requests:
      cpu:             10m
      memory:          50Mi
      nvidia.com/gpu:  0
    Liveness:          http-get http://:8080/status.php delay=25s timeout=5s period=10s #success=1 #failure=5
    Readiness:         http-get http://:8080/status.php delay=25s timeout=5s period=10s #success=1 #failure=5
    Startup:           http-get http://:8080/status.php delay=25s timeout=2s period=5s #success=1 #failure=60
    Environment Variables from:
      nextcloud        Secret     Optional: false
      nextcloudconfig  ConfigMap  Optional: false
    Environment:
      PUID:                                 568
      USER_ID:                              568
      UID:                                  568
      UMASK:                                2
      UMASK_SET:                            2
      PGID:                                 33
      GROUP_ID:                             33
      GID:                                  33
      NVIDIA_DRIVER_CAPABILITIES:           all
      TZ:                                   America/Los_Angeles
      AccessIP:                             10.5.5.10
      NC_check_data_directory_permissions:  true
      NEXTCLOUD_CHUNKSIZE:                  31457280
      NEXTCLOUD_INIT_HTACCESS:              true
      PHP_MEMORY_LIMIT:                     1G
      PHP_UPLOAD_LIMIT:                     10G
      POSTGRES_DB:                          nextcloud
      POSTGRES_HOST:                        <set to the key 'plainporthost' in secret 'dbcreds'>        Optional: false
      POSTGRES_PASSWORD:                    <set to the key 'postgresql-password' in secret 'dbcreds'>  Optional: false
      POSTGRES_USER:                        nextcloud
      REDIS_HOST:                           <set to the key 'plainhost' in secret 'rediscreds'>       Optional: false
      REDIS_HOST_PASSWORD:                  <set to the key 'redis-password' in secret 'rediscreds'>  Optional: false
      TRUSTED_PROXIES:                      172.16.0.0/16 127.0.0.1
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6g6lf (ro)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
  hpb:
    Container ID:  docker://cd580a6442cc0fa38a6486a94e5fe7c4ef4820a83b01e98a6e6b983e827116a5
    Image:         tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Image ID:      docker-pullable://tccr.io/truecharts/nextcloud-fpm@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:          7867/TCP
    Host Port:     0/TCP
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      set -m
      echo "Waiting for notify_push file to be available..."
      until [ -f /var/www/html/custom_apps/notify_push/bin/x86_64/notify_push ]
      do
           sleep 10
           echo "Notify_push app not found... waiting..."
      done
      echo "Waiting for Nextcloud to start..."
      until $(curl --output /dev/null --silent --head --fail -H "Host: test.fakedomain.dns" http://127.0.0.1:8080/status.php); do
          echo "Nextcloud not responding... waiting..."
          sleep 10
      done
      until $(curl --silent --fail -H "Host: test.fakedomain.dns" http://127.0.0.1:8080/status.php | jq --raw-output '.installed' | grep "true"); do
          echo "Nextcloud not installed... waiting..."
          sleep 10
      done
      echo "Nextcloud instance with Notify_push found... Launching High Performance Backend..."
      /var/www/html/custom_apps/notify_push/bin/x86_64/notify_push /var/www/html/config/config.php &
      
      echo  "Configuring CLI url..."
      php /var/www/html/occ config:system:set overwrite.cli.url --value='http://10.5.5.10:10020/'
      echo  "Imaginary High Performance Previews enabled, enabling it on Nextcloud..."
      php /var/www/html/occ config:system:set preview_imaginary_url --value='http://127.0.0.1:9090'
      php /var/www/html/occ config:system:set preview_max_x --value='2048'
      php /var/www/html/occ config:system:set preview_max_y --value='2048'
      php /var/www/html/occ config:system:set preview_max_memory --value='512'
      php /var/www/html/occ config:system:set preview_max_filesize_image --value='150'
      # Remove all preview providers and re-add only selected
      php /var/www/html/occ config:system:delete enabledPreviewProviders
      # Add imaginary always
       # Initialize counter
      php /var/www/html/occ config:system:set enabledPreviewProviders 0 --value='OC\Preview\Imaginary'
      php /var/www/html/occ config:system:set enabledPreviewProviders 1 --value='OC\Preview\PNG'
      php /var/www/html/occ config:system:set enabledPreviewProviders 2 --value='OC\Preview\JPEG'
      php /var/www/html/occ config:system:set enabledPreviewProviders 3 --value='OC\Preview\GIF'
      php /var/www/html/occ config:system:set enabledPreviewProviders 4 --value='OC\Preview\BMP'
      php /var/www/html/occ config:system:set enabledPreviewProviders 5 --value='OC\Preview\XBitmap'
      php /var/www/html/occ config:system:set enabledPreviewProviders 6 --value='OC\Preview\MP3'
      php /var/www/html/occ config:system:set enabledPreviewProviders 7 --value='OC\Preview\MarkDown'
      php /var/www/html/occ config:system:set enabledPreviewProviders 8 --value='OC\Preview\OpenDocument'
      php /var/www/html/occ config:system:set enabledPreviewProviders 9 --value='OC\Preview\TXT'
      php /var/www/html/occ config:system:set enabledPreviewProviders 10 --value='OC\Preview\Krita'
      php /var/www/html/occ config:system:set enabledPreviewProviders 11 --value='OC\Preview\Illustrator'
      php /var/www/html/occ config:system:set enabledPreviewProviders 12 --value='OC\Preview\HEIC'
      php /var/www/html/occ config:system:set enabledPreviewProviders 13 --value='OC\Preview\Movie'
      
      
      
      php /var/www/html/occ config:system:set enabledPreviewProviders 14 --value='OC\Preview\PDF'
      php /var/www/html/occ config:system:set enabledPreviewProviders 15 --value='OC\Preview\Photoshop'
      
      
      
      php /var/www/html/occ config:system:set enabledPreviewProviders 16 --value='OC\Preview\TIFF'
      
      
      # Set default phone region
      php /var/www/html/occ config:system:set default_phone_region --value='US'
      
      echo  "Configuring High Performance Backend for url: http://10.5.5.10:10020"
      php /var/www/html/occ config:app:set notify_push base_endpoint --value='http://10.5.5.10:10020/push'
      fg
      EOF
      
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:56 -0800
    Ready:          True
    Restart Count:  0
    Liveness:       http-get http://:7867/push/test/cookie delay=25s timeout=5s period=10s #success=1 #failure=5
    Readiness:      http-get http://:7867/push/test/cookie delay=25s timeout=5s period=10s #success=1 #failure=5
    Startup:        http-get http://:7867/push/test/cookie delay=25s timeout=2s period=5s #success=1 #failure=60
    Environment Variables from:
      nextcloudconfig  ConfigMap  Optional: false
    Environment:
      NEXTCLOUD_URL:        http://127.0.0.1:8080
      METRICS_PORT:         7868
      TRUSTED_PROXIES:      172.16.0.0/16 127.0.0.1
      POSTGRES_DB:          nextcloud
      POSTGRES_USER:        nextcloud
      POSTGRES_PASSWORD:    <set to the key 'postgresql-password' in secret 'dbcreds'>  Optional: false
      POSTGRES_HOST:        <set to the key 'plainporthost' in secret 'dbcreds'>        Optional: false
      REDIS_HOST:           <set to the key 'plainhost' in secret 'rediscreds'>         Optional: false
      REDIS_HOST_PASSWORD:  <set to the key 'redis-password' in secret 'rediscreds'>    Optional: false
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6g6lf (ro)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
  imaginary:
    Container ID:  docker://83220dfff21046a359438f1368f261248820cb48849a9e710c59893831669990
    Image:         h2non/imaginary:1.2.4@sha256:7facb4221047a5e79b9e902f380247f4e5bf4376400d0badbeb738d3e1c2f654
    Image ID:      docker-pullable://h2non/imaginary@sha256:7facb4221047a5e79b9e902f380247f4e5bf4376400d0badbeb738d3e1c2f654
    Port:          9090/TCP
    Host Port:     0/TCP
    Args:
      -enable-url-source
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:57 -0800
    Ready:          True
    Restart Count:  0
    Liveness:       http-get http://:9090/ delay=25s timeout=5s period=10s #success=1 #failure=5
    Readiness:      http-get http://:9090/ delay=25s timeout=5s period=10s #success=1 #failure=5
    Startup:        http-get http://:9090/ delay=25s timeout=2s period=5s #success=1 #failure=60
    Environment:
      PORT:  9090
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6g6lf (ro)
  nginx:
    Container ID:   docker://766847d5fdb20804dd11d8e6b0bd38204a31dfc249d2b511a44d8b6f88fadbef
    Image:          tccr.io/truecharts/nginx-unprivileged:1.23.2@sha256:86064448a2764926ca215714b36a80561bcfed064c435245cee641f4ce14768b
    Image ID:       docker-pullable://tccr.io/truecharts/nginx-unprivileged@sha256:86064448a2764926ca215714b36a80561bcfed064c435245cee641f4ce14768b
    Port:           8080/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 04:01:57 -0800
    Ready:          True
    Restart Count:  0
    Liveness:       http-get http://:8080/robots.txt delay=25s timeout=5s period=10s #success=1 #failure=5
    Readiness:      http-get http://:8080/robots.txt delay=25s timeout=5s period=10s #success=1 #failure=5
    Startup:        http-get http://:8080/robots.txt delay=25s timeout=2s period=5s #success=1 #failure=60
    Environment:    <none>
    Mounts:
      /dev/shm from shm (rw)
      /etc/nginx/nginx.conf from nginx (ro,path="nginx.conf")
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6g6lf (ro)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-data
    ReadOnly:   false
  html:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-html
    ReadOnly:   false
  nginx:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-nginx
    Optional:  false
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varrun:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  kube-api-access-6g6lf:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             nextcloud-cronjob-27900840-9n8ws
Namespace:        ix-nextcloud
Priority:         0
Service Account:  default
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 06:00:00 -0800
Labels:           controller-uid=895eb55e-5658-45c8-86ec-e5a38f2bbc39
                  job-name=nextcloud-cronjob-27900840
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.144"
                        ],
                        "mac": "4a:38:2c:de:ba:30",
                        "default": true,
                        "dns": {}
                    }]
Status:           Succeeded
IP:               172.16.5.144
IPs:
  IP:           172.16.5.144
Controlled By:  Job/nextcloud-cronjob-27900840
Containers:
  nextcloud:
    Container ID:  docker://3a3cdf0476c3d18a2b1c5153fd94ef6dcb3d38e668626ce34c06becbc609642d
    Image:         tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Image ID:      docker-pullable://tccr.io/truecharts/nextcloud-fpm@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "running nextcloud cronjob..."
      php -f /var/www/html/cron.php
      echo "cronjob finished"
      echo "Pre-generating Previews..."
      php /var/www/html/occ preview:pre-generate
      echo "Previews generated."
      EOF
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Wed, 18 Jan 2023 06:00:01 -0800
      Finished:     Wed, 18 Jan 2023 06:00:02 -0800
    Ready:          False
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-s2dj6 (ro)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-data
    ReadOnly:   false
  html:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-html
    ReadOnly:   false
  nginx:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-nginx
    Optional:  false
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varrun:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  kube-api-access-s2dj6:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type    Reason          Age    From               Message
  ----    ------          ----   ----               -------
  Normal  Scheduled       8m31s  default-scheduler  Successfully assigned ix-nextcloud/nextcloud-cronjob-27900840-9n8ws to ix-truenas
  Normal  AddedInterface  8m30s  multus             Add eth0 [172.16.5.144/16] from ix-net
  Normal  Pulled          8m30s  kubelet            Container image "tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f" already present on machine
  Normal  Created         8m30s  kubelet            Created container nextcloud
  Normal  Started         8m30s  kubelet            Started container nextcloud


Name:                 nvidia-device-plugin-daemonset-55rxn
Namespace:            kube-system
Priority:             2000001000
Priority Class Name:  system-node-critical
Service Account:      default
Node:                 ix-truenas/10.5.5.10
Start Time:           Wed, 18 Jan 2023 04:01:12 -0800
Labels:               controller-revision-hash=77f95bfc79
                      name=nvidia-device-plugin-ds
                      pod-template-generation=2
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.5.110"
                            ],
                            "mac": "8a:a4:cc:9c:65:0f",
                            "default": true,
                            "dns": {}
                        }]
                      scheduler.alpha.kubernetes.io/critical-pod: 
Status:               Running
IP:                   172.16.5.110
IPs:
  IP:           172.16.5.110
Controlled By:  DaemonSet/nvidia-device-plugin-daemonset
Containers:
  nvidia-device-plugin-ctr:
    Container ID:  docker://557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f
    Image:         nvidia/k8s-device-plugin:v0.10.0
    Image ID:      docker-pullable://nvidia/k8s-device-plugin@sha256:5b967b3e92900797a74e0f7cd71005747fa3154503986676f372b9d82fe1d898
    Port:          <none>
    Host Port:     <none>
    State:         Waiting
      Reason:      CrashLoopBackOff
    Last State:    Terminated
      Reason:      ContainerCannotRun
      Message:     failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
      Exit Code:    128
      Started:      Wed, 18 Jan 2023 06:04:30 -0800
      Finished:     Wed, 18 Jan 2023 06:04:30 -0800
    Ready:          False
    Restart Count:  29
    Environment:    <none>
    Mounts:
      /var/lib/kubelet/device-plugins from device-plugin (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-4njt6 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  device-plugin:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/device-plugins
    HostPathType:  
  kube-api-access-4njt6:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 CriticalAddonsOnly op=Exists
                             node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
                             nvidia.com/gpu:NoSchedule op=Exists
Events:
  Type     Reason   Age                    From     Message
  ----     ------   ----                   ----     -------
  Warning  BackOff  2m6s (x562 over 127m)  kubelet  Back-off restarting failed container


Name:             nextcloud-cronjob-27900845-w579j
Namespace:        ix-nextcloud
Priority:         0
Service Account:  default
Node:             ix-truenas/10.5.5.10
Start Time:       Wed, 18 Jan 2023 06:05:00 -0800
Labels:           controller-uid=9fba7890-87f0-49dc-af2b-02676bf350f8
                  job-name=nextcloud-cronjob-27900845
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.5.145"
                        ],
                        "mac": "ea:2a:da:b8:09:24",
                        "default": true,
                        "dns": {}
                    }]
Status:           Succeeded
IP:               172.16.5.145
IPs:
  IP:           172.16.5.145
Controlled By:  Job/nextcloud-cronjob-27900845
Containers:
  nextcloud:
    Container ID:  docker://f907ee9140a4679c72684939e03594911d00fa2cb01c7fb1bc18f9ee8f408b17
    Image:         tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Image ID:      docker-pullable://tccr.io/truecharts/nextcloud-fpm@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "running nextcloud cronjob..."
      php -f /var/www/html/cron.php
      echo "cronjob finished"
      echo "Pre-generating Previews..."
      php /var/www/html/occ preview:pre-generate
      echo "Previews generated."
      EOF
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Wed, 18 Jan 2023 06:05:01 -0800
      Finished:     Wed, 18 Jan 2023 06:05:02 -0800
    Ready:          False
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-dx6cr (ro)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-data
    ReadOnly:   false
  html:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-html
    ReadOnly:   false
  nginx:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-nginx
    Optional:  false
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varrun:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  kube-api-access-dx6cr:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type    Reason          Age    From               Message
  ----    ------          ----   ----               -------
  Normal  Scheduled       3m31s  default-scheduler  Successfully assigned ix-nextcloud/nextcloud-cronjob-27900845-w579j to ix-truenas
  Normal  AddedInterface  3m30s  multus             Add eth0 [172.16.5.145/16] from ix-net
  Normal  Pulled          3m30s  kubelet            Container image "tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f" already present on machine
  Normal  Created         3m30s  kubelet            Created container nextcloud
  Normal  Started         3m30s  kubelet            Started container nextcloud
debug finished in 1 seconds for k3s kubectl describe pods --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe services --all-namespaces @1674050911           +
+--------------------------------------------------------------------------------+
Name:              kubernetes
Namespace:         default
Labels:            component=apiserver
                   provider=kubernetes
Annotations:       <none>
Selector:          <none>
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.1
IPs:               172.17.0.1
Port:              https  443/TCP
TargetPort:        6443/TCP
Endpoints:         10.5.5.10:6443
Session Affinity:  None
Events:            <none>


Name:              kube-dns
Namespace:         kube-system
Labels:            k8s-app=kube-dns
                   kubernetes.io/cluster-service=true
                   kubernetes.io/name=CoreDNS
                   objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:       objectset.rio.cattle.io/applied:
                     H4sIAAAAAAAA/4yRwW7UMBCGXwXN2Qlxstukljig9oKQUKUCF8TBcWZZk8S2PJMgtMq7I++mYqEK9JZoPn/6558T6GA/YyTrHSiYJQjoretAwSPG2RoEASOy7jRrUCfQznnWbL2j9O...
                   objectset.rio.cattle.io/id: 
                   objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                   objectset.rio.cattle.io/owner-name: coredns
                   objectset.rio.cattle.io/owner-namespace: kube-system
                   prometheus.io/port: 9153
                   prometheus.io/scrape: true
Selector:          k8s-app=kube-dns
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.10
IPs:               172.17.0.10
Port:              dns  53/UDP
TargetPort:        53/UDP
Endpoints:         172.16.5.113:53
Port:              dns-tcp  53/TCP
TargetPort:        53/TCP
Endpoints:         172.16.5.113:53
Port:              metrics  9153/TCP
TargetPort:        9153/TCP
Endpoints:         172.16.5.113:9153
Session Affinity:  None
Events:            <none>


Name:              cnpg-webhook-service
Namespace:         cnpg-system
Labels:            <none>
Annotations:       <none>
Selector:          app.kubernetes.io/name=cloudnative-pg
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.130.213
IPs:               172.17.130.213
Port:              <unset>  443/TCP
TargetPort:        9443/TCP
Endpoints:         172.16.5.112:9443
Session Affinity:  None
Events:            <none>


Name:              kubelet
Namespace:         kube-system
Labels:            app.kubernetes.io/managed-by=prometheus-operator
                   app.kubernetes.io/name=kubelet
                   k8s-app=kubelet
Annotations:       <none>
Selector:          <none>
Type:              ClusterIP
IP Family Policy:  RequireDualStack
IP Families:       IPv4,IPv6
IP:                None
IPs:               None
Port:              https-metrics  10250/TCP
TargetPort:        10250/TCP
Endpoints:         10.5.5.10:10250
Port:              http-metrics  10255/TCP
TargetPort:        10255/TCP
Endpoints:         10.5.5.10:10255
Port:              cadvisor  4194/TCP
TargetPort:        4194/TCP
Endpoints:         10.5.5.10:4194
Session Affinity:  None
Events:            <none>


Name:              nextcloud-postgresql
Namespace:         ix-nextcloud
Labels:            app.kubernetes.io/instance=nextcloud
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=postgresql
                   app.kubernetes.io/version=14.5.0
                   helm-revision=10
                   helm.sh/chart=postgresql-10.0.2
Annotations:       meta.helm.sh/release-name: nextcloud
                   meta.helm.sh/release-namespace: ix-nextcloud
Selector:          app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=postgresql
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.25.24
IPs:               172.17.25.24
Port:              main  5432/TCP
TargetPort:        5432/TCP
Endpoints:         172.16.5.115:5432
Session Affinity:  None
Events:            <none>


Name:              nextcloud-redis
Namespace:         ix-nextcloud
Labels:            app.kubernetes.io/instance=nextcloud
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=redis
                   app.kubernetes.io/version=7.0.5
                   helm-revision=10
                   helm.sh/chart=redis-5.0.2
Annotations:       meta.helm.sh/release-name: nextcloud
                   meta.helm.sh/release-namespace: ix-nextcloud
Selector:          app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=redis
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.149.206
IPs:               172.17.149.206
Port:              main  6379/TCP
TargetPort:        6379/TCP
Endpoints:         172.16.5.120:6379
Session Affinity:  None
Events:            <none>


Name:              nextcloud-backend
Namespace:         ix-nextcloud
Labels:            app.kubernetes.io/instance=nextcloud
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=nextcloud
                   app.kubernetes.io/version=25.0.1
                   helm-revision=10
                   helm.sh/chart=nextcloud-19.0.4
Annotations:       meta.helm.sh/release-name: nextcloud
                   meta.helm.sh/release-namespace: ix-nextcloud
Selector:          app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=nextcloud
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.236.123
IPs:               172.17.236.123
Port:              fpm  9000/TCP
TargetPort:        9000/TCP
Endpoints:         172.16.5.116:9000
Port:              hpb  7867/TCP
TargetPort:        7867/TCP
Endpoints:         172.16.5.116:7867
Port:              hpb-metrics  7868/TCP
TargetPort:        7868/TCP
Endpoints:         172.16.5.116:7868
Port:              imaginary  9090/TCP
TargetPort:        9090/TCP
Endpoints:         172.16.5.116:9090
Session Affinity:  None
Events:            <none>


Name:              webhook-service
Namespace:         metallb-system
Labels:            <none>
Annotations:       <none>
Selector:          component=controller
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.80.175
IPs:               172.17.80.175
Port:              <unset>  443/TCP
TargetPort:        9443/TCP
Endpoints:         172.16.5.111:9443
Session Affinity:  None
Events:            <none>


Name:              prometheus-operator
Namespace:         prometheus-operator
Labels:            app.kubernetes.io/component=controller
                   app.kubernetes.io/name=prometheus-operator
                   app.kubernetes.io/version=0.60.1
Annotations:       <none>
Selector:          app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                None
IPs:               None
Port:              http  8080/TCP
TargetPort:        http/TCP
Endpoints:         172.16.5.118:8080
Session Affinity:  None
Events:            <none>
debug finished in 0 seconds for k3s kubectl describe services --all-namespaces


+--------------------------------------------------------------------------------+
+         k3s kubectl describe statefulset --all-namespaces @1674050911          +
+--------------------------------------------------------------------------------+
Name:               openebs-zfs-controller
Namespace:          kube-system
CreationTimestamp:  Sat, 05 Mar 2022 12:59:47 -0800
Selector:           app=openebs-zfs-controller,role=openebs-zfs
Labels:             objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
Annotations:        objectset.rio.cattle.io/applied:
                      H4sIAAAAAAAA/8xWXY/aOBT9KyurD60U8zEzu9ONxAMdshLqFBCgSlWFkHFughfH9to3Wego/33lBJh0Zpgvdau+ge17fM7xuY5vCDPiM1gntCIhYca4dtElAdkIFZOQzJAhJLmcAZ...
                    objectset.rio.cattle.io/id: 
                    objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                    objectset.rio.cattle.io/owner-name: zfs-operator
                    objectset.rio.cattle.io/owner-namespace: kube-system
Replicas:           1 desired | 1 total
Update Strategy:    RollingUpdate
  Partition:        0
Pods Status:        1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-controller
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-controller-sa
  Containers:
   csi-resizer:
    Image:      k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   csi-snapshotter:
    Image:      k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   snapshot-controller:
    Image:      k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --leader-election=true
    Environment:  <none>
    Mounts:       <none>
   csi-provisioner:
    Image:      k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:   (v1:metadata.namespace)
      POD_NAME:    (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:2.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
  Volumes:
   socket-dir:
    Type:               EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:             
    SizeLimit:          <unset>
  Priority Class Name:  openebs-zfs-csi-controller-critical
Volume Claims:          <none>
Events:                 <none>


Name:               nextcloud-postgresql
Namespace:          ix-nextcloud
CreationTimestamp:  Sun, 13 Nov 2022 17:54:14 -0800
Selector:           app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=postgresql
Labels:             app.kubernetes.io/instance=nextcloud
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=postgresql
                    app.kubernetes.io/version=14.5.0
                    helm-revision=10
                    helm.sh/chart=postgresql-10.0.2
Annotations:        meta.helm.sh/release-name: nextcloud
                    meta.helm.sh/release-namespace: ix-nextcloud
                    rollme: IXEfb
Replicas:           1 desired | 1 total
Update Strategy:    RollingUpdate
Pods Status:        1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app.kubernetes.io/instance=nextcloud
                    app.kubernetes.io/name=postgresql
  Service Account:  default
  Init Containers:
   prepare:
    Image:      tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      
      EOF
      
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:       <none>
  Containers:
   nextcloud-postgresql:
    Image:      tccr.io/truecharts/postgresql:14.5.0@sha256:1d5c95be8ddb4182509773cfc4d75101adcbd1ae97a47011cd52123dff59c691
    Port:       5432/TCP
    Host Port:  0/TCP
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   exec [sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2 ; done] delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  exec [sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2 ; done] delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    exec [sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2 ; done] delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                         2
      UMASK_SET:                     2
      S6_READ_ONLY_ROOT:             1
      NVIDIA_VISIBLE_DEVICES:        void
      TZ:                            UTC
      POSTGRESQL_POSTGRES_PASSWORD:  <set to the key 'postgresql-postgres-password' in secret 'dbcreds'>  Optional: false
      POSTGRES_DB:                   nextcloud
      POSTGRES_PASSWORD:             <set to the key 'postgresql-password' in secret 'dbcreds'>  Optional: false
      POSTGRES_USER:                 nextcloud
    Mounts:
      /bitnami/postgresql from db (rw)
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
  Volumes:
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
Volume Claims:
  Name:          db
  StorageClass:  ix-storage-class-nextcloud
  Labels:        <none>
  Annotations:   <none>
  Capacity:      999Gi
  Access Modes:  [ReadWriteOnce]
Events:          <none>


Name:               nextcloud-redis
Namespace:          ix-nextcloud
CreationTimestamp:  Sun, 13 Nov 2022 17:54:14 -0800
Selector:           app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=redis
Labels:             app.kubernetes.io/instance=nextcloud
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=redis
                    app.kubernetes.io/version=7.0.5
                    helm-revision=10
                    helm.sh/chart=redis-5.0.2
Annotations:        meta.helm.sh/release-name: nextcloud
                    meta.helm.sh/release-namespace: ix-nextcloud
                    rollme: dHHVd
Replicas:           1 desired | 1 total
Update Strategy:    RollingUpdate
Pods Status:        1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app.kubernetes.io/instance=nextcloud
                    app.kubernetes.io/name=redis
  Service Account:  default
  Init Containers:
   prepare:
    Image:      tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "Automatically correcting permissions..."
      
      EOF
      
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:       <none>
  Containers:
   nextcloud-redis:
    Image:      tccr.io/truecharts/redis:7.0.5@sha256:a0de36bdb830474aa89a387be8297edac22fc668cd01ad6831e1d3fa70b6b8df
    Port:       6379/TCP
    Host Port:  0/TCP
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   exec [sh -c /health/ping_liveness_local.sh 2] delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  exec [sh -c /health/ping_readiness_local.sh 2] delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    exec [sh -c /health/ping_readiness_local.sh 2] delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                   2
      UMASK_SET:               2
      S6_READ_ONLY_ROOT:       1
      NVIDIA_VISIBLE_DEVICES:  void
      TZ:                      UTC
      ALLOW_EMPTY_PASSWORD:    yes
      REDIS_PASSWORD:          <set to the key 'redis-password' in secret 'rediscreds'>  Optional: false
      REDIS_PORT:              6379
      REDIS_REPLICATION_MODE:  master
    Mounts:
      /bitnami/redis from data (rw)
      /dev/shm from shm (rw)
      /health from redis-health (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
  Volumes:
   redis-health:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-redis-health
    Optional:  false
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
Volume Claims:
  Name:          data
  StorageClass:  ix-storage-class-nextcloud
  Labels:        <none>
  Annotations:   <none>
  Capacity:      999Gi
  Access Modes:  [ReadWriteOnce]
Events:          <none>
debug finished in 0 seconds for k3s kubectl describe statefulset --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe job --all-namespaces @1674050911              +
+--------------------------------------------------------------------------------+
Name:             metallb-manifests
Namespace:        ix-metallb
Selector:         controller-uid=14a05d27-94f7-4ac8-80ad-80552718e0fd
Labels:           controller-uid=14a05d27-94f7-4ac8-80ad-80552718e0fd
                  job-name=metallb-manifests
Annotations:      batch.kubernetes.io/job-tracking: 
                  helm.sh/hook: pre-install, pre-upgrade
                  helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
                  helm.sh/hook-weight: -6
Parallelism:      1
Completions:      1
Completion Mode:  NonIndexed
Start Time:       Fri, 13 Jan 2023 10:53:47 -0800
Completed At:     Fri, 13 Jan 2023 10:58:26 -0800
Duration:         4m39s
Pods Statuses:    0 Active (0 Ready) / 1 Succeeded / 0 Failed
Pod Template:
  Labels:           controller-uid=14a05d27-94f7-4ac8-80ad-80552718e0fd
                    job-name=metallb-manifests
  Service Account:  metallb-manifests
  Containers:
   metallb-manifests:
    Image:      tccr.io/truecharts/kubectl:v1.26.0@sha256:6d6e0e50f28b961ed1c1c6a9c140553238641591fbdc9ac7c1a348636f78c552
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/sh <<'EOF'
      echo "installing manifests..."
      kubectl apply --server-side --force-conflicts -k https://github.com/truecharts/manifests/manifests  || echo "Manifest application failed..."
      EOF
      
    Environment:  <none>
    Mounts:
      /tmp from metallb-manifests-temp (rw)
  Volumes:
   metallb-manifests-temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
Events:         <none>


Name:             ubiquiti-unifi-manifests
Namespace:        ix-ubiquiti
Selector:         controller-uid=764e03cc-34bf-429d-b885-2a6d0c4a36ae
Labels:           controller-uid=764e03cc-34bf-429d-b885-2a6d0c4a36ae
                  job-name=ubiquiti-unifi-manifests
Annotations:      batch.kubernetes.io/job-tracking: 
                  helm.sh/hook: pre-install, pre-upgrade
                  helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
                  helm.sh/hook-weight: -6
Parallelism:      1
Completions:      1
Completion Mode:  NonIndexed
Start Time:       Fri, 13 Jan 2023 10:51:54 -0800
Completed At:     Fri, 13 Jan 2023 10:58:26 -0800
Duration:         6m32s
Pods Statuses:    0 Active (0 Ready) / 1 Succeeded / 1 Failed
Pod Template:
  Labels:           controller-uid=764e03cc-34bf-429d-b885-2a6d0c4a36ae
                    job-name=ubiquiti-unifi-manifests
  Service Account:  ubiquiti-unifi-manifests
  Containers:
   ubiquiti-unifi-manifests:
    Image:      tccr.io/truecharts/ubuntu:jammy-20221101@sha256:4b9475e08c5180d4e7417dc6a18a26dcce7691e4311e5353dbb952645c5ff43f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "installing manifests..."
      kubectl apply --server-side --force-conflicts -k https://github.com/truecharts/manifests/manifests || echo "Manifest application failed..." 
      EOF
      
    Environment:  <none>
    Mounts:       <none>
  Volumes:        <none>
Events:           <none>


Name:             nextcloud-cronjob-27900840
Namespace:        ix-nextcloud
Selector:         controller-uid=895eb55e-5658-45c8-86ec-e5a38f2bbc39
Labels:           controller-uid=895eb55e-5658-45c8-86ec-e5a38f2bbc39
                  job-name=nextcloud-cronjob-27900840
Annotations:      batch.kubernetes.io/job-tracking: 
Controlled By:    CronJob/nextcloud-cronjob
Parallelism:      1
Completions:      1
Completion Mode:  NonIndexed
Start Time:       Wed, 18 Jan 2023 06:00:00 -0800
Completed At:     Wed, 18 Jan 2023 06:00:06 -0800
Duration:         6s
Pods Statuses:    0 Active (0 Ready) / 1 Succeeded / 0 Failed
Pod Template:
  Labels:  controller-uid=895eb55e-5658-45c8-86ec-e5a38f2bbc39
           job-name=nextcloud-cronjob-27900840
  Containers:
   nextcloud:
    Image:      tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "running nextcloud cronjob..."
      php -f /var/www/html/cron.php
      echo "cronjob finished"
      echo "Pre-generating Previews..."
      php /var/www/html/occ preview:pre-generate
      echo "Previews generated."
      EOF
      
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
  Volumes:
   cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-data
    ReadOnly:   false
   html:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-html
    ReadOnly:   false
   nginx:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-nginx
    Optional:  false
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varrun:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
Events:
  Type    Reason            Age    From            Message
  ----    ------            ----   ----            -------
  Normal  SuccessfulCreate  8m31s  job-controller  Created pod: nextcloud-cronjob-27900840-9n8ws
  Normal  Completed         8m25s  job-controller  Job completed


Name:             nextcloud-cronjob-27900845
Namespace:        ix-nextcloud
Selector:         controller-uid=9fba7890-87f0-49dc-af2b-02676bf350f8
Labels:           controller-uid=9fba7890-87f0-49dc-af2b-02676bf350f8
                  job-name=nextcloud-cronjob-27900845
Annotations:      batch.kubernetes.io/job-tracking: 
Controlled By:    CronJob/nextcloud-cronjob
Parallelism:      1
Completions:      1
Completion Mode:  NonIndexed
Start Time:       Wed, 18 Jan 2023 06:05:00 -0800
Completed At:     Wed, 18 Jan 2023 06:05:06 -0800
Duration:         6s
Pods Statuses:    0 Active (0 Ready) / 1 Succeeded / 0 Failed
Pod Template:
  Labels:  controller-uid=9fba7890-87f0-49dc-af2b-02676bf350f8
           job-name=nextcloud-cronjob-27900845
  Containers:
   nextcloud:
    Image:      tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "running nextcloud cronjob..."
      php -f /var/www/html/cron.php
      echo "cronjob finished"
      echo "Pre-generating Previews..."
      php /var/www/html/occ preview:pre-generate
      echo "Previews generated."
      EOF
      
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
  Volumes:
   cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-data
    ReadOnly:   false
   html:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-html
    ReadOnly:   false
   nginx:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-nginx
    Optional:  false
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varrun:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
Events:
  Type    Reason            Age    From            Message
  ----    ------            ----   ----            -------
  Normal  SuccessfulCreate  3m31s  job-controller  Created pod: nextcloud-cronjob-27900845-w579j
  Normal  Completed         3m25s  job-controller  Job completed
debug finished in 0 seconds for k3s kubectl describe job --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe cronjob --all-namespaces @1674050911            +
+--------------------------------------------------------------------------------+
Name:                          nextcloud-cronjob
Namespace:                     ix-nextcloud
Labels:                        app.kubernetes.io/instance=nextcloud
                               app.kubernetes.io/managed-by=Helm
                               app.kubernetes.io/name=nextcloud
                               app.kubernetes.io/version=25.0.1
                               helm-revision=10
                               helm.sh/chart=nextcloud-19.0.4
Annotations:                   meta.helm.sh/release-name: nextcloud
                               meta.helm.sh/release-namespace: ix-nextcloud
Schedule:                      */5 * * * *
Concurrency Policy:            Forbid
Suspend:                       False
Successful Job History Limit:  2
Failed Job History Limit:      5
Starting Deadline Seconds:     <unset>
Selector:                      <unset>
Parallelism:                   <unset>
Completions:                   <unset>
Pod Template:
  Labels:  <none>
  Containers:
   nextcloud:
    Image:      tccr.io/truecharts/nextcloud-fpm:25.0.1@sha256:8049846f532013eb8fed5d3c13ff49f06b2328fe44c865300e847f9dd2289f4f
    Port:       <none>
    Host Port:  <none>
    Command:
      /bin/sh
      -c
      /bin/bash <<'EOF'
      echo "running nextcloud cronjob..."
      php -f /var/www/html/cron.php
      echo "cronjob finished"
      echo "Pre-generating Previews..."
      php /var/www/html/occ preview:pre-generate
      echo "Previews generated."
      EOF
      
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:        10m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/cache/nginx from cache (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/www/html from html (rw)
      /var/www/html/data from data (rw)
  Volumes:
   cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-data
    ReadOnly:   false
   html:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  nextcloud-html
    ReadOnly:   false
   nginx:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nextcloud-nginx
    Optional:  false
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varrun:
    Type:            EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:          Memory
    SizeLimit:       <unset>
Last Schedule Time:  Wed, 18 Jan 2023 06:05:00 -0800
Active Jobs:         <none>
Events:
  Type    Reason            Age                   From                Message
  ----    ------            ----                  ----                -------
  Normal  SawCompletedJob   41m (x9 over 83m)     cronjob-controller  (combined from similar events): Saw completed job: nextcloud-cronjob-27900805, status: Complete
  Normal  SuccessfulCreate  3m32s (x17 over 83m)  cronjob-controller  (combined from similar events): Created job nextcloud-cronjob-27900845
debug finished in 1 seconds for k3s kubectl describe cronjob --all-namespaces


+--------------------------------------------------------------------------------+
+          k3s kubectl describe daemonsets --all-namespaces @1674050912          +
+--------------------------------------------------------------------------------+
Name:           nvidia-device-plugin-daemonset
Selector:       name=nvidia-device-plugin-ds
Node-Selector:  <none>
Labels:         <none>
Annotations:    deprecated.daemonset.template.generation: 2
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 0
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:       name=nvidia-device-plugin-ds
  Annotations:  scheduler.alpha.kubernetes.io/critical-pod: 
  Containers:
   nvidia-device-plugin-ctr:
    Image:        nvidia/k8s-device-plugin:v0.10.0
    Port:         <none>
    Host Port:    <none>
    Environment:  <none>
    Mounts:
      /var/lib/kubelet/device-plugins from device-plugin (rw)
  Volumes:
   device-plugin:
    Type:               HostPath (bare host directory volume)
    Path:               /var/lib/kubelet/device-plugins
    HostPathType:       
  Priority Class Name:  system-node-critical
Events:                 <none>


Name:           openebs-zfs-node
Selector:       app=openebs-zfs-node
Node-Selector:  <none>
Labels:         objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                openebs.io/component-name=openebs-zfs-node
                openebs.io/version=ci
Annotations:    deprecated.daemonset.template.generation: 2
                objectset.rio.cattle.io/applied:
                  H4sIAAAAAAAA/6xWUY/aOBD+K5XVk1opIezSdnuR9oFCrl11FxBsew/VCjn2JPhw7Mh20uVW+e8nOwEChO2e1Lcknvlm5ptvJn5COGffQWkmBQoRznMdlBfIQ2smKArRGEMmxQIM8l...
                objectset.rio.cattle.io/id: 
                objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                objectset.rio.cattle.io/owner-name: zfs-operator
                objectset.rio.cattle.io/owner-namespace: kube-system
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-node
                    openebs.io/component-name=openebs-zfs-node
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-node-sa
  Containers:
   csi-node-driver-registrar:
    Image:      k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:2.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
  Volumes:
   device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
   encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
   chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
   host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
   registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
   plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
   pods-mount-dir:
    Type:               HostPath (bare host directory volume)
    Path:               /var/lib/kubelet/
    HostPathType:       Directory
  Priority Class Name:  openebs-zfs-csi-node-critical
Events:                 <none>


Name:           speaker
Selector:       app=metallb,component=speaker
Node-Selector:  kubernetes.io/os=linux
Labels:         app=metallb
                component=speaker
Annotations:    deprecated.daemonset.template.generation: 1
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=metallb
                    component=speaker
  Annotations:      prometheus.io/port: 7472
                    prometheus.io/scrape: true
  Service Account:  speaker
  Containers:
   speaker:
    Image:       quay.io/metallb/speaker:v0.13.7
    Ports:       7472/TCP, 7946/TCP, 7946/UDP
    Host Ports:  7472/TCP, 7946/TCP, 7946/UDP
    Args:
      --port=7472
      --log-level=info
    Liveness:   http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_NODE_NAME:       (v1:spec.nodeName)
      METALLB_HOST:            (v1:status.hostIP)
      METALLB_ML_BIND_ADDR:    (v1:status.podIP)
      METALLB_ML_LABELS:      app=metallb,component=speaker
      METALLB_ML_SECRET_KEY:  <set to the key 'secretkey' in secret 'memberlist'>  Optional: false
    Mounts:                   <none>
  Volumes:                    <none>
Events:                       <none>
debug finished in 0 seconds for k3s kubectl describe daemonsets --all-namespaces


+--------------------------------------------------------------------------------+
+                            docker ps -a @1674050912                            +
+--------------------------------------------------------------------------------+
CONTAINER ID   IMAGE                        COMMAND                  CREATED         STATUS                     PORTS     NAMES
f907ee9140a4   456c1c1a56e9                 "/bin/sh -c '/bin/ba…"   3 minutes ago   Exited (0) 3 minutes ago             k8s_nextcloud_nextcloud-cronjob-27900845-w579j_ix-nextcloud_73f1aac6-738e-4d1e-a0f0-f2caa2df987f_0
4ee8fc924c03   rancher/mirrored-pause:3.6   "/pause"                 3 minutes ago   Exited (0) 3 minutes ago             k8s_POD_nextcloud-cronjob-27900845-w579j_ix-nextcloud_73f1aac6-738e-4d1e-a0f0-f2caa2df987f_0
557e3efde8b7   1e326d146757                 "nvidia-device-plugin"   4 minutes ago   Created                              k8s_nvidia-device-plugin-ctr_nvidia-device-plugin-daemonset-55rxn_kube-system_f88bb792-6319-4f42-b4ed-8d3d8b15f26c_29
3a3cdf0476c3   456c1c1a56e9                 "/bin/sh -c '/bin/ba…"   8 minutes ago   Exited (0) 8 minutes ago             k8s_nextcloud_nextcloud-cronjob-27900840-9n8ws_ix-nextcloud_e068ece5-bb8b-48c4-86a1-eea81052efbd_0
d60856fa64dc   rancher/mirrored-pause:3.6   "/pause"                 8 minutes ago   Exited (0) 8 minutes ago             k8s_POD_nextcloud-cronjob-27900840-9n8ws_ix-nextcloud_e068ece5-bb8b-48c4-86a1-eea81052efbd_0
766847d5fdb2   d22303ec76b1                 "/docker-entrypoint.…"   2 hours ago     Up 2 hours                           k8s_nginx_nextcloud-785ff76589-q8zfn_ix-nextcloud_299b6694-d964-4fcf-8333-1bfcceeabe5e_0
83220dfff210   ccc2df4fa965                 "/usr/local/bin/imag…"   2 hours ago     Up 2 hours                           k8s_imaginary_nextcloud-785ff76589-q8zfn_ix-nextcloud_299b6694-d964-4fcf-8333-1bfcceeabe5e_0
cd580a6442cc   456c1c1a56e9                 "/bin/sh -c '/bin/ba…"   2 hours ago     Up 2 hours                           k8s_hpb_nextcloud-785ff76589-q8zfn_ix-nextcloud_299b6694-d964-4fcf-8333-1bfcceeabe5e_0
4d83ce42e8b3   456c1c1a56e9                 "/entrypoint.sh php-…"   2 hours ago     Up 2 hours                           k8s_nextcloud_nextcloud-785ff76589-q8zfn_ix-nextcloud_299b6694-d964-4fcf-8333-1bfcceeabe5e_0
410efb6ac878   456c1c1a56e9                 "/bin/sh -c '/bin/ba…"   2 hours ago     Exited (0) 2 hours ago               k8s_prestart_nextcloud-785ff76589-q8zfn_ix-nextcloud_299b6694-d964-4fcf-8333-1bfcceeabe5e_0
e13ab6d3c588   3b4750d0b451                 "/opt/bitnami/script…"   2 hours ago     Up 2 hours                           k8s_nextcloud-redis_nextcloud-redis-0_ix-nextcloud_cb40ec33-80a4-46a7-b963-0ed4e1fec740_0
13a5059388d7   b7349b403abe                 "/bin/sh -c '/bin/ba…"   2 hours ago     Exited (0) 2 hours ago               k8s_prepare_nextcloud-redis-0_ix-nextcloud_cb40ec33-80a4-46a7-b963-0ed4e1fec740_0
8da03858305e   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_nextcloud-redis-0_ix-nextcloud_cb40ec33-80a4-46a7-b963-0ed4e1fec740_0
4c5c27ec8d2c   2e17748d7c44                 "/usr/local/bin/zfs-…"   2 hours ago     Up 2 hours                           k8s_openebs-zfs-plugin_openebs-zfs-controller-0_kube-system_08f08f16-8d6a-48ed-b210-a068ffd193d6_0
aea6ab6cf132   fe0f921f3c92                 "/csi-provisioner --…"   2 hours ago     Up 2 hours                           k8s_csi-provisioner_openebs-zfs-controller-0_kube-system_08f08f16-8d6a-48ed-b210-a068ffd193d6_0
86b3c7d895ad   f1d8a00ae690                 "/snapshot-controlle…"   2 hours ago     Up 2 hours                           k8s_snapshot-controller_openebs-zfs-controller-0_kube-system_08f08f16-8d6a-48ed-b210-a068ffd193d6_0
44d16be0523f   da32a49a903a                 "/csi-snapshotter --…"   2 hours ago     Up 2 hours                           k8s_csi-snapshotter_openebs-zfs-controller-0_kube-system_08f08f16-8d6a-48ed-b210-a068ffd193d6_0
f0be549fa30a   0aa9629e1508                 "/csi-resizer --v=5 …"   2 hours ago     Up 2 hours                           k8s_csi-resizer_openebs-zfs-controller-0_kube-system_08f08f16-8d6a-48ed-b210-a068ffd193d6_0
867649c03b63   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_openebs-zfs-controller-0_kube-system_08f08f16-8d6a-48ed-b210-a068ffd193d6_0
04951e5db186   df968278084b                 "/bin/operator --kub…"   2 hours ago     Up 2 hours                           k8s_prometheus-operator_prometheus-operator-5c7445d877-npglz_prometheus-operator_ac30f7a3-9366-4261-99c1-8b5ff252e8b6_0
ed1f4f89db7e   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_prometheus-operator-5c7445d877-npglz_prometheus-operator_ac30f7a3-9366-4261-99c1-8b5ff252e8b6_0
14974236b05f   6141bfc77a44                 "/usr/local/bin/dock…"   2 hours ago     Up 2 hours                           k8s_ubiquiti-unifi_ubiquiti-unifi-68fbf8dd6d-g5b4t_ix-ubiquiti_c7efbfcd-72db-47f5-afe9-f7f55639f34e_0
dc8c8e55d908   b7349b403abe                 "/bin/sh -c '/bin/ba…"   2 hours ago     Exited (0) 2 hours ago               k8s_prepare_ubiquiti-unifi-68fbf8dd6d-g5b4t_ix-ubiquiti_c7efbfcd-72db-47f5-afe9-f7f55639f34e_0
2ac18804833c   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_ubiquiti-unifi-68fbf8dd6d-g5b4t_ix-ubiquiti_c7efbfcd-72db-47f5-afe9-f7f55639f34e_0
2122f696b0cd   b7349b403abe                 "/bin/sh -c '/bin/ba…"   2 hours ago     Exited (0) 2 hours ago               k8s_prepare_nextcloud-785ff76589-q8zfn_ix-nextcloud_299b6694-d964-4fcf-8333-1bfcceeabe5e_0
0e03487d75d2   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_nextcloud-785ff76589-q8zfn_ix-nextcloud_299b6694-d964-4fcf-8333-1bfcceeabe5e_0
73d7b3e9bea6   8e08e01e2471                 "/opt/bitnami/script…"   2 hours ago     Up 2 hours                           k8s_nextcloud-postgresql_nextcloud-postgresql-0_ix-nextcloud_f358af22-f279-487e-a7a7-e367e75dac98_0
1eafa1087136   b7349b403abe                 "/bin/sh -c '/bin/ba…"   2 hours ago     Exited (0) 2 hours ago               k8s_prepare_nextcloud-postgresql-0_ix-nextcloud_f358af22-f279-487e-a7a7-e367e75dac98_0
77ccac00d5f9   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_nextcloud-postgresql-0_ix-nextcloud_f358af22-f279-487e-a7a7-e367e75dac98_0
0234ca1bc77c   99376d8f35e0                 "/coredns -conf /etc…"   2 hours ago     Up 2 hours                           k8s_coredns_coredns-d76bd69b-plvk8_kube-system_2bfb7bcd-2653-4a87-8720-981acd255ba3_0
268e1a679bdf   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_coredns-d76bd69b-plvk8_kube-system_2bfb7bcd-2653-4a87-8720-981acd255ba3_0
1c852f226b2a   89e3f8696cdb                 "/manager controller…"   2 hours ago     Up 2 hours                           k8s_manager_cnpg-controller-manager-854876b995-ccsfw_cnpg-system_852af73c-8ac0-4aa5-b2c8-bfad45423df8_0
4571f7301388   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_cnpg-controller-manager-854876b995-ccsfw_cnpg-system_852af73c-8ac0-4aa5-b2c8-bfad45423df8_0
7b967504fa52   e73361dabfb8                 "/controller --port=…"   2 hours ago     Up 2 hours                           k8s_controller_controller-7597dd4f7b-6dxpb_metallb-system_376242ae-40e0-464f-b6e5-fa957316c496_0
23e384c75e0f   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_controller-7597dd4f7b-6dxpb_metallb-system_376242ae-40e0-464f-b6e5-fa957316c496_0
1b74e124ddec   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_nvidia-device-plugin-daemonset-55rxn_kube-system_f88bb792-6319-4f42-b4ed-8d3d8b15f26c_0
8905ce8bce6a   738c5d221d60                 "/speaker --port=747…"   2 hours ago     Up 2 hours                           k8s_speaker_speaker-rlx98_metallb-system_fad0829b-a646-4eff-b64e-c0f9def2889f_0
423f1252f142   2e17748d7c44                 "/usr/local/bin/zfs-…"   2 hours ago     Up 2 hours                           k8s_openebs-zfs-plugin_openebs-zfs-node-wjj5j_kube-system_e0ed93a0-504b-4742-9a38-7373c7ee5cce_0
df7aba3a34b1   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_speaker-rlx98_metallb-system_fad0829b-a646-4eff-b64e-c0f9def2889f_0
7f88253c0fe0   368ee7d6e60f                 "/csi-node-driver-re…"   2 hours ago     Up 2 hours                           k8s_csi-node-driver-registrar_openebs-zfs-node-wjj5j_kube-system_e0ed93a0-504b-4742-9a38-7373c7ee5cce_0
96996a48b4e4   rancher/mirrored-pause:3.6   "/pause"                 2 hours ago     Up 2 hours                           k8s_POD_openebs-zfs-node-wjj5j_kube-system_e0ed93a0-504b-4742-9a38-7373c7ee5cce_0
6fa473975653   99376d8f35e0                 "/coredns -conf /etc…"   19 hours ago    Exited (255) 2 hours ago             k8s_coredns_coredns-d76bd69b-6zwqv_kube-system_5b603b58-ed49-4b65-80c1-58e32a9b8afd_1
6f698353e8e5   rancher/mirrored-pause:3.6   "/pause"                 19 hours ago    Exited (255) 2 hours ago             k8s_POD_coredns-d76bd69b-6zwqv_kube-system_5b603b58-ed49-4b65-80c1-58e32a9b8afd_1
c21a3c4c5847   df968278084b                 "/bin/operator --kub…"   19 hours ago    Exited (255) 2 hours ago             k8s_prometheus-operator_prometheus-operator-5c7445d877-t9kml_prometheus-operator_a2b8a3be-60b0-410a-a88d-8fce688f73a7_1
aba9ba7e973a   rancher/mirrored-pause:3.6   "/pause"                 19 hours ago    Exited (255) 2 hours ago             k8s_POD_prometheus-operator-5c7445d877-t9kml_prometheus-operator_a2b8a3be-60b0-410a-a88d-8fce688f73a7_1
1a4f1a2b5588   89e3f8696cdb                 "/manager controller…"   19 hours ago    Exited (255) 2 hours ago             k8s_manager_cnpg-controller-manager-854876b995-wxd8f_cnpg-system_d5f73741-380f-41a5-85cd-cf36353b735d_1
fee6635222d0   rancher/mirrored-pause:3.6   "/pause"                 19 hours ago    Exited (255) 2 hours ago             k8s_POD_cnpg-controller-manager-854876b995-wxd8f_cnpg-system_d5f73741-380f-41a5-85cd-cf36353b735d_1
debug finished in 0 seconds for docker ps -a


+--------------------------------------------------------------------------------+
+                          docker images -a @1674050912                          +
+--------------------------------------------------------------------------------+
REPOSITORY                                         TAG       IMAGE ID       CREATED         SIZE
tccr.io/truecharts/kubectl                         <none>    672c44cd9781   3 weeks ago     117MB
rancher/mirrored-library-busybox                   1.34.1    334e4a014c81   6 weeks ago     4.86MB
tccr.io/truecharts/nextcloud-fpm                   <none>    456c1c1a56e9   2 months ago    1.48GB
tccr.io/truecharts/redis                           <none>    3b4750d0b451   2 months ago    94.6MB
tccr.io/truecharts/nginx-unprivileged              <none>    d22303ec76b1   2 months ago    142MB
ghcr.io/cloudnative-pg/cloudnative-pg              1.18.0    89e3f8696cdb   2 months ago    45.6MB
tccr.io/truecharts/ubuntu                          <none>    b7349b403abe   2 months ago    538MB
tccr.io/truecharts/unifi                           <none>    6141bfc77a44   2 months ago    713MB
tccr.io/truecharts/postgresql                      <none>    8e08e01e2471   2 months ago    285MB
quay.io/metallb/speaker                            v0.13.7   738c5d221d60   3 months ago    106MB
quay.io/metallb/controller                         v0.13.7   e73361dabfb8   3 months ago    58.7MB
quay.io/prometheus-operator/prometheus-operator    v0.60.1   df968278084b   3 months ago    50.3MB
rancher/mirrored-coredns-coredns                   1.9.1     99376d8f35e0   10 months ago   49.5MB
rancher/klipper-lb                                 v0.3.5    dbd43b6716a0   11 months ago   8.09MB
openebs/zfs-driver                                 2.0.0     2e17748d7c44   12 months ago   176MB
nvidia/k8s-device-plugin                           v0.10.0   1e326d146757   14 months ago   136MB
rancher/mirrored-pause                             3.6       6270bb605e12   17 months ago   683kB
k8s.gcr.io/sig-storage/csi-provisioner             v3.0.0    fe0f921f3c92   17 months ago   56.7MB
k8s.gcr.io/sig-storage/csi-node-driver-registrar   v2.3.0    368ee7d6e60f   17 months ago   18.7MB
k8s.gcr.io/sig-storage/csi-resizer                 v1.2.0    0aa9629e1508   20 months ago   54MB
k8s.gcr.io/sig-storage/csi-snapshotter             v4.0.0    da32a49a903a   2 years ago     49.6MB
k8s.gcr.io/sig-storage/snapshot-controller         v4.0.0    f1d8a00ae690   2 years ago     46.6MB
h2non/imaginary                                    <none>    ccc2df4fa965   2 years ago     226MB
debug finished in 0 seconds for docker images -a


+--------------------------------------------------------------------------------+
+                     K3s Logs (last 1000 lines) @1674050912                     +
+--------------------------------------------------------------------------------+
Jan 18 05:42:04 truenas k3s[10049]: E0118 05:42:04.463927   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:42:07 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:42:07.249-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:42:09 truenas k3s[10049]: time="2023-01-18T05:42:09-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:14 truenas k3s[10049]: time="2023-01-18T05:42:14-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:19 truenas k3s[10049]: time="2023-01-18T05:42:19-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:19 truenas k3s[10049]: I0118 05:42:19.464580   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:42:19 truenas k3s[10049]: E0118 05:42:19.465338   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:42:24 truenas k3s[10049]: time="2023-01-18T05:42:24-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:29 truenas k3s[10049]: time="2023-01-18T05:42:29-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:34 truenas k3s[10049]: time="2023-01-18T05:42:34-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:34 truenas k3s[10049]: I0118 05:42:34.463947   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:42:34 truenas k3s[10049]: E0118 05:42:34.464265   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:42:39 truenas k3s[10049]: time="2023-01-18T05:42:39-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:44 truenas k3s[10049]: time="2023-01-18T05:42:44-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:45 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:42:45.918-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:42:47 truenas k3s[10049]: I0118 05:42:47.464101   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:42:47 truenas k3s[10049]: E0118 05:42:47.464600   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:42:49 truenas k3s[10049]: time="2023-01-18T05:42:49-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:54 truenas k3s[10049]: time="2023-01-18T05:42:54-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:42:59 truenas k3s[10049]: time="2023-01-18T05:42:59-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:00 truenas k3s[10049]: I0118 05:43:00.464458   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:43:00 truenas k3s[10049]: E0118 05:43:00.464969   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:43:04 truenas k3s[10049]: time="2023-01-18T05:43:04-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:09 truenas k3s[10049]: time="2023-01-18T05:43:09-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:14 truenas k3s[10049]: time="2023-01-18T05:43:14-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:15 truenas k3s[10049]: I0118 05:43:15.464175   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:43:15 truenas k3s[10049]: E0118 05:43:15.464474   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:43:17 truenas k3s[10049]: I0118 05:43:17.853046   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:43:17 truenas k3s[10049]: E0118 05:43:17.866651   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 05:45:19.86660911 -0800 PST m=+6266.028019153 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:43:18 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:43:18.835-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:43:19 truenas k3s[10049]: time="2023-01-18T05:43:19-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:24 truenas k3s[10049]: time="2023-01-18T05:43:24-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:28 truenas k3s[10049]: I0118 05:43:28.463830   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:43:28 truenas k3s[10049]: E0118 05:43:28.464356   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:43:29 truenas k3s[10049]: time="2023-01-18T05:43:29-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:34 truenas k3s[10049]: time="2023-01-18T05:43:34-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:39 truenas k3s[10049]: time="2023-01-18T05:43:39-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:43 truenas k3s[10049]: I0118 05:43:43.464106   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:43:43 truenas k3s[10049]: E0118 05:43:43.464869   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:43:44 truenas k3s[10049]: time="2023-01-18T05:43:44-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:49 truenas k3s[10049]: time="2023-01-18T05:43:49-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:54 truenas k3s[10049]: time="2023-01-18T05:43:54-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:43:58 truenas k3s[10049]: I0118 05:43:58.464472   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:43:58 truenas k3s[10049]: E0118 05:43:58.827483   10049 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 18 05:43:58 truenas k3s[10049]:         rpc error: code = Unknown desc = failed to start container "01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 05:43:58 truenas k3s[10049]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 05:43:58 truenas k3s[10049]:  > containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:43:58 truenas k3s[10049]: E0118 05:43:58.827627   10049 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-4njt6,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c): RunContainerError: failed to start container "01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 05:43:58 truenas k3s[10049]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 05:43:58 truenas k3s[10049]: E0118 05:43:58.827721   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:43:59 truenas k3s[10049]: time="2023-01-18T05:43:59-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:00 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:44:00.451-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:44:04 truenas k3s[10049]: time="2023-01-18T05:44:04-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:05 truenas k3s[10049]: I0118 05:44:05.054896   10049 scope.go:115] "RemoveContainer" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:44:06 truenas k3s[10049]: I0118 05:44:06.054545   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="9907760b6b42279c91edb653675ac917e288997f77cc52b1616332abb97d8894"
Jan 18 05:44:06 truenas k3s[10049]: I0118 05:44:06.055145   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:44:06 truenas k3s[10049]: E0118 05:44:06.055699   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:44:09 truenas k3s[10049]: time="2023-01-18T05:44:09-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:14 truenas k3s[10049]: time="2023-01-18T05:44:14-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:19 truenas k3s[10049]: time="2023-01-18T05:44:19-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:20 truenas k3s[10049]: I0118 05:44:20.464472   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:44:20 truenas k3s[10049]: E0118 05:44:20.464986   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:44:24 truenas k3s[10049]: time="2023-01-18T05:44:24-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:29 truenas k3s[10049]: time="2023-01-18T05:44:29-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:33 truenas k3s[10049]: I0118 05:44:33.464237   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:44:33 truenas k3s[10049]: E0118 05:44:33.464749   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:44:34 truenas k3s[10049]: time="2023-01-18T05:44:34-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:39 truenas k3s[10049]: time="2023-01-18T05:44:39-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:44 truenas k3s[10049]: time="2023-01-18T05:44:44-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:44 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:44:44.855-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:44:48 truenas k3s[10049]: I0118 05:44:48.464032   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:44:48 truenas k3s[10049]: E0118 05:44:48.464556   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:44:49 truenas k3s[10049]: time="2023-01-18T05:44:49-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:54 truenas k3s[10049]: time="2023-01-18T05:44:54-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:44:59 truenas k3s[10049]: time="2023-01-18T05:44:59-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.114572   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created job nextcloud-cronjob-27900825"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.115145   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.127872   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900825" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-cronjob-27900825-jjlks"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.128838   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.138010   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.139696   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.141824   10049 topology_manager.go:205] "Topology Admit Handler"
Jan 18 05:45:00 truenas k3s[10049]: E0118 05:45:00.141952   10049 cpu_manager.go:394] "RemoveStaleState: removing container" podUID="5ddcccf3-7926-464a-9a8e-30b8a2f96a3f" containerName="nextcloud"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.142056   10049 memory_manager.go:345] "RemoveStaleState removing state" podUID="5ddcccf3-7926-464a-9a8e-30b8a2f96a3f" containerName="nextcloud"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.161959   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.230784   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-varlogs\") pod \"nextcloud-cronjob-27900825-jjlks\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") " pod="ix-nextcloud/nextcloud-cronjob-27900825-jjlks"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.230892   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-varrun\") pod \"nextcloud-cronjob-27900825-jjlks\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") " pod="ix-nextcloud/nextcloud-cronjob-27900825-jjlks"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.230986   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-shared\") pod \"nextcloud-cronjob-27900825-jjlks\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") " pod="ix-nextcloud/nextcloud-cronjob-27900825-jjlks"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.231042   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-dhznj\" (UniqueName: \"kubernetes.io/projected/8fc4d99a-35f5-45e3-949d-ba274f40fbca-kube-api-access-dhznj\") pod \"nextcloud-cronjob-27900825-jjlks\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") " pod="ix-nextcloud/nextcloud-cronjob-27900825-jjlks"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.231184   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-cache\") pod \"nextcloud-cronjob-27900825-jjlks\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") " pod="ix-nextcloud/nextcloud-cronjob-27900825-jjlks"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.231257   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-shm\") pod \"nextcloud-cronjob-27900825-jjlks\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") " pod="ix-nextcloud/nextcloud-cronjob-27900825-jjlks"
Jan 18 05:45:00 truenas k3s[10049]: I0118 05:45:00.231318   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-temp\") pod \"nextcloud-cronjob-27900825-jjlks\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") " pod="ix-nextcloud/nextcloud-cronjob-27900825-jjlks"
Jan 18 05:45:00 truenas k3s[10049]: time="2023-01-18T05:45:00-08:00" level=info msg="Will attempt to re-write config file /mnt/VMs/ix-applications/docker/containers/25fe585e67a1d2844d25f958f8289f7769ca3b35eaed3eacca74c82d70635d1d/resolv.conf as [nameserver 172.17.0.10 search ix-nextcloud.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Jan 18 05:45:01 truenas k3s[10049]: I0118 05:45:01.111127   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:01 truenas k3s[10049]: I0118 05:45:01.099717  444335 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ix-nextcloud", Name:"nextcloud-cronjob-27900825-jjlks", UID:"8fc4d99a-35f5-45e3-949d-ba274f40fbca", APIVersion:"v1", ResourceVersion:"29468735", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.5.141/16] from ix-net
Jan 18 05:45:01 truenas k3s[10049]: I0118 05:45:01.463588   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:45:01 truenas k3s[10049]: E0118 05:45:01.463891   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:45:02 truenas k3s[10049]: I0118 05:45:02.198812   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:03 truenas k3s[10049]: I0118 05:45:03.210442   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:03 truenas k3s[10049]: I0118 05:45:03.242623   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.252589   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.668219   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"kube-api-access-dhznj\" (UniqueName: \"kubernetes.io/projected/8fc4d99a-35f5-45e3-949d-ba274f40fbca-kube-api-access-dhznj\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.668447   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"html\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.668529   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-temp\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.668598   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-cache\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.668663   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-shared\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.668728   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-varrun\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.668800   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-varlogs\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.668860   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-shm\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.669166   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"data\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586\") pod \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\" (UID: \"8fc4d99a-35f5-45e3-949d-ba274f40fbca\") "
Jan 18 05:45:04 truenas k3s[10049]: W0118 05:45:04.669484   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/8fc4d99a-35f5-45e3-949d-ba274f40fbca/volumes/kubernetes.io~empty-dir/temp: clearQuota called, but quotas disabled
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.669717   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-temp" (OuterVolumeSpecName: "temp") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "temp". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: W0118 05:45:04.670240   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/8fc4d99a-35f5-45e3-949d-ba274f40fbca/volumes/kubernetes.io~empty-dir/varlogs: clearQuota called, but quotas disabled
Jan 18 05:45:04 truenas k3s[10049]: W0118 05:45:04.670340   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/8fc4d99a-35f5-45e3-949d-ba274f40fbca/volumes/kubernetes.io~empty-dir/shared: clearQuota called, but quotas disabled
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.670479   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-varlogs" (OuterVolumeSpecName: "varlogs") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "varlogs". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.670652   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-shared" (OuterVolumeSpecName: "shared") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "shared". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.672783   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/8fc4d99a-35f5-45e3-949d-ba274f40fbca-kube-api-access-dhznj" (OuterVolumeSpecName: "kube-api-access-dhznj") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "kube-api-access-dhznj". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.672832   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-varrun" (OuterVolumeSpecName: "varrun") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "varrun". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.672968   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-cache" (OuterVolumeSpecName: "cache") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "cache". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.674020   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-shm" (OuterVolumeSpecName: "shm") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "shm". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.723195   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586" (OuterVolumeSpecName: "data") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.723270   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68" (OuterVolumeSpecName: "html") pod "8fc4d99a-35f5-45e3-949d-ba274f40fbca" (UID: "8fc4d99a-35f5-45e3-949d-ba274f40fbca"). InnerVolumeSpecName "pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 05:45:04 truenas k3s[10049]: time="2023-01-18T05:45:04-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.770295   10049 reconciler.go:399] "Volume detached for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-shm\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.770364   10049 reconciler.go:399] "Volume detached for volume \"kube-api-access-dhznj\" (UniqueName: \"kubernetes.io/projected/8fc4d99a-35f5-45e3-949d-ba274f40fbca-kube-api-access-dhznj\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.770386   10049 reconciler.go:399] "Volume detached for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-temp\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.770406   10049 reconciler.go:399] "Volume detached for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-cache\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.770425   10049 reconciler.go:399] "Volume detached for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-shared\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.770443   10049 reconciler.go:399] "Volume detached for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-varrun\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:45:04 truenas k3s[10049]: I0118 05:45:04.770462   10049 reconciler.go:399] "Volume detached for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/8fc4d99a-35f5-45e3-949d-ba274f40fbca-varlogs\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:45:05 truenas k3s[10049]: I0118 05:45:05.277269   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="25fe585e67a1d2844d25f958f8289f7769ca3b35eaed3eacca74c82d70635d1d"
Jan 18 05:45:05 truenas k3s[10049]: I0118 05:45:05.305994   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:06 truenas k3s[10049]: I0118 05:45:06.317368   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:06 truenas k3s[10049]: I0118 05:45:06.330300   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900825" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="Completed" message="Job completed"
Jan 18 05:45:06 truenas k3s[10049]: I0118 05:45:06.330978   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:06 truenas k3s[10049]: I0118 05:45:06.342828   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:45:06 truenas k3s[10049]: I0118 05:45:06.343104   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-27900825, status: Complete"
Jan 18 05:45:06 truenas k3s[10049]: I0118 05:45:06.351003   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulDelete" message="Deleted job nextcloud-cronjob-27900815"
Jan 18 05:45:06 truenas k3s[10049]: I0118 05:45:06.351563   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900815
Jan 18 05:45:06 truenas k3s[10049]: E0118 05:45:06.353365   10049 job_controller.go:545] syncing job: tracking status: adding uncounted pods to status: Operation cannot be fulfilled on jobs.batch "nextcloud-cronjob-27900825": the object has been modified; please apply your changes to the latest version and try again
Jan 18 05:45:06 truenas k3s[10049]: I0118 05:45:06.360288   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-27900825, status: Complete"
Jan 18 05:45:06 truenas k3s[10049]: E0118 05:45:06.372668   10049 cronjob_controllerv2.go:164] error syncing CronJobController ix-nextcloud/nextcloud-cronjob, requeuing: Operation cannot be fulfilled on cronjobs.batch "nextcloud-cronjob": the object has been modified; please apply your changes to the latest version and try again
Jan 18 05:45:06 truenas k3s[10049]: E0118 05:45:06.401127   10049 cronjob_controllerv2.go:164] error syncing CronJobController ix-nextcloud/nextcloud-cronjob, requeuing: Operation cannot be fulfilled on cronjobs.batch "nextcloud-cronjob": the object has been modified; please apply your changes to the latest version and try again
Jan 18 05:45:07 truenas k3s[10049]: I0118 05:45:07.497657   10049 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=f6719baa-30f0-4942-97dc-9c638d6dc623 path="/var/lib/kubelet/pods/f6719baa-30f0-4942-97dc-9c638d6dc623/volumes"
Jan 18 05:45:09 truenas k3s[10049]: time="2023-01-18T05:45:09-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:14 truenas k3s[10049]: I0118 05:45:14.463915   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:45:14 truenas k3s[10049]: E0118 05:45:14.464238   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:45:14 truenas k3s[10049]: time="2023-01-18T05:45:14-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:15 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:45:15.492-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:45:19 truenas k3s[10049]: time="2023-01-18T05:45:19-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:19 truenas k3s[10049]: I0118 05:45:19.894674   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:45:19 truenas k3s[10049]: E0118 05:45:19.906328   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 05:47:21.906289266 -0800 PST m=+6388.067699332 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:45:24 truenas k3s[10049]: time="2023-01-18T05:45:24-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:29 truenas k3s[10049]: I0118 05:45:29.464844   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:45:29 truenas k3s[10049]: E0118 05:45:29.465358   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:45:29 truenas k3s[10049]: time="2023-01-18T05:45:29-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:34 truenas k3s[10049]: time="2023-01-18T05:45:34-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:39 truenas k3s[10049]: time="2023-01-18T05:45:39-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:43 truenas k3s[10049]: I0118 05:45:43.464398   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:45:43 truenas k3s[10049]: E0118 05:45:43.464904   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:45:44 truenas k3s[10049]: time="2023-01-18T05:45:44-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:47 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:45:47.964-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:45:49 truenas k3s[10049]: time="2023-01-18T05:45:49-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:54 truenas k3s[10049]: time="2023-01-18T05:45:54-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:45:56 truenas k3s[10049]: I0118 05:45:56.463678   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:45:56 truenas k3s[10049]: E0118 05:45:56.464190   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:45:59 truenas k3s[10049]: time="2023-01-18T05:45:59-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:04 truenas k3s[10049]: time="2023-01-18T05:46:04-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:05 truenas k3s[10049]: I0118 05:46:05.119954   10049 scope.go:115] "RemoveContainer" containerID="59cb590bdd13f4fe523895f2794d3b8d0eab2c060b746450bfcbeacb145423b3"
Jan 18 05:46:08 truenas k3s[10049]: I0118 05:46:08.464217   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:46:08 truenas k3s[10049]: E0118 05:46:08.464731   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:46:09 truenas k3s[10049]: time="2023-01-18T05:46:09-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:14 truenas k3s[10049]: time="2023-01-18T05:46:14-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:19 truenas k3s[10049]: time="2023-01-18T05:46:19-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:23 truenas k3s[10049]: I0118 05:46:23.463923   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:46:23 truenas k3s[10049]: E0118 05:46:23.464422   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:46:24 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:46:24.567-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:46:24 truenas k3s[10049]: time="2023-01-18T05:46:24-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:30 truenas k3s[10049]: time="2023-01-18T05:46:30-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:35 truenas k3s[10049]: time="2023-01-18T05:46:35-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:35 truenas k3s[10049]: I0118 05:46:35.464107   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:46:35 truenas k3s[10049]: E0118 05:46:35.464721   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:46:40 truenas k3s[10049]: time="2023-01-18T05:46:40-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:45 truenas k3s[10049]: time="2023-01-18T05:46:45-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:50 truenas k3s[10049]: time="2023-01-18T05:46:50-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:46:50 truenas k3s[10049]: I0118 05:46:50.463584   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:46:50 truenas k3s[10049]: E0118 05:46:50.464102   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:46:55 truenas k3s[10049]: time="2023-01-18T05:46:55-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:00 truenas k3s[10049]: time="2023-01-18T05:47:00-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:01 truenas k3s[10049]: I0118 05:47:01.464177   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:47:01 truenas k3s[10049]: E0118 05:47:01.464932   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:47:02 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:47:02.742-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:47:05 truenas k3s[10049]: time="2023-01-18T05:47:05-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:10 truenas k3s[10049]: time="2023-01-18T05:47:10-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:15 truenas k3s[10049]: time="2023-01-18T05:47:15-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:15 truenas k3s[10049]: I0118 05:47:15.463788   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:47:15 truenas k3s[10049]: E0118 05:47:15.464089   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:47:20 truenas k3s[10049]: time="2023-01-18T05:47:20-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:21 truenas k3s[10049]: I0118 05:47:21.937124   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:47:21 truenas k3s[10049]: E0118 05:47:21.948815   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 05:49:23.948771631 -0800 PST m=+6510.110181700 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:47:25 truenas k3s[10049]: time="2023-01-18T05:47:25-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:27 truenas k3s[10049]: I0118 05:47:27.464294   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:47:27 truenas k3s[10049]: E0118 05:47:27.464806   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:47:30 truenas k3s[10049]: time="2023-01-18T05:47:30-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:35 truenas k3s[10049]: time="2023-01-18T05:47:35-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:36 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:47:36.394-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:47:39 truenas k3s[10049]: I0118 05:47:39.464578   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:47:39 truenas k3s[10049]: E0118 05:47:39.465108   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:47:40 truenas k3s[10049]: time="2023-01-18T05:47:40-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:45 truenas k3s[10049]: time="2023-01-18T05:47:45-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:50 truenas k3s[10049]: time="2023-01-18T05:47:50-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:47:54 truenas k3s[10049]: I0118 05:47:54.463622   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:47:54 truenas k3s[10049]: E0118 05:47:54.463947   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:47:55 truenas k3s[10049]: time="2023-01-18T05:47:55-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:00 truenas k3s[10049]: time="2023-01-18T05:48:00-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:05 truenas k3s[10049]: time="2023-01-18T05:48:05-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:06 truenas k3s[10049]: I0118 05:48:06.463999   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:48:06 truenas k3s[10049]: E0118 05:48:06.464512   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:48:10 truenas k3s[10049]: time="2023-01-18T05:48:10-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:15 truenas k3s[10049]: time="2023-01-18T05:48:15-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:17 truenas k3s[10049]: I0118 05:48:17.464053   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:48:17 truenas k3s[10049]: E0118 05:48:17.464538   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:48:17 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:48:17.484-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:48:20 truenas k3s[10049]: time="2023-01-18T05:48:20-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:25 truenas k3s[10049]: time="2023-01-18T05:48:25-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:30 truenas k3s[10049]: time="2023-01-18T05:48:30-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:32 truenas k3s[10049]: I0118 05:48:32.463866   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:48:32 truenas k3s[10049]: E0118 05:48:32.464377   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:48:35 truenas k3s[10049]: time="2023-01-18T05:48:35-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:40 truenas k3s[10049]: time="2023-01-18T05:48:40-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:43 truenas k3s[10049]: I0118 05:48:43.464407   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:48:43 truenas k3s[10049]: E0118 05:48:43.464920   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:48:45 truenas k3s[10049]: time="2023-01-18T05:48:45-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:50 truenas k3s[10049]: time="2023-01-18T05:48:50-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:55 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:48:55.309-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:48:55 truenas k3s[10049]: time="2023-01-18T05:48:55-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:48:56 truenas k3s[10049]: I0118 05:48:56.464160   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:48:56 truenas k3s[10049]: E0118 05:48:56.464687   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:49:00 truenas k3s[10049]: time="2023-01-18T05:49:00-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:05 truenas k3s[10049]: time="2023-01-18T05:49:05-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:10 truenas k3s[10049]: time="2023-01-18T05:49:10-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:11 truenas k3s[10049]: I0118 05:49:11.463614   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:49:11 truenas k3s[10049]: E0118 05:49:11.816643   10049 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 18 05:49:11 truenas k3s[10049]:         rpc error: code = Unknown desc = failed to start container "f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 05:49:11 truenas k3s[10049]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 05:49:11 truenas k3s[10049]:  > containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:49:11 truenas k3s[10049]: E0118 05:49:11.816814   10049 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-4njt6,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c): RunContainerError: failed to start container "f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 05:49:11 truenas k3s[10049]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 05:49:11 truenas k3s[10049]: E0118 05:49:11.816885   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:49:15 truenas k3s[10049]: time="2023-01-18T05:49:15-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:20 truenas k3s[10049]: time="2023-01-18T05:49:20-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:22 truenas k3s[10049]: I0118 05:49:22.463967   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:49:22 truenas k3s[10049]: E0118 05:49:22.464470   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:49:23 truenas k3s[10049]: I0118 05:49:23.984495   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:49:23 truenas k3s[10049]: E0118 05:49:23.995861   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 05:51:25.995821008 -0800 PST m=+6632.157231077 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:49:25 truenas k3s[10049]: time="2023-01-18T05:49:25-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:29 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:49:29.031-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:49:30 truenas k3s[10049]: time="2023-01-18T05:49:30-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:35 truenas k3s[10049]: time="2023-01-18T05:49:35-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:36 truenas k3s[10049]: I0118 05:49:36.463950   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:49:36 truenas k3s[10049]: E0118 05:49:36.464459   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:49:40 truenas k3s[10049]: time="2023-01-18T05:49:40-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:45 truenas k3s[10049]: time="2023-01-18T05:49:45-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:50 truenas k3s[10049]: time="2023-01-18T05:49:50-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:49:51 truenas k3s[10049]: I0118 05:49:51.464514   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:49:51 truenas k3s[10049]: E0118 05:49:51.465026   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:49:55 truenas k3s[10049]: time="2023-01-18T05:49:55-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.113335   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created job nextcloud-cronjob-27900830"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.113708   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.129720   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900830" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-cronjob-27900830-gwnhg"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.130067   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.139280   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.140573   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.142181   10049 topology_manager.go:205] "Topology Admit Handler"
Jan 18 05:50:00 truenas k3s[10049]: E0118 05:50:00.142302   10049 cpu_manager.go:394] "RemoveStaleState: removing container" podUID="8fc4d99a-35f5-45e3-949d-ba274f40fbca" containerName="nextcloud"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.142400   10049 memory_manager.go:345] "RemoveStaleState removing state" podUID="8fc4d99a-35f5-45e3-949d-ba274f40fbca" containerName="nextcloud"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.157146   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.303923   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-shared\") pod \"nextcloud-cronjob-27900830-gwnhg\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") " pod="ix-nextcloud/nextcloud-cronjob-27900830-gwnhg"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.303989   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-varrun\") pod \"nextcloud-cronjob-27900830-gwnhg\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") " pod="ix-nextcloud/nextcloud-cronjob-27900830-gwnhg"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.304127   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-shm\") pod \"nextcloud-cronjob-27900830-gwnhg\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") " pod="ix-nextcloud/nextcloud-cronjob-27900830-gwnhg"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.304193   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-varlogs\") pod \"nextcloud-cronjob-27900830-gwnhg\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") " pod="ix-nextcloud/nextcloud-cronjob-27900830-gwnhg"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.304322   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-temp\") pod \"nextcloud-cronjob-27900830-gwnhg\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") " pod="ix-nextcloud/nextcloud-cronjob-27900830-gwnhg"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.304382   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-cache\") pod \"nextcloud-cronjob-27900830-gwnhg\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") " pod="ix-nextcloud/nextcloud-cronjob-27900830-gwnhg"
Jan 18 05:50:00 truenas k3s[10049]: I0118 05:50:00.304514   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-wc9j6\" (UniqueName: \"kubernetes.io/projected/1fe74375-7d23-4612-8884-cdefc0ecc3d1-kube-api-access-wc9j6\") pod \"nextcloud-cronjob-27900830-gwnhg\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") " pod="ix-nextcloud/nextcloud-cronjob-27900830-gwnhg"
Jan 18 05:50:00 truenas k3s[10049]: time="2023-01-18T05:50:00-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:00 truenas k3s[10049]: time="2023-01-18T05:50:00-08:00" level=info msg="Will attempt to re-write config file /mnt/VMs/ix-applications/docker/containers/e5e3242be4b520a119b415c765e5f682b14d37e5bc0433683ca35516d4b3382f/resolv.conf as [nameserver 172.17.0.10 search ix-nextcloud.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Jan 18 05:50:01 truenas k3s[10049]: I0118 05:50:01.149003   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:01 truenas k3s[10049]: I0118 05:50:01.135979  463128 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ix-nextcloud", Name:"nextcloud-cronjob-27900830-gwnhg", UID:"1fe74375-7d23-4612-8884-cdefc0ecc3d1", APIVersion:"v1", ResourceVersion:"29469228", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.5.142/16] from ix-net
Jan 18 05:50:02 truenas k3s[10049]: I0118 05:50:02.144979   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:02 truenas k3s[10049]: I0118 05:50:02.155430   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:02 truenas k3s[10049]: I0118 05:50:02.463801   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:50:02 truenas k3s[10049]: E0118 05:50:02.464177   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:50:02 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:50:02.748-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:50:03 truenas k3s[10049]: I0118 05:50:03.164596   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.176361   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540316   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-temp\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540534   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"data\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540580   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-shared\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540617   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-shm\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540658   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-varlogs\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540759   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"html\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540800   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-varrun\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540834   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-cache\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: W0118 05:50:04.540858   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/1fe74375-7d23-4612-8884-cdefc0ecc3d1/volumes/kubernetes.io~empty-dir/temp: clearQuota called, but quotas disabled
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540873   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"kube-api-access-wc9j6\" (UniqueName: \"kubernetes.io/projected/1fe74375-7d23-4612-8884-cdefc0ecc3d1-kube-api-access-wc9j6\") pod \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\" (UID: \"1fe74375-7d23-4612-8884-cdefc0ecc3d1\") "
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.540981   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-temp" (OuterVolumeSpecName: "temp") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "temp". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: W0118 05:50:04.541300   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/1fe74375-7d23-4612-8884-cdefc0ecc3d1/volumes/kubernetes.io~empty-dir/shared: clearQuota called, but quotas disabled
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.541465   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-shared" (OuterVolumeSpecName: "shared") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "shared". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: W0118 05:50:04.541500   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/1fe74375-7d23-4612-8884-cdefc0ecc3d1/volumes/kubernetes.io~empty-dir/varlogs: clearQuota called, but quotas disabled
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.541624   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-varlogs" (OuterVolumeSpecName: "varlogs") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "varlogs". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.543846   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-varrun" (OuterVolumeSpecName: "varrun") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "varrun". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.544074   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-shm" (OuterVolumeSpecName: "shm") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "shm". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.544240   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/1fe74375-7d23-4612-8884-cdefc0ecc3d1-kube-api-access-wc9j6" (OuterVolumeSpecName: "kube-api-access-wc9j6") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "kube-api-access-wc9j6". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.545353   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-cache" (OuterVolumeSpecName: "cache") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "cache". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.591938   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586" (OuterVolumeSpecName: "data") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.592211   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68" (OuterVolumeSpecName: "html") pod "1fe74375-7d23-4612-8884-cdefc0ecc3d1" (UID: "1fe74375-7d23-4612-8884-cdefc0ecc3d1"). InnerVolumeSpecName "pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.641524   10049 reconciler.go:399] "Volume detached for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-varrun\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.641590   10049 reconciler.go:399] "Volume detached for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-cache\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.641616   10049 reconciler.go:399] "Volume detached for volume \"kube-api-access-wc9j6\" (UniqueName: \"kubernetes.io/projected/1fe74375-7d23-4612-8884-cdefc0ecc3d1-kube-api-access-wc9j6\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.641637   10049 reconciler.go:399] "Volume detached for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-temp\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.641656   10049 reconciler.go:399] "Volume detached for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-shared\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.641674   10049 reconciler.go:399] "Volume detached for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-shm\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:50:04 truenas k3s[10049]: I0118 05:50:04.641693   10049 reconciler.go:399] "Volume detached for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/1fe74375-7d23-4612-8884-cdefc0ecc3d1-varlogs\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:50:05 truenas k3s[10049]: I0118 05:50:05.195761   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="e5e3242be4b520a119b415c765e5f682b14d37e5bc0433683ca35516d4b3382f"
Jan 18 05:50:05 truenas k3s[10049]: I0118 05:50:05.224127   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:05 truenas k3s[10049]: I0118 05:50:05.257736   10049 scope.go:115] "RemoveContainer" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:50:05 truenas k3s[10049]: time="2023-01-18T05:50:05-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.233190   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="01d5e360d0e92ecae200515733575ea88d50e3f23e2d52df9747ba41963c05f1"
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.233746   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:50:06 truenas k3s[10049]: E0118 05:50:06.234231   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.234670   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.245421   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900830" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="Completed" message="Job completed"
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.245556   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.253063   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.253277   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-27900830, status: Complete"
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.258626   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulDelete" message="Deleted job nextcloud-cronjob-27900820"
Jan 18 05:50:06 truenas k3s[10049]: E0118 05:50:06.258967   10049 job_controller.go:545] syncing job: tracking status: adding uncounted pods to status: Operation cannot be fulfilled on jobs.batch "nextcloud-cronjob-27900830": the object has been modified; please apply your changes to the latest version and try again
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.259120   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900820
Jan 18 05:50:06 truenas k3s[10049]: I0118 05:50:06.268287   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-27900830, status: Complete"
Jan 18 05:50:06 truenas k3s[10049]: E0118 05:50:06.274701   10049 cronjob_controllerv2.go:164] error syncing CronJobController ix-nextcloud/nextcloud-cronjob, requeuing: Operation cannot be fulfilled on cronjobs.batch "nextcloud-cronjob": the object has been modified; please apply your changes to the latest version and try again
Jan 18 05:50:06 truenas k3s[10049]: E0118 05:50:06.287906   10049 cronjob_controllerv2.go:164] error syncing CronJobController ix-nextcloud/nextcloud-cronjob, requeuing: Operation cannot be fulfilled on cronjobs.batch "nextcloud-cronjob": the object has been modified; please apply your changes to the latest version and try again
Jan 18 05:50:07 truenas k3s[10049]: I0118 05:50:07.483733   10049 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=5ddcccf3-7926-464a-9a8e-30b8a2f96a3f path="/var/lib/kubelet/pods/5ddcccf3-7926-464a-9a8e-30b8a2f96a3f/volumes"
Jan 18 05:50:10 truenas k3s[10049]: time="2023-01-18T05:50:10-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:15 truenas k3s[10049]: time="2023-01-18T05:50:15-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:17 truenas k3s[10049]: I0118 05:50:17.464421   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:50:17 truenas k3s[10049]: E0118 05:50:17.464935   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:50:20 truenas k3s[10049]: time="2023-01-18T05:50:20-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:25 truenas k3s[10049]: time="2023-01-18T05:50:25-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:30 truenas k3s[10049]: time="2023-01-18T05:50:30-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:31 truenas k3s[10049]: I0118 05:50:31.463961   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:50:31 truenas k3s[10049]: E0118 05:50:31.464476   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:50:35 truenas k3s[10049]: time="2023-01-18T05:50:35-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:40 truenas k3s[10049]: time="2023-01-18T05:50:40-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:44 truenas k3s[10049]: I0118 05:50:44.463654   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:50:44 truenas k3s[10049]: E0118 05:50:44.464158   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:50:45 truenas k3s[10049]: time="2023-01-18T05:50:45-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:47 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:50:47.580-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:50:50 truenas k3s[10049]: time="2023-01-18T05:50:50-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:55 truenas k3s[10049]: time="2023-01-18T05:50:55-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:50:57 truenas k3s[10049]: I0118 05:50:57.463896   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:50:57 truenas k3s[10049]: E0118 05:50:57.464222   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:51:00 truenas k3s[10049]: time="2023-01-18T05:51:00-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:05 truenas k3s[10049]: I0118 05:51:05.311219   10049 scope.go:115] "RemoveContainer" containerID="7988543d2c8c00c0d71a84dc6a2f26fe2aeb8590493d0a4fdce1c8a686089955"
Jan 18 05:51:05 truenas k3s[10049]: time="2023-01-18T05:51:05-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:10 truenas k3s[10049]: I0118 05:51:10.464365   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:51:10 truenas k3s[10049]: E0118 05:51:10.464887   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:51:10 truenas k3s[10049]: time="2023-01-18T05:51:10-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:15 truenas k3s[10049]: time="2023-01-18T05:51:15-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:20 truenas k3s[10049]: time="2023-01-18T05:51:20-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:21 truenas k3s[10049]: I0118 05:51:21.463918   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:51:21 truenas k3s[10049]: E0118 05:51:21.464461   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:51:25 truenas k3s[10049]: time="2023-01-18T05:51:25-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:26 truenas k3s[10049]: I0118 05:51:26.051690   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:51:26 truenas k3s[10049]: E0118 05:51:26.065878   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 05:53:28.065838172 -0800 PST m=+6754.227248242 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:51:28 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:51:28.654-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:51:30 truenas k3s[10049]: time="2023-01-18T05:51:30-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:34 truenas k3s[10049]: I0118 05:51:34.464417   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:51:34 truenas k3s[10049]: E0118 05:51:34.464780   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:51:36 truenas k3s[10049]: time="2023-01-18T05:51:36-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:41 truenas k3s[10049]: time="2023-01-18T05:51:41-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:46 truenas k3s[10049]: time="2023-01-18T05:51:46-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:48 truenas k3s[10049]: I0118 05:51:48.464180   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:51:48 truenas k3s[10049]: E0118 05:51:48.464685   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:51:51 truenas k3s[10049]: time="2023-01-18T05:51:51-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:56 truenas k3s[10049]: time="2023-01-18T05:51:56-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:51:59 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:51:59.832-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:52:01 truenas k3s[10049]: time="2023-01-18T05:52:01-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:03 truenas k3s[10049]: I0118 05:52:03.464139   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:52:03 truenas k3s[10049]: E0118 05:52:03.464676   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:52:06 truenas k3s[10049]: time="2023-01-18T05:52:06-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:11 truenas k3s[10049]: time="2023-01-18T05:52:11-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:16 truenas k3s[10049]: time="2023-01-18T05:52:16-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:16 truenas k3s[10049]: I0118 05:52:16.463822   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:52:16 truenas k3s[10049]: E0118 05:52:16.464329   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:52:21 truenas k3s[10049]: time="2023-01-18T05:52:21-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:26 truenas k3s[10049]: time="2023-01-18T05:52:26-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:29 truenas k3s[10049]: I0118 05:52:29.464990   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:52:29 truenas k3s[10049]: E0118 05:52:29.465495   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:52:31 truenas k3s[10049]: time="2023-01-18T05:52:31-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:32 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:52:32.943-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:52:36 truenas k3s[10049]: time="2023-01-18T05:52:36-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:41 truenas k3s[10049]: time="2023-01-18T05:52:41-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:41 truenas k3s[10049]: I0118 05:52:41.464102   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:52:41 truenas k3s[10049]: E0118 05:52:41.464635   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:52:46 truenas k3s[10049]: time="2023-01-18T05:52:46-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:51 truenas k3s[10049]: time="2023-01-18T05:52:51-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:52:54 truenas k3s[10049]: I0118 05:52:54.464208   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:52:54 truenas k3s[10049]: E0118 05:52:54.464542   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:52:56 truenas k3s[10049]: time="2023-01-18T05:52:56-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:01 truenas k3s[10049]: time="2023-01-18T05:53:01-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:06 truenas k3s[10049]: time="2023-01-18T05:53:06-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:06 truenas k3s[10049]: I0118 05:53:06.464329   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:53:06 truenas k3s[10049]: E0118 05:53:06.464856   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:53:11 truenas k3s[10049]: time="2023-01-18T05:53:11-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:12 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:53:12.101-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:53:16 truenas k3s[10049]: time="2023-01-18T05:53:16-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:21 truenas k3s[10049]: time="2023-01-18T05:53:21-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:21 truenas k3s[10049]: I0118 05:53:21.464122   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:53:21 truenas k3s[10049]: E0118 05:53:21.464624   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:53:26 truenas k3s[10049]: time="2023-01-18T05:53:26-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:28 truenas k3s[10049]: I0118 05:53:28.110305   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:53:28 truenas k3s[10049]: E0118 05:53:28.122620   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 05:55:30.122579841 -0800 PST m=+6876.283989887 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:53:31 truenas k3s[10049]: time="2023-01-18T05:53:31-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:32 truenas k3s[10049]: I0118 05:53:32.463476   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:53:32 truenas k3s[10049]: E0118 05:53:32.463985   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:53:36 truenas k3s[10049]: time="2023-01-18T05:53:36-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:41 truenas k3s[10049]: time="2023-01-18T05:53:41-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:45 truenas k3s[10049]: I0118 05:53:45.464700   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:53:45 truenas k3s[10049]: E0118 05:53:45.465461   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:53:46 truenas k3s[10049]: time="2023-01-18T05:53:46-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:51 truenas k3s[10049]: time="2023-01-18T05:53:51-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:52 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:53:52.374-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:53:56 truenas k3s[10049]: time="2023-01-18T05:53:56-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:53:59 truenas k3s[10049]: I0118 05:53:59.464602   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:53:59 truenas k3s[10049]: E0118 05:53:59.464953   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:54:01 truenas k3s[10049]: time="2023-01-18T05:54:01-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:06 truenas k3s[10049]: time="2023-01-18T05:54:06-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:11 truenas k3s[10049]: time="2023-01-18T05:54:11-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:14 truenas k3s[10049]: I0118 05:54:14.463985   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:54:14 truenas k3s[10049]: E0118 05:54:14.779449   10049 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 18 05:54:14 truenas k3s[10049]:         rpc error: code = Unknown desc = failed to start container "e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 05:54:14 truenas k3s[10049]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 05:54:14 truenas k3s[10049]:  > containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:54:14 truenas k3s[10049]: E0118 05:54:14.779613   10049 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-4njt6,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c): RunContainerError: failed to start container "e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 05:54:14 truenas k3s[10049]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 05:54:14 truenas k3s[10049]: E0118 05:54:14.779685   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:54:16 truenas k3s[10049]: time="2023-01-18T05:54:16-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:21 truenas k3s[10049]: time="2023-01-18T05:54:21-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:26 truenas k3s[10049]: time="2023-01-18T05:54:26-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:27 truenas k3s[10049]: I0118 05:54:27.464438   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:54:27 truenas k3s[10049]: E0118 05:54:27.464954   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:54:31 truenas k3s[10049]: time="2023-01-18T05:54:31-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:36 truenas k3s[10049]: time="2023-01-18T05:54:36-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:36 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:54:36.837-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:54:41 truenas k3s[10049]: I0118 05:54:41.464478   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:54:41 truenas k3s[10049]: E0118 05:54:41.464977   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:54:41 truenas k3s[10049]: time="2023-01-18T05:54:41-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:46 truenas k3s[10049]: time="2023-01-18T05:54:46-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:51 truenas k3s[10049]: time="2023-01-18T05:54:51-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:54:56 truenas k3s[10049]: I0118 05:54:56.463575   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:54:56 truenas k3s[10049]: E0118 05:54:56.464096   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:54:56 truenas k3s[10049]: time="2023-01-18T05:54:56-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.108653   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created job nextcloud-cronjob-27900835"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.109141   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.118258   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900835" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-cronjob-27900835-6prsd"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.118665   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.123191   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.124268   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.124480   10049 topology_manager.go:205] "Topology Admit Handler"
Jan 18 05:55:00 truenas k3s[10049]: E0118 05:55:00.124596   10049 cpu_manager.go:394] "RemoveStaleState: removing container" podUID="1fe74375-7d23-4612-8884-cdefc0ecc3d1" containerName="nextcloud"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.124698   10049 memory_manager.go:345] "RemoveStaleState removing state" podUID="1fe74375-7d23-4612-8884-cdefc0ecc3d1" containerName="nextcloud"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.135316   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.193489   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-shared\") pod \"nextcloud-cronjob-27900835-6prsd\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") " pod="ix-nextcloud/nextcloud-cronjob-27900835-6prsd"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.193572   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-temp\") pod \"nextcloud-cronjob-27900835-6prsd\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") " pod="ix-nextcloud/nextcloud-cronjob-27900835-6prsd"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.193615   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-8cr7s\" (UniqueName: \"kubernetes.io/projected/ead5c4bb-d3c6-4842-8c25-2c1315382909-kube-api-access-8cr7s\") pod \"nextcloud-cronjob-27900835-6prsd\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") " pod="ix-nextcloud/nextcloud-cronjob-27900835-6prsd"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.193710   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-shm\") pod \"nextcloud-cronjob-27900835-6prsd\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") " pod="ix-nextcloud/nextcloud-cronjob-27900835-6prsd"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.193759   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-varrun\") pod \"nextcloud-cronjob-27900835-6prsd\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") " pod="ix-nextcloud/nextcloud-cronjob-27900835-6prsd"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.193886   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-varlogs\") pod \"nextcloud-cronjob-27900835-6prsd\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") " pod="ix-nextcloud/nextcloud-cronjob-27900835-6prsd"
Jan 18 05:55:00 truenas k3s[10049]: I0118 05:55:00.194028   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-cache\") pod \"nextcloud-cronjob-27900835-6prsd\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") " pod="ix-nextcloud/nextcloud-cronjob-27900835-6prsd"
Jan 18 05:55:00 truenas k3s[10049]: time="2023-01-18T05:55:00-08:00" level=info msg="Will attempt to re-write config file /mnt/VMs/ix-applications/docker/containers/bf1d9c491980705fcc06a1d084270fd6933af7423b5b684042a7eb44d07f2bcc/resolv.conf as [nameserver 172.17.0.10 search ix-nextcloud.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Jan 18 05:55:01 truenas k3s[10049]: I0118 05:55:01.048476   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:01 truenas k3s[10049]: I0118 05:55:01.035815  481820 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ix-nextcloud", Name:"nextcloud-cronjob-27900835-6prsd", UID:"ead5c4bb-d3c6-4842-8c25-2c1315382909", APIVersion:"v1", ResourceVersion:"29469720", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.5.143/16] from ix-net
Jan 18 05:55:01 truenas k3s[10049]: time="2023-01-18T05:55:01-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:02 truenas k3s[10049]: I0118 05:55:02.002775   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:03 truenas k3s[10049]: I0118 05:55:03.010450   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:03 truenas k3s[10049]: I0118 05:55:03.028561   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.040045   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.428200   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"data\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.428254   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-temp\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.428359   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"html\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.428404   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-shared\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.428437   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-varrun\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: W0118 05:55:04.428805   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/ead5c4bb-d3c6-4842-8c25-2c1315382909/volumes/kubernetes.io~empty-dir/temp: clearQuota called, but quotas disabled
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.428932   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-temp" (OuterVolumeSpecName: "temp") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "temp". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: W0118 05:55:04.428972   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/ead5c4bb-d3c6-4842-8c25-2c1315382909/volumes/kubernetes.io~empty-dir/shared: clearQuota called, but quotas disabled
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.429077   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-shared" (OuterVolumeSpecName: "shared") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "shared". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.451121   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-varrun" (OuterVolumeSpecName: "varrun") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "varrun". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.495798   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586" (OuterVolumeSpecName: "data") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.495928   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68" (OuterVolumeSpecName: "html") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.529346   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-cache\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.529398   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"kube-api-access-8cr7s\" (UniqueName: \"kubernetes.io/projected/ead5c4bb-d3c6-4842-8c25-2c1315382909-kube-api-access-8cr7s\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.529435   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-shm\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.529579   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-varlogs\") pod \"ead5c4bb-d3c6-4842-8c25-2c1315382909\" (UID: \"ead5c4bb-d3c6-4842-8c25-2c1315382909\") "
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.529684   10049 reconciler.go:399] "Volume detached for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-temp\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.529708   10049 reconciler.go:399] "Volume detached for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-shared\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.529725   10049 reconciler.go:399] "Volume detached for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-varrun\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:55:04 truenas k3s[10049]: W0118 05:55:04.529994   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/ead5c4bb-d3c6-4842-8c25-2c1315382909/volumes/kubernetes.io~empty-dir/varlogs: clearQuota called, but quotas disabled
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.530108   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-varlogs" (OuterVolumeSpecName: "varlogs") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "varlogs". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.532062   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-cache" (OuterVolumeSpecName: "cache") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "cache". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.532159   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/ead5c4bb-d3c6-4842-8c25-2c1315382909-kube-api-access-8cr7s" (OuterVolumeSpecName: "kube-api-access-8cr7s") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "kube-api-access-8cr7s". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.532173   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-shm" (OuterVolumeSpecName: "shm") pod "ead5c4bb-d3c6-4842-8c25-2c1315382909" (UID: "ead5c4bb-d3c6-4842-8c25-2c1315382909"). InnerVolumeSpecName "shm". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.630871   10049 reconciler.go:399] "Volume detached for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-cache\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.630935   10049 reconciler.go:399] "Volume detached for volume \"kube-api-access-8cr7s\" (UniqueName: \"kubernetes.io/projected/ead5c4bb-d3c6-4842-8c25-2c1315382909-kube-api-access-8cr7s\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.630969   10049 reconciler.go:399] "Volume detached for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-shm\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:55:04 truenas k3s[10049]: I0118 05:55:04.631000   10049 reconciler.go:399] "Volume detached for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/ead5c4bb-d3c6-4842-8c25-2c1315382909-varlogs\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 05:55:05 truenas k3s[10049]: I0118 05:55:05.057714   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="bf1d9c491980705fcc06a1d084270fd6933af7423b5b684042a7eb44d07f2bcc"
Jan 18 05:55:05 truenas k3s[10049]: I0118 05:55:05.075445   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:05 truenas k3s[10049]: I0118 05:55:05.508701   10049 scope.go:115] "RemoveContainer" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.087029   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.091876   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="f46e070511497b35b9d9f9730cc9dced50d45295e48e19b6dc5382bce0816e9d"
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.092440   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:55:06 truenas k3s[10049]: E0118 05:55:06.092920   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.099112   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900835" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="Completed" message="Job completed"
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.099669   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.106651   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.106883   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-27900835, status: Complete"
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.113057   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulDelete" message="Deleted job nextcloud-cronjob-27900825"
Jan 18 05:55:06 truenas k3s[10049]: I0118 05:55:06.113743   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900825
Jan 18 05:55:06 truenas k3s[10049]: E0118 05:55:06.114237   10049 job_controller.go:545] syncing job: tracking status: adding uncounted pods to status: Operation cannot be fulfilled on jobs.batch "nextcloud-cronjob-27900835": the object has been modified; please apply your changes to the latest version and try again
Jan 18 05:55:06 truenas k3s[10049]: time="2023-01-18T05:55:06-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:07 truenas k3s[10049]: I0118 05:55:07.497950   10049 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=8fc4d99a-35f5-45e3-949d-ba274f40fbca path="/var/lib/kubelet/pods/8fc4d99a-35f5-45e3-949d-ba274f40fbca/volumes"
Jan 18 05:55:10 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:55:10.012-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:55:11 truenas k3s[10049]: time="2023-01-18T05:55:11-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:16 truenas k3s[10049]: time="2023-01-18T05:55:16-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:18 truenas k3s[10049]: I0118 05:55:18.464022   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:55:18 truenas k3s[10049]: E0118 05:55:18.464536   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:55:21 truenas k3s[10049]: time="2023-01-18T05:55:21-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:26 truenas k3s[10049]: time="2023-01-18T05:55:26-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:30 truenas k3s[10049]: I0118 05:55:30.147456   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:55:30 truenas k3s[10049]: E0118 05:55:30.159675   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 05:57:32.159635357 -0800 PST m=+6998.321045423 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:55:31 truenas k3s[10049]: time="2023-01-18T05:55:31-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:33 truenas k3s[10049]: I0118 05:55:33.464316   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:55:33 truenas k3s[10049]: E0118 05:55:33.464827   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:55:36 truenas k3s[10049]: time="2023-01-18T05:55:36-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:41 truenas k3s[10049]: time="2023-01-18T05:55:41-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:45 truenas k3s[10049]: I0118 05:55:45.463931   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:55:45 truenas k3s[10049]: E0118 05:55:45.464444   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:55:46 truenas k3s[10049]: time="2023-01-18T05:55:46-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:49 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:55:49.031-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:55:51 truenas k3s[10049]: time="2023-01-18T05:55:51-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:55:56 truenas k3s[10049]: time="2023-01-18T05:55:56-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:00 truenas k3s[10049]: I0118 05:56:00.464119   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:56:00 truenas k3s[10049]: E0118 05:56:00.464625   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:56:01 truenas k3s[10049]: time="2023-01-18T05:56:01-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:05 truenas k3s[10049]: I0118 05:56:05.580413   10049 scope.go:115] "RemoveContainer" containerID="57e6afb7ccfd3f820e3d54fa26da9320015e2195c5d4a788a2f64cb5788c835f"
Jan 18 05:56:06 truenas k3s[10049]: time="2023-01-18T05:56:06-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:11 truenas k3s[10049]: time="2023-01-18T05:56:11-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:12 truenas k3s[10049]: I0118 05:56:12.464053   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:56:12 truenas k3s[10049]: E0118 05:56:12.464556   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:56:16 truenas k3s[10049]: time="2023-01-18T05:56:16-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:19 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:56:19.615-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:56:21 truenas k3s[10049]: time="2023-01-18T05:56:21-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:24 truenas k3s[10049]: I0118 05:56:24.464005   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:56:24 truenas k3s[10049]: E0118 05:56:24.464540   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:56:26 truenas k3s[10049]: time="2023-01-18T05:56:26-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:31 truenas k3s[10049]: time="2023-01-18T05:56:31-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:36 truenas k3s[10049]: time="2023-01-18T05:56:36-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:39 truenas k3s[10049]: I0118 05:56:39.464472   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:56:39 truenas k3s[10049]: E0118 05:56:39.464959   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:56:41 truenas k3s[10049]: time="2023-01-18T05:56:41-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:46 truenas k3s[10049]: time="2023-01-18T05:56:46-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:52 truenas k3s[10049]: time="2023-01-18T05:56:52-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:56:53 truenas k3s[10049]: I0118 05:56:53.464282   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:56:53 truenas k3s[10049]: E0118 05:56:53.464788   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:56:54 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:56:54.364-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:56:57 truenas k3s[10049]: time="2023-01-18T05:56:57-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:02 truenas k3s[10049]: time="2023-01-18T05:57:02-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:05 truenas k3s[10049]: I0118 05:57:05.463737   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:57:05 truenas k3s[10049]: E0118 05:57:05.464259   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:57:07 truenas k3s[10049]: time="2023-01-18T05:57:07-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:12 truenas k3s[10049]: time="2023-01-18T05:57:12-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:16 truenas k3s[10049]: I0118 05:57:16.463546   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:57:16 truenas k3s[10049]: E0118 05:57:16.464058   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:57:17 truenas k3s[10049]: time="2023-01-18T05:57:17-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:22 truenas k3s[10049]: time="2023-01-18T05:57:22-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:27 truenas k3s[10049]: time="2023-01-18T05:57:27-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:29 truenas k3s[10049]: I0118 05:57:29.464344   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:57:29 truenas k3s[10049]: E0118 05:57:29.464844   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:57:29 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:57:29.883-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:57:32 truenas k3s[10049]: time="2023-01-18T05:57:32-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:32 truenas k3s[10049]: I0118 05:57:32.168277   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:57:32 truenas k3s[10049]: E0118 05:57:32.176955   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 05:59:34.176907392 -0800 PST m=+7120.338317414 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:57:37 truenas k3s[10049]: time="2023-01-18T05:57:37-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:42 truenas k3s[10049]: time="2023-01-18T05:57:42-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:43 truenas k3s[10049]: I0118 05:57:43.464254   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:57:43 truenas k3s[10049]: E0118 05:57:43.464768   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:57:47 truenas k3s[10049]: time="2023-01-18T05:57:47-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:52 truenas k3s[10049]: time="2023-01-18T05:57:52-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:57:55 truenas k3s[10049]: I0118 05:57:55.464673   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:57:55 truenas k3s[10049]: E0118 05:57:55.465007   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:57:57 truenas k3s[10049]: time="2023-01-18T05:57:57-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:02 truenas k3s[10049]: time="2023-01-18T05:58:02-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:03 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:58:03.965-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:58:07 truenas k3s[10049]: time="2023-01-18T05:58:07-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:09 truenas k3s[10049]: I0118 05:58:09.464009   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:58:09 truenas k3s[10049]: E0118 05:58:09.464355   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:58:12 truenas k3s[10049]: time="2023-01-18T05:58:12-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:17 truenas k3s[10049]: time="2023-01-18T05:58:17-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:21 truenas k3s[10049]: I0118 05:58:21.464433   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:58:21 truenas k3s[10049]: E0118 05:58:21.464948   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:58:22 truenas k3s[10049]: time="2023-01-18T05:58:22-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:27 truenas k3s[10049]: time="2023-01-18T05:58:27-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:32 truenas k3s[10049]: time="2023-01-18T05:58:32-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:32 truenas k3s[10049]: I0118 05:58:32.464120   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:58:32 truenas k3s[10049]: E0118 05:58:32.464657   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:58:36 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:58:36.154-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:58:37 truenas k3s[10049]: time="2023-01-18T05:58:37-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:42 truenas k3s[10049]: time="2023-01-18T05:58:42-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:46 truenas k3s[10049]: I0118 05:58:46.464339   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:58:46 truenas k3s[10049]: E0118 05:58:46.464892   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:58:47 truenas k3s[10049]: time="2023-01-18T05:58:47-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:52 truenas k3s[10049]: time="2023-01-18T05:58:52-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:58:57 truenas k3s[10049]: time="2023-01-18T05:58:57-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:00 truenas k3s[10049]: I0118 05:59:00.463635   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:59:00 truenas k3s[10049]: E0118 05:59:00.464133   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:59:02 truenas k3s[10049]: time="2023-01-18T05:59:02-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:07 truenas k3s[10049]: time="2023-01-18T05:59:07-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:12 truenas k3s[10049]: time="2023-01-18T05:59:12-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:14 truenas k3s[10049]: I0118 05:59:14.464071   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:59:14 truenas k3s[10049]: E0118 05:59:14.464427   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:59:17 truenas k3s[10049]: time="2023-01-18T05:59:17-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:20 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:59:20.086-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:59:22 truenas k3s[10049]: time="2023-01-18T05:59:22-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:27 truenas k3s[10049]: I0118 05:59:27.464249   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 05:59:27 truenas k3s[10049]: time="2023-01-18T05:59:27-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:27 truenas k3s[10049]: E0118 05:59:27.818586   10049 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 18 05:59:27 truenas k3s[10049]:         rpc error: code = Unknown desc = failed to start container "894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 05:59:27 truenas k3s[10049]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 05:59:27 truenas k3s[10049]:  > containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 05:59:27 truenas k3s[10049]: E0118 05:59:27.818705   10049 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-4njt6,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c): RunContainerError: failed to start container "894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 05:59:27 truenas k3s[10049]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 05:59:27 truenas k3s[10049]: E0118 05:59:27.818813   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:59:32 truenas k3s[10049]: time="2023-01-18T05:59:32-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:34 truenas k3s[10049]: I0118 05:59:34.203655   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 05:59:34 truenas k3s[10049]: E0118 05:59:34.215719   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 06:01:36.215677404 -0800 PST m=+7242.377087470 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 05:59:37 truenas k3s[10049]: time="2023-01-18T05:59:37-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:41 truenas k3s[10049]: I0118 05:59:41.463817   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 05:59:41 truenas k3s[10049]: E0118 05:59:41.464336   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:59:42 truenas k3s[10049]: time="2023-01-18T05:59:42-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:47 truenas k3s[10049]: time="2023-01-18T05:59:47-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:50 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T05:59:50.510-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 05:59:52 truenas k3s[10049]: time="2023-01-18T05:59:52-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 05:59:53 truenas k3s[10049]: I0118 05:59:53.463677   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 05:59:53 truenas k3s[10049]: E0118 05:59:53.464232   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 05:59:57 truenas k3s[10049]: time="2023-01-18T05:59:57-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.115607   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created job nextcloud-cronjob-27900840"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.119734   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.137406   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900840" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-cronjob-27900840-9n8ws"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.137864   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.145151   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.146769   10049 topology_manager.go:205] "Topology Admit Handler"
Jan 18 06:00:00 truenas k3s[10049]: E0118 06:00:00.146911   10049 cpu_manager.go:394] "RemoveStaleState: removing container" podUID="ead5c4bb-d3c6-4842-8c25-2c1315382909" containerName="nextcloud"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.146915   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.147015   10049 memory_manager.go:345] "RemoveStaleState removing state" podUID="ead5c4bb-d3c6-4842-8c25-2c1315382909" containerName="nextcloud"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.160576   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.221397   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-s2dj6\" (UniqueName: \"kubernetes.io/projected/e068ece5-bb8b-48c4-86a1-eea81052efbd-kube-api-access-s2dj6\") pod \"nextcloud-cronjob-27900840-9n8ws\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") " pod="ix-nextcloud/nextcloud-cronjob-27900840-9n8ws"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.221451   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-cache\") pod \"nextcloud-cronjob-27900840-9n8ws\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") " pod="ix-nextcloud/nextcloud-cronjob-27900840-9n8ws"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.221492   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-shm\") pod \"nextcloud-cronjob-27900840-9n8ws\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") " pod="ix-nextcloud/nextcloud-cronjob-27900840-9n8ws"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.221619   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-varlogs\") pod \"nextcloud-cronjob-27900840-9n8ws\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") " pod="ix-nextcloud/nextcloud-cronjob-27900840-9n8ws"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.221680   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-varrun\") pod \"nextcloud-cronjob-27900840-9n8ws\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") " pod="ix-nextcloud/nextcloud-cronjob-27900840-9n8ws"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.221717   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-shared\") pod \"nextcloud-cronjob-27900840-9n8ws\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") " pod="ix-nextcloud/nextcloud-cronjob-27900840-9n8ws"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.221813   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-temp\") pod \"nextcloud-cronjob-27900840-9n8ws\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") " pod="ix-nextcloud/nextcloud-cronjob-27900840-9n8ws"
Jan 18 06:00:00 truenas k3s[10049]: time="2023-01-18T06:00:00-08:00" level=info msg="Will attempt to re-write config file /mnt/VMs/ix-applications/docker/containers/d60856fa64dcebe03d2120caab73be04d5795822150efe28b2348fb419bfdb29/resolv.conf as [nameserver 172.17.0.10 search ix-nextcloud.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Jan 18 06:00:00 truenas k3s[10049]: I0118 06:00:00.961146   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="d60856fa64dcebe03d2120caab73be04d5795822150efe28b2348fb419bfdb29"
Jan 18 06:00:01 truenas k3s[10049]: I0118 06:00:01.099742   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:01 truenas k3s[10049]: I0118 06:00:01.087815  501053 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ix-nextcloud", Name:"nextcloud-cronjob-27900840-9n8ws", UID:"e068ece5-bb8b-48c4-86a1-eea81052efbd", APIVersion:"v1", ResourceVersion:"29470211", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.5.144/16] from ix-net
Jan 18 06:00:01 truenas k3s[10049]: I0118 06:00:01.993849   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:02 truenas k3s[10049]: time="2023-01-18T06:00:02-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:03 truenas k3s[10049]: I0118 06:00:03.005107   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:03 truenas k3s[10049]: I0118 06:00:03.038075   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.047106   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357276   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-cache\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357373   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-varlogs\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357415   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-varrun\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357528   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"html\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357569   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-temp\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357612   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-shared\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357737   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-shm\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357778   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"kube-api-access-s2dj6\" (UniqueName: \"kubernetes.io/projected/e068ece5-bb8b-48c4-86a1-eea81052efbd-kube-api-access-s2dj6\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.357890   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"data\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586\") pod \"e068ece5-bb8b-48c4-86a1-eea81052efbd\" (UID: \"e068ece5-bb8b-48c4-86a1-eea81052efbd\") "
Jan 18 06:00:04 truenas k3s[10049]: W0118 06:00:04.358201   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/e068ece5-bb8b-48c4-86a1-eea81052efbd/volumes/kubernetes.io~empty-dir/varlogs: clearQuota called, but quotas disabled
Jan 18 06:00:04 truenas k3s[10049]: W0118 06:00:04.358247   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/e068ece5-bb8b-48c4-86a1-eea81052efbd/volumes/kubernetes.io~empty-dir/shared: clearQuota called, but quotas disabled
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.358401   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-varlogs" (OuterVolumeSpecName: "varlogs") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "varlogs". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: W0118 06:00:04.358416   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/e068ece5-bb8b-48c4-86a1-eea81052efbd/volumes/kubernetes.io~empty-dir/temp: clearQuota called, but quotas disabled
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.358525   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-shared" (OuterVolumeSpecName: "shared") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "shared". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.358671   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-temp" (OuterVolumeSpecName: "temp") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "temp". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.360740   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-cache" (OuterVolumeSpecName: "cache") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "cache". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.361214   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-shm" (OuterVolumeSpecName: "shm") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "shm". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.361425   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-varrun" (OuterVolumeSpecName: "varrun") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "varrun". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.361527   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/e068ece5-bb8b-48c4-86a1-eea81052efbd-kube-api-access-s2dj6" (OuterVolumeSpecName: "kube-api-access-s2dj6") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "kube-api-access-s2dj6". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.410194   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586" (OuterVolumeSpecName: "data") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.410466   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68" (OuterVolumeSpecName: "html") pod "e068ece5-bb8b-48c4-86a1-eea81052efbd" (UID: "e068ece5-bb8b-48c4-86a1-eea81052efbd"). InnerVolumeSpecName "pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.458606   10049 reconciler.go:399] "Volume detached for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-shared\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.458685   10049 reconciler.go:399] "Volume detached for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-shm\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.458725   10049 reconciler.go:399] "Volume detached for volume \"kube-api-access-s2dj6\" (UniqueName: \"kubernetes.io/projected/e068ece5-bb8b-48c4-86a1-eea81052efbd-kube-api-access-s2dj6\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.458760   10049 reconciler.go:399] "Volume detached for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-cache\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.458810   10049 reconciler.go:399] "Volume detached for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-varlogs\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.458858   10049 reconciler.go:399] "Volume detached for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-varrun\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:00:04 truenas k3s[10049]: I0118 06:00:04.458904   10049 reconciler.go:399] "Volume detached for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/e068ece5-bb8b-48c4-86a1-eea81052efbd-temp\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:00:05 truenas k3s[10049]: I0118 06:00:05.059445   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="d60856fa64dcebe03d2120caab73be04d5795822150efe28b2348fb419bfdb29"
Jan 18 06:00:05 truenas k3s[10049]: I0118 06:00:05.092564   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:05 truenas k3s[10049]: I0118 06:00:05.769519   10049 scope.go:115] "RemoveContainer" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.092060   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="e8cbb7bf6865c8432922a200598156013e1e5e830604cdd8528a18d8373d5c5a"
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.092656   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:00:06 truenas k3s[10049]: E0118 06:00:06.093146   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.103646   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.116576   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900840" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="Completed" message="Job completed"
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.117046   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.124913   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900840
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.125171   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-27900840, status: Complete"
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.132390   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulDelete" message="Deleted job nextcloud-cronjob-27900830"
Jan 18 06:00:06 truenas k3s[10049]: I0118 06:00:06.132870   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900830
Jan 18 06:00:06 truenas k3s[10049]: E0118 06:00:06.133646   10049 job_controller.go:545] syncing job: tracking status: adding uncounted pods to status: Operation cannot be fulfilled on jobs.batch "nextcloud-cronjob-27900840": the object has been modified; please apply your changes to the latest version and try again
Jan 18 06:00:07 truenas k3s[10049]: I0118 06:00:07.497738   10049 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=1fe74375-7d23-4612-8884-cdefc0ecc3d1 path="/var/lib/kubelet/pods/1fe74375-7d23-4612-8884-cdefc0ecc3d1/volumes"
Jan 18 06:00:07 truenas k3s[10049]: time="2023-01-18T06:00:07-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:12 truenas k3s[10049]: time="2023-01-18T06:00:12-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:17 truenas k3s[10049]: time="2023-01-18T06:00:17-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:20 truenas k3s[10049]: I0118 06:00:20.464277   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:00:20 truenas k3s[10049]: E0118 06:00:20.464792   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:00:22 truenas k3s[10049]: time="2023-01-18T06:00:22-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:27 truenas k3s[10049]: time="2023-01-18T06:00:27-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:31 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:00:31.259-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:00:32 truenas k3s[10049]: I0118 06:00:32.463481   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:00:32 truenas k3s[10049]: E0118 06:00:32.463983   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:00:32 truenas k3s[10049]: time="2023-01-18T06:00:32-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:37 truenas k3s[10049]: time="2023-01-18T06:00:37-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:42 truenas k3s[10049]: time="2023-01-18T06:00:42-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:44 truenas k3s[10049]: I0118 06:00:44.463652   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:00:44 truenas k3s[10049]: E0118 06:00:44.464000   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:00:47 truenas k3s[10049]: time="2023-01-18T06:00:47-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:52 truenas k3s[10049]: time="2023-01-18T06:00:52-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:00:55 truenas k3s[10049]: I0118 06:00:55.464463   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:00:55 truenas k3s[10049]: E0118 06:00:55.464972   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:00:57 truenas k3s[10049]: time="2023-01-18T06:00:57-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:01 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:01:01.647-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:01:02 truenas k3s[10049]: time="2023-01-18T06:01:02-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:05 truenas k3s[10049]: I0118 06:01:05.830187   10049 scope.go:115] "RemoveContainer" containerID="6d816b4d6c49e28c4638a7e6c187c5a827066853f9cf39984e150fb0252e5692"
Jan 18 06:01:07 truenas k3s[10049]: time="2023-01-18T06:01:07-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:10 truenas k3s[10049]: I0118 06:01:10.464486   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:01:10 truenas k3s[10049]: E0118 06:01:10.464989   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:01:12 truenas k3s[10049]: time="2023-01-18T06:01:12-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:17 truenas k3s[10049]: time="2023-01-18T06:01:17-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:22 truenas k3s[10049]: time="2023-01-18T06:01:22-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:23 truenas k3s[10049]: I0118 06:01:23.463856   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:01:23 truenas k3s[10049]: E0118 06:01:23.464354   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:01:27 truenas k3s[10049]: time="2023-01-18T06:01:27-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:32 truenas k3s[10049]: time="2023-01-18T06:01:32-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:36 truenas k3s[10049]: I0118 06:01:36.244432   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 06:01:36 truenas k3s[10049]: E0118 06:01:36.252343   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 06:03:38.252315143 -0800 PST m=+7364.413725156 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 06:01:37 truenas k3s[10049]: I0118 06:01:37.464503   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:01:37 truenas k3s[10049]: E0118 06:01:37.465021   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:01:37 truenas k3s[10049]: time="2023-01-18T06:01:37-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:42 truenas k3s[10049]: time="2023-01-18T06:01:42-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:44 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:01:44.473-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:01:47 truenas k3s[10049]: time="2023-01-18T06:01:47-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:48 truenas k3s[10049]: I0118 06:01:48.463422   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:01:48 truenas k3s[10049]: E0118 06:01:48.463740   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:01:52 truenas k3s[10049]: time="2023-01-18T06:01:52-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:01:57 truenas k3s[10049]: time="2023-01-18T06:01:57-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:02 truenas k3s[10049]: I0118 06:02:02.464359   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:02:02 truenas k3s[10049]: E0118 06:02:02.465122   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:02:03 truenas k3s[10049]: time="2023-01-18T06:02:03-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:08 truenas k3s[10049]: time="2023-01-18T06:02:08-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:13 truenas k3s[10049]: time="2023-01-18T06:02:13-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:14 truenas k3s[10049]: I0118 06:02:14.463920   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:02:14 truenas k3s[10049]: E0118 06:02:14.464231   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:02:18 truenas k3s[10049]: time="2023-01-18T06:02:18-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:19 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:02:19.114-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:02:23 truenas k3s[10049]: time="2023-01-18T06:02:23-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:25 truenas k3s[10049]: I0118 06:02:25.463272   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:02:25 truenas k3s[10049]: E0118 06:02:25.463563   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:02:28 truenas k3s[10049]: time="2023-01-18T06:02:28-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:33 truenas k3s[10049]: time="2023-01-18T06:02:33-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:38 truenas k3s[10049]: time="2023-01-18T06:02:38-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:40 truenas k3s[10049]: I0118 06:02:40.463803   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:02:40 truenas k3s[10049]: E0118 06:02:40.464309   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:02:43 truenas k3s[10049]: time="2023-01-18T06:02:43-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:48 truenas k3s[10049]: time="2023-01-18T06:02:48-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:53 truenas k3s[10049]: time="2023-01-18T06:02:53-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:02:53 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:02:53.928-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:02:55 truenas k3s[10049]: I0118 06:02:55.463898   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:02:55 truenas k3s[10049]: E0118 06:02:55.464414   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:02:58 truenas k3s[10049]: time="2023-01-18T06:02:58-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:03 truenas k3s[10049]: time="2023-01-18T06:03:03-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:07 truenas k3s[10049]: I0118 06:03:07.463933   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:03:07 truenas k3s[10049]: E0118 06:03:07.464435   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:03:08 truenas k3s[10049]: time="2023-01-18T06:03:08-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:13 truenas k3s[10049]: time="2023-01-18T06:03:13-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:18 truenas k3s[10049]: time="2023-01-18T06:03:18-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:22 truenas k3s[10049]: I0118 06:03:22.464345   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:03:22 truenas k3s[10049]: E0118 06:03:22.464851   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:03:23 truenas k3s[10049]: time="2023-01-18T06:03:23-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:25 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:03:25.639-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:03:28 truenas k3s[10049]: time="2023-01-18T06:03:28-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:33 truenas k3s[10049]: time="2023-01-18T06:03:33-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:35 truenas k3s[10049]: I0118 06:03:35.464137   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:03:35 truenas k3s[10049]: E0118 06:03:35.464889   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:03:38 truenas k3s[10049]: I0118 06:03:38.287282   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 06:03:38 truenas k3s[10049]: E0118 06:03:38.296271   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 06:05:40.296245279 -0800 PST m=+7486.457655312 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 06:03:38 truenas k3s[10049]: time="2023-01-18T06:03:38-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:43 truenas k3s[10049]: time="2023-01-18T06:03:43-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:48 truenas k3s[10049]: time="2023-01-18T06:03:48-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:48 truenas k3s[10049]: I0118 06:03:48.463440   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:03:48 truenas k3s[10049]: E0118 06:03:48.463948   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:03:53 truenas k3s[10049]: time="2023-01-18T06:03:53-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:03:58 truenas k3s[10049]: time="2023-01-18T06:03:58-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:00 truenas k3s[10049]: I0118 06:04:00.463780   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:04:00 truenas k3s[10049]: E0118 06:04:00.464281   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:04:03 truenas k3s[10049]: time="2023-01-18T06:04:03-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:06 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:04:06.136-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:04:08 truenas k3s[10049]: time="2023-01-18T06:04:08-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:13 truenas k3s[10049]: time="2023-01-18T06:04:13-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:15 truenas k3s[10049]: I0118 06:04:15.463477   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:04:15 truenas k3s[10049]: E0118 06:04:15.463779   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:04:18 truenas k3s[10049]: time="2023-01-18T06:04:18-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:23 truenas k3s[10049]: time="2023-01-18T06:04:23-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:28 truenas k3s[10049]: time="2023-01-18T06:04:28-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:30 truenas k3s[10049]: I0118 06:04:30.463913   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:04:30 truenas k3s[10049]: E0118 06:04:30.827409   10049 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 18 06:04:30 truenas k3s[10049]:         rpc error: code = Unknown desc = failed to start container "557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 06:04:30 truenas k3s[10049]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 06:04:30 truenas k3s[10049]:  > containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:04:30 truenas k3s[10049]: E0118 06:04:30.827517   10049 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-4njt6,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c): RunContainerError: failed to start container "557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 18 06:04:30 truenas k3s[10049]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 18 06:04:30 truenas k3s[10049]: E0118 06:04:30.827605   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:04:33 truenas k3s[10049]: time="2023-01-18T06:04:33-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:38 truenas k3s[10049]: time="2023-01-18T06:04:38-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:43 truenas k3s[10049]: time="2023-01-18T06:04:43-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:44 truenas k3s[10049]: I0118 06:04:44.463718   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:04:44 truenas k3s[10049]: E0118 06:04:44.464063   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:04:45 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:04:45.324-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:04:48 truenas k3s[10049]: time="2023-01-18T06:04:48-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:53 truenas k3s[10049]: time="2023-01-18T06:04:53-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:58 truenas k3s[10049]: time="2023-01-18T06:04:58-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:04:59 truenas k3s[10049]: I0118 06:04:59.464418   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:04:59 truenas k3s[10049]: E0118 06:04:59.464909   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.113889   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created job nextcloud-cronjob-27900845"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.114665   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.127971   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900845" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-cronjob-27900845-w579j"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.128613   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.136017   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.137949   10049 topology_manager.go:205] "Topology Admit Handler"
Jan 18 06:05:00 truenas k3s[10049]: E0118 06:05:00.138073   10049 cpu_manager.go:394] "RemoveStaleState: removing container" podUID="e068ece5-bb8b-48c4-86a1-eea81052efbd" containerName="nextcloud"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.138177   10049 memory_manager.go:345] "RemoveStaleState removing state" podUID="e068ece5-bb8b-48c4-86a1-eea81052efbd" containerName="nextcloud"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.139780   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.156924   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.286239   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-temp\") pod \"nextcloud-cronjob-27900845-w579j\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") " pod="ix-nextcloud/nextcloud-cronjob-27900845-w579j"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.286293   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-varlogs\") pod \"nextcloud-cronjob-27900845-w579j\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") " pod="ix-nextcloud/nextcloud-cronjob-27900845-w579j"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.286493   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-varrun\") pod \"nextcloud-cronjob-27900845-w579j\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") " pod="ix-nextcloud/nextcloud-cronjob-27900845-w579j"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.286575   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-shared\") pod \"nextcloud-cronjob-27900845-w579j\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") " pod="ix-nextcloud/nextcloud-cronjob-27900845-w579j"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.286703   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-cache\") pod \"nextcloud-cronjob-27900845-w579j\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") " pod="ix-nextcloud/nextcloud-cronjob-27900845-w579j"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.286767   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-shm\") pod \"nextcloud-cronjob-27900845-w579j\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") " pod="ix-nextcloud/nextcloud-cronjob-27900845-w579j"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.286880   10049 reconciler.go:357] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-dx6cr\" (UniqueName: \"kubernetes.io/projected/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-kube-api-access-dx6cr\") pod \"nextcloud-cronjob-27900845-w579j\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") " pod="ix-nextcloud/nextcloud-cronjob-27900845-w579j"
Jan 18 06:05:00 truenas k3s[10049]: time="2023-01-18T06:05:00-08:00" level=info msg="Will attempt to re-write config file /mnt/VMs/ix-applications/docker/containers/4ee8fc924c0320b7a67ea274203a9a26d95983cc2640903b801ea7eba20b28e2/resolv.conf as [nameserver 172.17.0.10 search ix-nextcloud.svc.cluster.local svc.cluster.local cluster.local options ndots:5]"
Jan 18 06:05:00 truenas k3s[10049]: I0118 06:05:00.968804   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="4ee8fc924c0320b7a67ea274203a9a26d95983cc2640903b801ea7eba20b28e2"
Jan 18 06:05:01 truenas k3s[10049]: I0118 06:05:01.107029   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:01 truenas k3s[10049]: I0118 06:05:01.095680  520072 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ix-nextcloud", Name:"nextcloud-cronjob-27900845-w579j", UID:"73f1aac6-738e-4d1e-a0f0-f2caa2df987f", APIVersion:"v1", ResourceVersion:"29470703", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.5.145/16] from ix-net
Jan 18 06:05:02 truenas k3s[10049]: I0118 06:05:02.001887   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:03 truenas k3s[10049]: I0118 06:05:03.013348   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:03 truenas k3s[10049]: I0118 06:05:03.045016   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:03 truenas k3s[10049]: time="2023-01-18T06:05:03-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.054655   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418036   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-varlogs\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418096   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-cache\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418137   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-varrun\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418322   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"kube-api-access-dx6cr\" (UniqueName: \"kubernetes.io/projected/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-kube-api-access-dx6cr\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418425   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"data\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418469   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-temp\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418512   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-shared\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418610   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"html\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418651   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-shm\") pod \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\" (UID: \"73f1aac6-738e-4d1e-a0f0-f2caa2df987f\") "
Jan 18 06:05:04 truenas k3s[10049]: W0118 06:05:04.418701   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/73f1aac6-738e-4d1e-a0f0-f2caa2df987f/volumes/kubernetes.io~empty-dir/varlogs: clearQuota called, but quotas disabled
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.418865   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-varlogs" (OuterVolumeSpecName: "varlogs") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "varlogs". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: W0118 06:05:04.419138   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/73f1aac6-738e-4d1e-a0f0-f2caa2df987f/volumes/kubernetes.io~empty-dir/shared: clearQuota called, but quotas disabled
Jan 18 06:05:04 truenas k3s[10049]: W0118 06:05:04.419234   10049 empty_dir.go:523] Warning: Failed to clear quota on /var/lib/kubelet/pods/73f1aac6-738e-4d1e-a0f0-f2caa2df987f/volumes/kubernetes.io~empty-dir/temp: clearQuota called, but quotas disabled
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.419231   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-shared" (OuterVolumeSpecName: "shared") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "shared". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.419358   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-temp" (OuterVolumeSpecName: "temp") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "temp". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.421529   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-cache" (OuterVolumeSpecName: "cache") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "cache". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.421690   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-varrun" (OuterVolumeSpecName: "varrun") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "varrun". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.421793   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-shm" (OuterVolumeSpecName: "shm") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "shm". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.421802   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-kube-api-access-dx6cr" (OuterVolumeSpecName: "kube-api-access-dx6cr") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "kube-api-access-dx6cr". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.471666   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586" (OuterVolumeSpecName: "data") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "pvc-ddb18e62-2a5f-4e59-8d57-0b4ab557c586". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.472014   10049 operation_generator.go:890] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68" (OuterVolumeSpecName: "html") pod "73f1aac6-738e-4d1e-a0f0-f2caa2df987f" (UID: "73f1aac6-738e-4d1e-a0f0-f2caa2df987f"). InnerVolumeSpecName "pvc-0bebe0a2-60b5-4b01-837b-76c24e167b68". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.519607   10049 reconciler.go:399] "Volume detached for volume \"shm\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-shm\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.519640   10049 reconciler.go:399] "Volume detached for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-varlogs\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.519661   10049 reconciler.go:399] "Volume detached for volume \"cache\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-cache\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.519680   10049 reconciler.go:399] "Volume detached for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-varrun\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.519700   10049 reconciler.go:399] "Volume detached for volume \"kube-api-access-dx6cr\" (UniqueName: \"kubernetes.io/projected/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-kube-api-access-dx6cr\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.519717   10049 reconciler.go:399] "Volume detached for volume \"temp\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-temp\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:05:04 truenas k3s[10049]: I0118 06:05:04.519734   10049 reconciler.go:399] "Volume detached for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/73f1aac6-738e-4d1e-a0f0-f2caa2df987f-shared\") on node \"ix-truenas\" DevicePath \"\""
Jan 18 06:05:05 truenas k3s[10049]: I0118 06:05:05.072153   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="4ee8fc924c0320b7a67ea274203a9a26d95983cc2640903b801ea7eba20b28e2"
Jan 18 06:05:05 truenas k3s[10049]: I0118 06:05:05.092161   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.020815   10049 scope.go:115] "RemoveContainer" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.104214   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.106082   10049 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="894437b326b7abef29dc646025546d90d821cbbeb06b914ae030179c3f7f18cd"
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.106682   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:05:06 truenas k3s[10049]: E0118 06:05:06.107254   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.117347   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-27900845" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="Completed" message="Job completed"
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.117759   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.126684   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900845
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.126990   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-27900845, status: Complete"
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.133376   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulDelete" message="Deleted job nextcloud-cronjob-27900835"
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.134016   10049 job_controller.go:510] enqueueing job ix-nextcloud/nextcloud-cronjob-27900835
Jan 18 06:05:06 truenas k3s[10049]: E0118 06:05:06.135244   10049 job_controller.go:545] syncing job: tracking status: adding uncounted pods to status: Operation cannot be fulfilled on jobs.batch "nextcloud-cronjob-27900845": the object has been modified; please apply your changes to the latest version and try again
Jan 18 06:05:06 truenas k3s[10049]: I0118 06:05:06.142919   10049 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-27900845, status: Complete"
Jan 18 06:05:06 truenas k3s[10049]: E0118 06:05:06.154429   10049 cronjob_controllerv2.go:164] error syncing CronJobController ix-nextcloud/nextcloud-cronjob, requeuing: Operation cannot be fulfilled on cronjobs.batch "nextcloud-cronjob": the object has been modified; please apply your changes to the latest version and try again
Jan 18 06:05:06 truenas k3s[10049]: E0118 06:05:06.173227   10049 cronjob_controllerv2.go:164] error syncing CronJobController ix-nextcloud/nextcloud-cronjob, requeuing: Operation cannot be fulfilled on cronjobs.batch "nextcloud-cronjob": the object has been modified; please apply your changes to the latest version and try again
Jan 18 06:05:07 truenas k3s[10049]: I0118 06:05:07.498711   10049 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=ead5c4bb-d3c6-4842-8c25-2c1315382909 path="/var/lib/kubelet/pods/ead5c4bb-d3c6-4842-8c25-2c1315382909/volumes"
Jan 18 06:05:08 truenas k3s[10049]: time="2023-01-18T06:05:08-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:13 truenas k3s[10049]: time="2023-01-18T06:05:13-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:18 truenas k3s[10049]: time="2023-01-18T06:05:18-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:21 truenas k3s[10049]: I0118 06:05:21.463882   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:05:21 truenas k3s[10049]: E0118 06:05:21.464385   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:05:22 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:05:22.048-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:05:23 truenas k3s[10049]: time="2023-01-18T06:05:23-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:28 truenas k3s[10049]: time="2023-01-18T06:05:28-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:33 truenas k3s[10049]: time="2023-01-18T06:05:33-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:35 truenas k3s[10049]: I0118 06:05:35.463904   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:05:35 truenas k3s[10049]: E0118 06:05:35.464673   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:05:38 truenas k3s[10049]: time="2023-01-18T06:05:38-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:40 truenas k3s[10049]: I0118 06:05:40.330181   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 06:05:40 truenas k3s[10049]: E0118 06:05:40.337591   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 06:07:42.337569088 -0800 PST m=+7608.498979113 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 06:05:43 truenas k3s[10049]: time="2023-01-18T06:05:43-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:48 truenas k3s[10049]: time="2023-01-18T06:05:48-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:49 truenas k3s[10049]: I0118 06:05:49.466072   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:05:49 truenas k3s[10049]: E0118 06:05:49.466869   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:05:53 truenas k3s[10049]: time="2023-01-18T06:05:53-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:05:57 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:05:57.146-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:05:58 truenas k3s[10049]: time="2023-01-18T06:05:58-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:00 truenas k3s[10049]: I0118 06:06:00.463485   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:06:00 truenas k3s[10049]: E0118 06:06:00.464004   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:06:03 truenas k3s[10049]: time="2023-01-18T06:06:03-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:06 truenas k3s[10049]: I0118 06:06:06.092460   10049 scope.go:115] "RemoveContainer" containerID="828b5b03cf0f1342b1e8772a217c6b87ac88271db5d7f983a6a6817cf3a372d4"
Jan 18 06:06:08 truenas k3s[10049]: time="2023-01-18T06:06:08-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:11 truenas k3s[10049]: I0118 06:06:11.463718   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:06:11 truenas k3s[10049]: E0118 06:06:11.464215   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:06:13 truenas k3s[10049]: time="2023-01-18T06:06:13-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:18 truenas k3s[10049]: time="2023-01-18T06:06:18-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:23 truenas k3s[10049]: time="2023-01-18T06:06:23-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:25 truenas k3s[10049]: I0118 06:06:25.464276   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:06:25 truenas k3s[10049]: E0118 06:06:25.464775   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:06:27 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:06:27.304-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:06:28 truenas k3s[10049]: time="2023-01-18T06:06:28-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:33 truenas k3s[10049]: time="2023-01-18T06:06:33-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:38 truenas k3s[10049]: time="2023-01-18T06:06:38-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:39 truenas k3s[10049]: I0118 06:06:39.464859   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:06:39 truenas k3s[10049]: E0118 06:06:39.465378   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:06:43 truenas k3s[10049]: time="2023-01-18T06:06:43-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:48 truenas k3s[10049]: time="2023-01-18T06:06:48-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:50 truenas k3s[10049]: I0118 06:06:50.463587   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:06:50 truenas k3s[10049]: E0118 06:06:50.464102   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:06:53 truenas k3s[10049]: time="2023-01-18T06:06:53-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:58 truenas k3s[10049]: time="2023-01-18T06:06:58-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:06:59 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:06:59.570-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:07:02 truenas k3s[10049]: I0118 06:07:02.463996   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:07:02 truenas k3s[10049]: E0118 06:07:02.464504   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:07:03 truenas k3s[10049]: time="2023-01-18T06:07:03-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:08 truenas k3s[10049]: time="2023-01-18T06:07:08-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:14 truenas k3s[10049]: time="2023-01-18T06:07:14-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:14 truenas k3s[10049]: I0118 06:07:14.463601   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:07:14 truenas k3s[10049]: E0118 06:07:14.463965   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:07:19 truenas k3s[10049]: time="2023-01-18T06:07:19-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:24 truenas k3s[10049]: time="2023-01-18T06:07:24-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:28 truenas k3s[10049]: I0118 06:07:28.464485   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:07:28 truenas k3s[10049]: E0118 06:07:28.465242   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:07:29 truenas k3s[10049]: time="2023-01-18T06:07:29-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:32 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:07:32.241-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:07:34 truenas k3s[10049]: time="2023-01-18T06:07:34-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:39 truenas k3s[10049]: time="2023-01-18T06:07:39-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:41 truenas k3s[10049]: I0118 06:07:41.464172   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:07:41 truenas k3s[10049]: E0118 06:07:41.464925   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:07:42 truenas k3s[10049]: I0118 06:07:42.347484   10049 reconciler.go:211] "operationExecutor.UnmountVolume started for volume \"pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96\") pod \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\" (UID: \"fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe\") "
Jan 18 06:07:42 truenas k3s[10049]: E0118 06:07:42.359070   10049 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96 podName:fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe nodeName:}" failed. No retries permitted until 2023-01-18 06:09:44.35902916 -0800 PST m=+7730.520439226 (durationBeforeRetry 2m2s). Error: UnmountVolume.TearDown failed for volume "pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96") pod "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe" (UID: "fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe") : kubernetes.io/csi: Unmounter.TearDownAt failed to clean mount dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: kubernetes.io/csi: failed to remove dir [/var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount]: remove /var/lib/kubelet/pods/fcfb5f28-bd22-48c5-91f6-17b8dd6d53fe/volumes/kubernetes.io~csi/pvc-241d4ec5-913d-4e5a-b314-4607f92b2f96/mount: directory not empty
Jan 18 06:07:44 truenas k3s[10049]: time="2023-01-18T06:07:44-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:49 truenas k3s[10049]: time="2023-01-18T06:07:49-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:54 truenas k3s[10049]: time="2023-01-18T06:07:54-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:07:55 truenas k3s[10049]: I0118 06:07:55.463800   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:07:55 truenas k3s[10049]: E0118 06:07:55.464326   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:07:59 truenas k3s[10049]: time="2023-01-18T06:07:59-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:08:02 truenas k3s[10049]: {"level":"warn","ts":"2023-01-18T06:08:02.656-0800","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014f81c0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 18 06:08:04 truenas k3s[10049]: time="2023-01-18T06:08:04-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:08:08 truenas k3s[10049]: I0118 06:08:08.464486   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:08:08 truenas k3s[10049]: E0118 06:08:08.464984   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:08:09 truenas k3s[10049]: time="2023-01-18T06:08:09-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:08:14 truenas k3s[10049]: time="2023-01-18T06:08:14-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:08:19 truenas k3s[10049]: time="2023-01-18T06:08:19-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:08:22 truenas k3s[10049]: I0118 06:08:22.463932   10049 scope.go:115] "RemoveContainer" containerID="557e3efde8b7a7d2da4ca016c0eef3d5dd9aa1b4d9516f48083873207d6c403f"
Jan 18 06:08:22 truenas k3s[10049]: E0118 06:08:22.464442   10049 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-55rxn_kube-system(f88bb792-6319-4f42-b4ed-8d3d8b15f26c)\"" pod="kube-system/nvidia-device-plugin-daemonset-55rxn" podUID=f88bb792-6319-4f42-b4ed-8d3d8b15f26c
Jan 18 06:08:24 truenas k3s[10049]: time="2023-01-18T06:08:24-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 18 06:08:29 truenas k3s[10049]: time="2023-01-18T06:08:29-08:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
debug finished in 0 seconds for K3s Logs (last 1000 lines)


