
-------------
1) K3s logs
-------------

Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.551802    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for volumesnapshots.snapshot.storage.k8s.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.551833    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for tlsoptions.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.551860    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for endpoints
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552042    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingresses.networking.k8s.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552117    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for orders.acme.cert-manager.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552139    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for thanosrulers.monitoring.coreos.com
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552159    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsbackups.zfs.openebs.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552176    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsvolumes.zfs.openebs.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552202    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for podtemplates
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552217    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for statefulsets.apps
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552258    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for csistoragecapacities.storage.k8s.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552291    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for podmonitors.monitoring.coreos.com
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552320    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for middlewaretcps.traefik.containo.us
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552352    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsnodes.zfs.openebs.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552406    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for daemonsets.apps
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552589    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for horizontalpodautoscalers.autoscaling
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552610    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for challenges.acme.cert-manager.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552631    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for helmcharts.helm.cattle.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552650    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for servicemonitors.monitoring.coreos.com
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552674    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingressrouteudps.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552691    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for cronjobs.batch
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552712    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for addons.k3s.cattle.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552739    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingressroutetcps.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552785    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for limitranges
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552822    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for poddisruptionbudgets.policy
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552847    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for endpointslices.discovery.k8s.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552875    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for middlewaretcps.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552904    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for networkpolicies.networking.k8s.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552932    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for traefikservices.traefik.containo.us
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552959    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingressroutes.traefik.containo.us
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552977    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for serverstransports.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.552993    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for middlewares.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553011    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for network-attachment-definitions.k8s.cni.cncf.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553027    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for tlsstores.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553044    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for serverstransporttcps.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553112    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for serviceaccounts
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553132    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for roles.rbac.authorization.k8s.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553150    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for alertmanagerconfigs.monitoring.coreos.com
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553169    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfssnapshots.zfs.openebs.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553196    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for deployments.apps
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553220    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for tlsstores.traefik.containo.us
Jan 22 12:39:58 truenas k3s[5440]: W0122 12:39:58.553372    5440 shared_informer.go:548] resyncPeriod 16h50m11.317134259s is smaller than resyncCheckPeriod 17h19m37.293619594s and the informer has already started. Changing it to 17h19m37.293619594s
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553458    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for controllerrevisions.apps
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553492    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for rolebindings.rbac.authorization.k8s.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553519    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingressroutes.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553540    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for leases.coordination.k8s.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553573    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for certificaterequests.cert-manager.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553602    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for certificates.cert-manager.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553631    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for serverstransports.traefik.containo.us
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553659    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for middlewares.traefik.containo.us
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553691    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for replicasets.apps
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553717    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for helmchartconfigs.helm.cattle.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553747    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for prometheuses.monitoring.coreos.com
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553773    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsrestores.zfs.openebs.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553799    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingressroutetcps.traefik.containo.us
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553825    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for traefikservices.traefik.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553849    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for issuers.cert-manager.io
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553874    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for prometheusrules.monitoring.coreos.com
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553896    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for jobs.batch
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553922    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for alertmanagers.monitoring.coreos.com
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553948    5440 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for tlsoptions.traefik.containo.us
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.553967    5440 controllermanager.go:622] Started "resourcequota"
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.554016    5440 resource_quota_controller.go:277] Starting resource quota controller
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.554032    5440 shared_informer.go:270] Waiting for caches to sync for resource quota
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.554071    5440 resource_quota_monitor.go:295] QuotaMonitor running
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.556726    5440 controllermanager.go:622] Started "csrcleaner"
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.557047    5440 cleaner.go:82] Starting CSR cleaner controller
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.587260    5440 shared_informer.go:277] Caches are synced for tokens
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.930558    5440 serving.go:355] Generated self-signed cert in-memory
Jan 22 12:39:58 truenas k3s[5440]: I0122 12:39:58.948148    5440 event.go:294] "Event occurred" object="ix-truenas" fieldPath="" kind="Node" apiVersion="" type="Normal" reason="NodePasswordValidationComplete" message="Deferred node password secret validation complete"
Jan 22 12:39:59 truenas k3s[5440]: time="2024-01-22T12:39:59+01:00" level=info msg="containerd is now running"
Jan 22 12:39:59 truenas k3s[5440]: time="2024-01-22T12:39:59+01:00" level=info msg="Connecting to proxy" url="wss://127.0.0.1:6443/v1-k3s/connect"
Jan 22 12:39:59 truenas k3s[5440]: time="2024-01-22T12:39:59+01:00" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/mnt/SSD Hector Pool/ix-applications/k3s/agent/client-ca.crt --cluster-dns=172.17.0.10 --cluster-domain=cluster.local --config=/etc/rancher/k3s/kubelet_config.yaml --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ix-truenas --kubeconfig=/mnt/SSD Hector Pool/ix-applications/k3s/agent/kubelet.kubeconfig --node-labels= --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/mnt/SSD Hector Pool/ix-applications/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/etc/resolv.conf --serialize-image-pulls=false --tls-cert-file=/mnt/SSD Hector Pool/ix-applications/k3s/agent/serving-kubelet.crt --tls-private-key-file=/mnt/SSD Hector Pool/ix-applications/k3s/agent/serving-kubelet.key"
Jan 22 12:39:59 truenas k3s[5440]: time="2024-01-22T12:39:59+01:00" level=info msg="Handling backend connection request [ix-truenas]"
Jan 22 12:39:59 truenas k3s[5440]: Flag --containerd has been deprecated, This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed.
Jan 22 12:39:59 truenas k3s[5440]: Flag --pod-infra-container-image has been deprecated, will be removed in 1.27. Image garbage collector will get sandbox image information from CRI.
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.426034    5440 server.go:197] "--pod-infra-container-image will not be pruned by the image garbage collector in kubelet and should also be set in the remote runtime"
Jan 22 12:39:59 truenas k3s[5440]: time="2024-01-22T12:39:59+01:00" level=info msg="Annotations and labels have already set on node: ix-truenas"
Jan 22 12:39:59 truenas k3s[5440]: Flag --pod-infra-container-image has been deprecated, will be removed in 1.27. Image garbage collector will get sandbox image information from CRI.
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.436017    5440 server.go:407] "Kubelet version" kubeletVersion="v1.26.6+k3s-e18037a7-dirty"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.436050    5440 server.go:409] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.437747    5440 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/SSD Hector Pool/ix-applications/k3s/agent/client-ca.crt"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.446750    5440 server.go:654] "--cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.448412    5440 container_manager_linux.go:267] "Container manager verified user specified cgroup-root exists" cgroupRoot=[]
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.448621    5440 container_manager_linux.go:272] "Creating Container Manager object based on Node Config" nodeConfig={RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: KubeletOOMScoreAdj:-999 ContainerRuntime: CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:systemd KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:true NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] CPUManagerPolicy:none CPUManagerPolicyOptions:map[] ExperimentalTopologyManagerScope:container CPUManagerReconcilePeriod:10s ExperimentalMemoryManagerPolicy:None ExperimentalMemoryManagerReservedMemory:[] ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none ExperimentalTopologyManagerPolicyOptions:map[]}
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.449443    5440 topology_manager.go:134] "Creating topology manager with policy per scope" topologyPolicyName="none" topologyScopeName="container"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.449611    5440 container_manager_linux.go:308] "Creating device plugin manager"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.450456    5440 state_mem.go:36] "Initialized new in-memory state store"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.454801    5440 kubelet.go:398] "Attempting to sync node with API server"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.454852    5440 kubelet.go:286] "Adding static pod path" path="/mnt/SSD Hector Pool/ix-applications/k3s/agent/pod-manifests"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.455155    5440 kubelet.go:297] "Adding apiserver pod source"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.455224    5440 apiserver.go:42] "Waiting for node sync before watching apiserver pods"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.459162    5440 kuberuntime_manager.go:244] "Container runtime initialized" containerRuntime="containerd" version="truenas/v1.7.1-k3s1" apiVersion="v1"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.462541    5440 nodeshutdown_manager_linux.go:137] "Creating node shutdown manager" shutdownGracePeriodRequested="15s" shutdownGracePeriodCriticalPods="10s" shutdownGracePeriodByPodPriority=[{Priority:0 ShutdownGracePeriodSeconds:5} {Priority:2000000000 ShutdownGracePeriodSeconds:10}]
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.463117    5440 server.go:1181] "Started kubelet"
Jan 22 12:39:59 truenas k3s[5440]: E0122 12:39:59.464677    5440 cri_stats_provider.go:455] "Failed to get the info of the filesystem with mountpoint" err="unable to find data in memory cache" mountpoint="/mnt/SSD Hector Pool/ix-applications/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs"
Jan 22 12:39:59 truenas k3s[5440]: E0122 12:39:59.464726    5440 kubelet.go:1386] "Image garbage collection failed once. Stats initialization may not have completed yet" err="invalid capacity 0 on image filesystem"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.464982    5440 server.go:161] "Starting to listen" address="0.0.0.0" port=10250
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.466711    5440 server.go:451] "Adding debug handlers to kubelet server"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.466840    5440 fs_resource_analyzer.go:67] "Starting FS ResourceAnalyzer"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.468012    5440 scope.go:115] "RemoveContainer" containerID="dc49d2ce1af21770b13ea758a9f6879b87e4cbf38c26df3a0225935d173776fa"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.471714    5440 volume_manager.go:293] "Starting Kubelet Volume Manager"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.471845    5440 desired_state_of_world_populator.go:151] "Desired state populator starts to run"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.502148    5440 cpu_manager.go:214] "Starting CPU manager" policy="none"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.502171    5440 cpu_manager.go:215] "Reconciling" reconcilePeriod="10s"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.502204    5440 state_mem.go:36] "Initialized new in-memory state store"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.503115    5440 state_mem.go:88] "Updated default CPUSet" cpuSet=""
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.503138    5440 state_mem.go:96] "Updated CPUSet assignments" assignments=map[]
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.503198    5440 policy_none.go:49] "None policy: Start"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.504304    5440 memory_manager.go:169] "Starting memorymanager" policy="None"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.504372    5440 state_mem.go:35] "Initializing new in-memory state store"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.504872    5440 state_mem.go:75] "Updated machine memory state"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.567406    5440 plugin_manager.go:118] "Starting Kubelet Plugin Manager"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.601898    5440 kubelet_network_linux.go:63] "Initialized iptables rules." protocol=IPv4
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.617211    5440 kubelet_node_status.go:70] "Attempting to register node" node="ix-truenas"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.624249    5440 kubelet_network_linux.go:63] "Initialized iptables rules." protocol=IPv6
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.624647    5440 status_manager.go:176] "Starting to sync pod status with apiserver"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.624722    5440 kubelet.go:2113] "Starting kubelet main sync loop"
Jan 22 12:39:59 truenas k3s[5440]: E0122 12:39:59.624851    5440 kubelet.go:2137] "Skipping pod synchronization" err="PLEG is not healthy: pleg has yet to be successful"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.624866    5440 kubelet_node_status.go:108] "Node was previously registered" node="ix-truenas"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.624983    5440 kubelet_node_status.go:73] "Successfully registered node" node="ix-truenas"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.626698    5440 kuberuntime_manager.go:1114] "Updating runtime config through cri with podcidr" CIDR="172.16.0.0/16"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.627376    5440 kubelet_network.go:61] "Updating Pod CIDR" originalPodCIDR="" newPodCIDR="172.16.0.0/16"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.725942    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="6ba33a1ff0fd185de8ea22d2ba9c6b767be5b49456aee8a616e751c01f4f4913"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.725972    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="cc0c09b2855320b3319f0106b5e7b6fca608c081aaf79c84b919cf703e0d1ede"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.725988    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="88afee7761e7b5432d8d3a14245492ae84138c1b1aa81eaa3738f58223173629"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726005    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="e5068d25cd9a127ca2c7cfa169d0d69fbe249ece54b370846a30b733e91cd296"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726026    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="471215021b00f25ac4e2c43067dd4b4d4cdacfd555f6aa21c07b021176eac680"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726042    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="4dd5a58d3637636971a3ef8a8e6ccc864edc398d78996d00cdf23b0c19d63766"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726058    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="42882c49036bd6e034887924c9b443d491f9eca4d910483b4d4f3b7c299084eb"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726076    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="7b62a0409c09086b76e577e307ae273b4de59fc91e4c3871c335a5d403a8bf14"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726103    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="835004a1ebcfb23c46ac986d7f46c3941c1453ca3901af265f71baff59173390"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726120    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="0503825162aed53a58d4dbf29511823a04f954c0cc983518e9360085a694afd9"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726136    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="1199b63e0a744a1df157eade2e0b0bf2c4e5a9999b0a0c2993cf2531242d8743"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726152    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="79e2da185997e54f9554665e5daf02b9c502c4cf27d25dc690729f4928e5410f"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726168    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="6d3cb0c761193504cdafa28211ffdba6f622cb7f8465454addc7d142e688b197"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726198    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="f377822db61ac9fd4a0bc7893cf1cc862d2c1b6ae7b0b9a31b81e26febd85e66"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726223    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="81049eef5134f851cd5dc16913f81a451ea9db94aec59d119789fadbe122d567"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726238    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="ae478a031f4a2d663468379c817d1a3244333e72bee0ecc52230567be3fa7d48"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726259    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="69417bf91ed6cb034e674f7928482c6b95b944cc29dd6f4e97bbc093b6af6395"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726273    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="c7bb6e5e91c1a467e261369c4a04b644a9dce4194a16d9d2006a02b95194cff1"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726291    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="d8be9f7623ea6316709f45aee9a9dac8faf676a39620d03d382e7d65999c09eb"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726309    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="af2bce47ecbbae841a6692f020caeffbfd19de5e0c607f3eb88d4fb9fb5503c6"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726328    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="955d576f2228749e1907e44242e7a64286f2a19f602acf64f029d7fae20ad503"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726350    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="043021eacc5eb79a66434d43f602d13edce5cc1301dd74c5d0d05aee69f8ee6a"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726370    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="6926ff0c755322651c131512a1925067ab83d92fe812835e53cc23aac85fc792"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.726394    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="e700a97b6b7e13d91fab28f5d6c49b36bb37dd01ea20a84f5d01112b807e3335"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.756060    5440 server.go:152] "Starting Kubernetes Scheduler" version="v1.26.6+k3s-e18037a7-dirty"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.756080    5440 server.go:154] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.758850    5440 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.758869    5440 shared_informer.go:270] Waiting for caches to sync for RequestHeaderAuthRequestController
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.758876    5440 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.758891    5440 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.758857    5440 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.758911    5440 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.758955    5440 secure_serving.go:210] Serving securely on 127.0.0.1:10259
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.759534    5440 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.859853    5440 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.859866    5440 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.860091    5440 shared_informer.go:277] Caches are synced for RequestHeaderAuthRequestController
Jan 22 12:39:59 truenas k3s[5440]: I0122 12:39:59.963705    5440 event.go:294] "Event occurred" object="kube-system/zfs-operator" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD Hector Pool/ix-applications/k3s/server/manifests/zfs-operator.yaml\""
Jan 22 12:40:00 truenas k3s[5440]: time="2024-01-22T12:40:00+01:00" level=info msg="Stopped tunnel to 127.0.0.1:6443"
Jan 22 12:40:00 truenas k3s[5440]: time="2024-01-22T12:40:00+01:00" level=info msg="Connecting to proxy" url="wss://192.168.0.100:6443/v1-k3s/connect"
Jan 22 12:40:00 truenas k3s[5440]: time="2024-01-22T12:40:00+01:00" level=info msg="Proxy done" err="context canceled" url="wss://127.0.0.1:6443/v1-k3s/connect"
Jan 22 12:40:00 truenas k3s[5440]: time="2024-01-22T12:40:00+01:00" level=info msg="error in remotedialer server [400]: websocket: close 1006 (abnormal closure): unexpected EOF"
Jan 22 12:40:00 truenas k3s[5440]: time="2024-01-22T12:40:00+01:00" level=info msg="Handling backend connection request [ix-truenas]"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.456548    5440 apiserver.go:52] "Watching apiserver"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.463780    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.464620    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.464710    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465069    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465238    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465334    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465415    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465488    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465565    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465645    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465715    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465786    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465853    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.465939    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466024    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466124    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466203    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466408    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466503    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466587    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466680    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466770    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466869    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.466954    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480292    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-5r4tt\" (UniqueName: \"kubernetes.io/projected/4c9adf6d-9e15-4725-a0bd-0513c49e51f6-kube-api-access-5r4tt\") pod \"snapshot-controller-546868dfb4-thslt\" (UID: \"4c9adf6d-9e15-4725-a0bd-0513c49e51f6\") " pod="kube-system/snapshot-controller-546868dfb4-thslt"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480630    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-host-root\") pod \"openebs-zfs-node-cfml4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") " pod="kube-system/openebs-zfs-node-cfml4"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480701    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs\") pod \"openebs-zfs-node-cfml4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") " pod="kube-system/openebs-zfs-node-cfml4"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480727    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-pods-mount-dir\") pod \"openebs-zfs-node-cfml4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") " pod="kube-system/openebs-zfs-node-cfml4"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480780    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"mountpoint-dir\" (UniqueName: \"kubernetes.io/host-path/5dfd1c94-3eef-47bf-b2dc-ada1ea0a4273-mountpoint-dir\") pod \"csi-smb-node-hj2wt\" (UID: \"5dfd1c94-3eef-47bf-b2dc-ada1ea0a4273\") " pod="kube-system/csi-smb-node-hj2wt"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480806    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/5dfd1c94-3eef-47bf-b2dc-ada1ea0a4273-registration-dir\") pod \"csi-smb-node-hj2wt\" (UID: \"5dfd1c94-3eef-47bf-b2dc-ada1ea0a4273\") " pod="kube-system/csi-smb-node-hj2wt"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480829    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/host-path/5a357666-a6c7-4305-b83d-4596bbe56356-socket-dir\") pod \"csi-nfs-node-jhnfd\" (UID: \"5a357666-a6c7-4305-b83d-4596bbe56356\") " pod="kube-system/csi-nfs-node-jhnfd"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480850    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-device-dir\") pod \"openebs-zfs-node-cfml4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") " pod="kube-system/openebs-zfs-node-cfml4"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480886    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-encr-keys\") pod \"openebs-zfs-node-cfml4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") " pod="kube-system/openebs-zfs-node-cfml4"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480926    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-mq76x\" (UniqueName: \"kubernetes.io/projected/12818b5c-7610-4023-bce4-e74fe62310b4-kube-api-access-mq76x\") pod \"openebs-zfs-node-cfml4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") " pod="kube-system/openebs-zfs-node-cfml4"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.480966    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-fdclj\" (UniqueName: \"kubernetes.io/projected/5dfd1c94-3eef-47bf-b2dc-ada1ea0a4273-kube-api-access-fdclj\") pod \"csi-smb-node-hj2wt\" (UID: \"5dfd1c94-3eef-47bf-b2dc-ada1ea0a4273\") " pod="kube-system/csi-smb-node-hj2wt"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481004    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-swdwf\" (UniqueName: \"kubernetes.io/projected/5a357666-a6c7-4305-b83d-4596bbe56356-kube-api-access-swdwf\") pod \"csi-nfs-node-jhnfd\" (UID: \"5a357666-a6c7-4305-b83d-4596bbe56356\") " pod="kube-system/csi-nfs-node-jhnfd"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481048    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-njshv\" (UniqueName: \"kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv\") pod \"cert-manager-8444f6f86b-gcmtm\" (UID: \"89472a65-6558-4aa6-aa8e-59a5692fd061\") " pod="cert-manager/cert-manager-8444f6f86b-gcmtm"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481082    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/0c60407c-b2f4-4a08-85cb-ec68308d717c-socket-dir\") pod \"csi-nfs-controller-7b74694749-skm24\" (UID: \"0c60407c-b2f4-4a08-85cb-ec68308d717c\") " pod="kube-system/csi-nfs-controller-7b74694749-skm24"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481109    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-jcnt5\" (UniqueName: \"kubernetes.io/projected/0c60407c-b2f4-4a08-85cb-ec68308d717c-kube-api-access-jcnt5\") pod \"csi-nfs-controller-7b74694749-skm24\" (UID: \"0c60407c-b2f4-4a08-85cb-ec68308d717c\") " pod="kube-system/csi-nfs-controller-7b74694749-skm24"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481156    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/0c60407c-b2f4-4a08-85cb-ec68308d717c-pods-mount-dir\") pod \"csi-nfs-controller-7b74694749-skm24\" (UID: \"0c60407c-b2f4-4a08-85cb-ec68308d717c\") " pod="kube-system/csi-nfs-controller-7b74694749-skm24"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481190    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-registration-dir\") pod \"openebs-zfs-node-cfml4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") " pod="kube-system/openebs-zfs-node-cfml4"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481226    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-plugin-dir\") pod \"openebs-zfs-node-cfml4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") " pod="kube-system/openebs-zfs-node-cfml4"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481249    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/host-path/5dfd1c94-3eef-47bf-b2dc-ada1ea0a4273-socket-dir\") pod \"csi-smb-node-hj2wt\" (UID: \"5dfd1c94-3eef-47bf-b2dc-ada1ea0a4273\") " pod="kube-system/csi-smb-node-hj2wt"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481278    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/5a357666-a6c7-4305-b83d-4596bbe56356-pods-mount-dir\") pod \"csi-nfs-node-jhnfd\" (UID: \"5a357666-a6c7-4305-b83d-4596bbe56356\") " pod="kube-system/csi-nfs-node-jhnfd"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481318    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/5a357666-a6c7-4305-b83d-4596bbe56356-registration-dir\") pod \"csi-nfs-node-jhnfd\" (UID: \"5a357666-a6c7-4305-b83d-4596bbe56356\") " pod="kube-system/csi-nfs-node-jhnfd"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481352    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-mhkk9\" (UniqueName: \"kubernetes.io/projected/2106327d-69ea-48e4-8e6c-733aef332cf3-kube-api-access-mhkk9\") pod \"snapshot-controller-546868dfb4-jrg5t\" (UID: \"2106327d-69ea-48e4-8e6c-733aef332cf3\") " pod="kube-system/snapshot-controller-546868dfb4-jrg5t"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.481401    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-5kbbr\" (UniqueName: \"kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr\") pod \"cert-manager-cainjector-ffb4747bb-sfktp\" (UID: \"32eb535e-4f44-4b9d-b331-1450dc7b1e04\") " pod="cert-manager/cert-manager-cainjector-ffb4747bb-sfktp"
Jan 22 12:40:00 truenas k3s[5440]: E0122 12:40:00.582770    5440 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:00 truenas k3s[5440]: E0122 12:40:00.582927    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs podName:12818b5c-7610-4023-bce4-e74fe62310b4 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:01.082843595 +0100 CET m=+9.493481465 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs") pod "openebs-zfs-node-cfml4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4") : object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.682896    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"nginx-proxy-manager-58666cd54f-pb5w8\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-pb5w8"
Jan 22 12:40:00 truenas k3s[5440]: I0122 12:40:00.683027    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"devshm\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-devshm\") pod \"nginx-proxy-manager-58666cd54f-pb5w8\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-pb5w8"
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.068831    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.068918    5440 projected.go:198] Error preparing data for projected volume kube-api-access-5kbbr for pod cert-manager/cert-manager-cainjector-ffb4747bb-sfktp: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.069052    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr podName:32eb535e-4f44-4b9d-b331-1450dc7b1e04 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:01.569015518 +0100 CET m=+9.979653438 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-5kbbr" (UniqueName: "kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr") pod "cert-manager-cainjector-ffb4747bb-sfktp" (UID: "32eb535e-4f44-4b9d-b331-1450dc7b1e04") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.086621    5440 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.086717    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs podName:12818b5c-7610-4023-bce4-e74fe62310b4 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:02.086690528 +0100 CET m=+10.497328419 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs") pod "openebs-zfs-node-cfml4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4") : object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.590957    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.590991    5440 projected.go:198] Error preparing data for projected volume kube-api-access-5kbbr for pod cert-manager/cert-manager-cainjector-ffb4747bb-sfktp: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.591057    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr podName:32eb535e-4f44-4b9d-b331-1450dc7b1e04 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:02.591037936 +0100 CET m=+11.001675814 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-5kbbr" (UniqueName: "kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr") pod "cert-manager-cainjector-ffb4747bb-sfktp" (UID: "32eb535e-4f44-4b9d-b331-1450dc7b1e04") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:01 truenas k3s[5440]: I0122 12:40:01.656214    5440 request.go:690] Waited for 1.072966331s due to client-side throttling, not priority and fairness, request: POST:https://127.0.0.1:6443/api/v1/namespaces/kube-system/serviceaccounts/openebs-zfs-node-sa/token
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.861509    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.861536    5440 projected.go:198] Error preparing data for projected volume kube-api-access-njshv for pod cert-manager/cert-manager-8444f6f86b-gcmtm: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:01 truenas k3s[5440]: E0122 12:40:01.861598    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv podName:89472a65-6558-4aa6-aa8e-59a5692fd061 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:02.361576618 +0100 CET m=+10.772214489 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-njshv" (UniqueName: "kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv") pod "cert-manager-8444f6f86b-gcmtm" (UID: "89472a65-6558-4aa6-aa8e-59a5692fd061") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.094423    5440 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.094474    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs podName:12818b5c-7610-4023-bce4-e74fe62310b4 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:04.094459306 +0100 CET m=+12.505097180 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs") pod "openebs-zfs-node-cfml4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4") : object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.396950    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.396972    5440 projected.go:198] Error preparing data for projected volume kube-api-access-njshv for pod cert-manager/cert-manager-8444f6f86b-gcmtm: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.397028    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv podName:89472a65-6558-4aa6-aa8e-59a5692fd061 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:03.397009225 +0100 CET m=+11.807647104 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-njshv" (UniqueName: "kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv") pod "cert-manager-8444f6f86b-gcmtm" (UID: "89472a65-6558-4aa6-aa8e-59a5692fd061") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.597972    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.598005    5440 projected.go:198] Error preparing data for projected volume kube-api-access-5kbbr for pod cert-manager/cert-manager-cainjector-ffb4747bb-sfktp: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.598072    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr podName:32eb535e-4f44-4b9d-b331-1450dc7b1e04 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:04.598050475 +0100 CET m=+13.008688354 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-5kbbr" (UniqueName: "kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr") pod "cert-manager-cainjector-ffb4747bb-sfktp" (UID: "32eb535e-4f44-4b9d-b331-1450dc7b1e04") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:02 truenas k3s[5440]: E0122 12:40:02.900887    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:03.400862663 +0100 CET m=+11.811500552 (durationBeforeRetry 500ms). Error: MountVolume.MountDevice failed for volume "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de") pod "nginx-proxy-manager-58666cd54f-pb5w8" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: attacher.MountDevice failed to create newCsiDriverClient: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.257671    5440 desired_state_of_world_populator.go:159] "Finished populating initial desired state of world"
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302127    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"devshm\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-devshm\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302182    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-5r4tt\" (UniqueName: \"kubernetes.io/projected/4c9adf6d-9e15-4725-a0bd-0513c49e51f6-kube-api-access-5r4tt\") pod \"4c9adf6d-9e15-4725-a0bd-0513c49e51f6\" (UID: \"4c9adf6d-9e15-4725-a0bd-0513c49e51f6\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302226    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-device-dir\") pod \"12818b5c-7610-4023-bce4-e74fe62310b4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302266    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-jcnt5\" (UniqueName: \"kubernetes.io/projected/0c60407c-b2f4-4a08-85cb-ec68308d717c-kube-api-access-jcnt5\") pod \"0c60407c-b2f4-4a08-85cb-ec68308d717c\" (UID: \"0c60407c-b2f4-4a08-85cb-ec68308d717c\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302298    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/0c60407c-b2f4-4a08-85cb-ec68308d717c-pods-mount-dir\") pod \"0c60407c-b2f4-4a08-85cb-ec68308d717c\" (UID: \"0c60407c-b2f4-4a08-85cb-ec68308d717c\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302326    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-pods-mount-dir\") pod \"12818b5c-7610-4023-bce4-e74fe62310b4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302347    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-encr-keys\") pod \"12818b5c-7610-4023-bce4-e74fe62310b4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302375    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-mq76x\" (UniqueName: \"kubernetes.io/projected/12818b5c-7610-4023-bce4-e74fe62310b4-kube-api-access-mq76x\") pod \"12818b5c-7610-4023-bce4-e74fe62310b4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302400    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-plugin-dir\") pod \"12818b5c-7610-4023-bce4-e74fe62310b4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302439    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-mhkk9\" (UniqueName: \"kubernetes.io/projected/2106327d-69ea-48e4-8e6c-733aef332cf3-kube-api-access-mhkk9\") pod \"2106327d-69ea-48e4-8e6c-733aef332cf3\" (UID: \"2106327d-69ea-48e4-8e6c-733aef332cf3\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302468    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-host-root\") pod \"12818b5c-7610-4023-bce4-e74fe62310b4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302492    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/0c60407c-b2f4-4a08-85cb-ec68308d717c-socket-dir\") pod \"0c60407c-b2f4-4a08-85cb-ec68308d717c\" (UID: \"0c60407c-b2f4-4a08-85cb-ec68308d717c\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302511    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-registration-dir\") pod \"12818b5c-7610-4023-bce4-e74fe62310b4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302555    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-varlogs\") pod \"nginx-proxy-manager-58666cd54f-pb5w8\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-pb5w8"
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302640    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-varrun\") pod \"nginx-proxy-manager-58666cd54f-pb5w8\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-pb5w8"
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302697    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"nginx-proxy-manager-58666cd54f-pb5w8\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-pb5w8"
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302736    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-shared\") pod \"nginx-proxy-manager-58666cd54f-pb5w8\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-pb5w8"
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302771    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-tmp\") pod \"nginx-proxy-manager-58666cd54f-pb5w8\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-pb5w8"
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.302821    5440 reconciler.go:41] "Reconciler: start to sync state"
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.303092    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-encr-keys" (OuterVolumeSpecName: "encr-keys") pod "12818b5c-7610-4023-bce4-e74fe62310b4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4"). InnerVolumeSpecName "encr-keys". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.303143    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-pods-mount-dir" (OuterVolumeSpecName: "pods-mount-dir") pod "12818b5c-7610-4023-bce4-e74fe62310b4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4"). InnerVolumeSpecName "pods-mount-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.303153    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-registration-dir" (OuterVolumeSpecName: "registration-dir") pod "12818b5c-7610-4023-bce4-e74fe62310b4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4"). InnerVolumeSpecName "registration-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.303204    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-plugin-dir" (OuterVolumeSpecName: "plugin-dir") pod "12818b5c-7610-4023-bce4-e74fe62310b4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4"). InnerVolumeSpecName "plugin-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.303569    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/0c60407c-b2f4-4a08-85cb-ec68308d717c-pods-mount-dir" (OuterVolumeSpecName: "pods-mount-dir") pod "0c60407c-b2f4-4a08-85cb-ec68308d717c" (UID: "0c60407c-b2f4-4a08-85cb-ec68308d717c"). InnerVolumeSpecName "pods-mount-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.303689    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-device-dir" (OuterVolumeSpecName: "device-dir") pod "12818b5c-7610-4023-bce4-e74fe62310b4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4"). InnerVolumeSpecName "device-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.303797    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-host-root" (OuterVolumeSpecName: "host-root") pod "12818b5c-7610-4023-bce4-e74fe62310b4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4"). InnerVolumeSpecName "host-root". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: W0122 12:40:03.304528    5440 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/0c60407c-b2f4-4a08-85cb-ec68308d717c/volumes/kubernetes.io~empty-dir/socket-dir: clearQuota called, but quotas disabled
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.304905    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/0c60407c-b2f4-4a08-85cb-ec68308d717c-socket-dir" (OuterVolumeSpecName: "socket-dir") pod "0c60407c-b2f4-4a08-85cb-ec68308d717c" (UID: "0c60407c-b2f4-4a08-85cb-ec68308d717c"). InnerVolumeSpecName "socket-dir". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.306788    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-devshm" (OuterVolumeSpecName: "devshm") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d"). InnerVolumeSpecName "devshm". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.306941    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/12818b5c-7610-4023-bce4-e74fe62310b4-kube-api-access-mq76x" (OuterVolumeSpecName: "kube-api-access-mq76x") pod "12818b5c-7610-4023-bce4-e74fe62310b4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4"). InnerVolumeSpecName "kube-api-access-mq76x". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.307156    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/0c60407c-b2f4-4a08-85cb-ec68308d717c-kube-api-access-jcnt5" (OuterVolumeSpecName: "kube-api-access-jcnt5") pod "0c60407c-b2f4-4a08-85cb-ec68308d717c" (UID: "0c60407c-b2f4-4a08-85cb-ec68308d717c"). InnerVolumeSpecName "kube-api-access-jcnt5". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.307170    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/2106327d-69ea-48e4-8e6c-733aef332cf3-kube-api-access-mhkk9" (OuterVolumeSpecName: "kube-api-access-mhkk9") pod "2106327d-69ea-48e4-8e6c-733aef332cf3" (UID: "2106327d-69ea-48e4-8e6c-733aef332cf3"). InnerVolumeSpecName "kube-api-access-mhkk9". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.307670    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/4c9adf6d-9e15-4725-a0bd-0513c49e51f6-kube-api-access-5r4tt" (OuterVolumeSpecName: "kube-api-access-5r4tt") pod "4c9adf6d-9e15-4725-a0bd-0513c49e51f6" (UID: "4c9adf6d-9e15-4725-a0bd-0513c49e51f6"). InnerVolumeSpecName "kube-api-access-5r4tt". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.518199    5440 kubelet_node_status.go:493] "Fast updating node status as it just became ready"
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.909967    5440 controller.go:615] quota admission added evaluator for: daemonsets.apps
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.958931    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/0c60407c-b2f4-4a08-85cb-ec68308d717c-socket-dir\") pod \"0c60407c-b2f4-4a08-85cb-ec68308d717c\" (UID: \"0c60407c-b2f4-4a08-85cb-ec68308d717c\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.958990    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"devshm\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-devshm\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959029    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-5r4tt\" (UniqueName: \"kubernetes.io/projected/4c9adf6d-9e15-4725-a0bd-0513c49e51f6-kube-api-access-5r4tt\") pod \"4c9adf6d-9e15-4725-a0bd-0513c49e51f6\" (UID: \"4c9adf6d-9e15-4725-a0bd-0513c49e51f6\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959153    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959198    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-mhkk9\" (UniqueName: \"kubernetes.io/projected/2106327d-69ea-48e4-8e6c-733aef332cf3-kube-api-access-mhkk9\") pod \"2106327d-69ea-48e4-8e6c-733aef332cf3\" (UID: \"2106327d-69ea-48e4-8e6c-733aef332cf3\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959237    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-mq76x\" (UniqueName: \"kubernetes.io/projected/12818b5c-7610-4023-bce4-e74fe62310b4-kube-api-access-mq76x\") pod \"12818b5c-7610-4023-bce4-e74fe62310b4\" (UID: \"12818b5c-7610-4023-bce4-e74fe62310b4\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959275    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-jcnt5\" (UniqueName: \"kubernetes.io/projected/0c60407c-b2f4-4a08-85cb-ec68308d717c-kube-api-access-jcnt5\") pod \"0c60407c-b2f4-4a08-85cb-ec68308d717c\" (UID: \"0c60407c-b2f4-4a08-85cb-ec68308d717c\") "
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959507    5440 reconciler_common.go:295] "Volume detached for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-encr-keys\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959530    5440 reconciler_common.go:295] "Volume detached for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/0c60407c-b2f4-4a08-85cb-ec68308d717c-pods-mount-dir\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959549    5440 reconciler_common.go:295] "Volume detached for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-host-root\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959568    5440 reconciler_common.go:295] "Volume detached for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-registration-dir\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959588    5440 reconciler_common.go:295] "Volume detached for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-device-dir\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959608    5440 reconciler_common.go:295] "Volume detached for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-pods-mount-dir\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.959627    5440 reconciler_common.go:295] "Volume detached for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/12818b5c-7610-4023-bce4-e74fe62310b4-plugin-dir\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:03 truenas k3s[5440]: W0122 12:40:03.960454    5440 empty_dir.go:499] Warning: Unmount skipped because path does not exist: /var/lib/kubelet/pods/0c60407c-b2f4-4a08-85cb-ec68308d717c/volumes/kubernetes.io~projected/kube-api-access-jcnt5
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.960476    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/0c60407c-b2f4-4a08-85cb-ec68308d717c-kube-api-access-jcnt5" (OuterVolumeSpecName: "kube-api-access-jcnt5") pod "0c60407c-b2f4-4a08-85cb-ec68308d717c" (UID: "0c60407c-b2f4-4a08-85cb-ec68308d717c"). InnerVolumeSpecName "kube-api-access-jcnt5". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: W0122 12:40:03.960544    5440 empty_dir.go:499] Warning: Unmount skipped because path does not exist: /var/lib/kubelet/pods/0c60407c-b2f4-4a08-85cb-ec68308d717c/volumes/kubernetes.io~empty-dir/socket-dir
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.960554    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/0c60407c-b2f4-4a08-85cb-ec68308d717c-socket-dir" (OuterVolumeSpecName: "socket-dir") pod "0c60407c-b2f4-4a08-85cb-ec68308d717c" (UID: "0c60407c-b2f4-4a08-85cb-ec68308d717c"). InnerVolumeSpecName "socket-dir". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: W0122 12:40:03.960606    5440 empty_dir.go:499] Warning: Unmount skipped because path does not exist: /var/lib/kubelet/pods/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d/volumes/kubernetes.io~empty-dir/devshm
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.960617    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-devshm" (OuterVolumeSpecName: "devshm") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d"). InnerVolumeSpecName "devshm". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: W0122 12:40:03.960680    5440 empty_dir.go:499] Warning: Unmount skipped because path does not exist: /var/lib/kubelet/pods/4c9adf6d-9e15-4725-a0bd-0513c49e51f6/volumes/kubernetes.io~projected/kube-api-access-5r4tt
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.960693    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/4c9adf6d-9e15-4725-a0bd-0513c49e51f6-kube-api-access-5r4tt" (OuterVolumeSpecName: "kube-api-access-5r4tt") pod "4c9adf6d-9e15-4725-a0bd-0513c49e51f6" (UID: "4c9adf6d-9e15-4725-a0bd-0513c49e51f6"). InnerVolumeSpecName "kube-api-access-5r4tt". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: E0122 12:40:03.961198    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:04.461034914 +0100 CET m=+12.871672798 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:03 truenas k3s[5440]: W0122 12:40:03.961265    5440 empty_dir.go:499] Warning: Unmount skipped because path does not exist: /var/lib/kubelet/pods/2106327d-69ea-48e4-8e6c-733aef332cf3/volumes/kubernetes.io~projected/kube-api-access-mhkk9
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.961280    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/2106327d-69ea-48e4-8e6c-733aef332cf3-kube-api-access-mhkk9" (OuterVolumeSpecName: "kube-api-access-mhkk9") pod "2106327d-69ea-48e4-8e6c-733aef332cf3" (UID: "2106327d-69ea-48e4-8e6c-733aef332cf3"). InnerVolumeSpecName "kube-api-access-mhkk9". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: W0122 12:40:03.961349    5440 empty_dir.go:499] Warning: Unmount skipped because path does not exist: /var/lib/kubelet/pods/12818b5c-7610-4023-bce4-e74fe62310b4/volumes/kubernetes.io~projected/kube-api-access-mq76x
Jan 22 12:40:03 truenas k3s[5440]: I0122 12:40:03.961361    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/12818b5c-7610-4023-bce4-e74fe62310b4-kube-api-access-mq76x" (OuterVolumeSpecName: "kube-api-access-mq76x") pod "12818b5c-7610-4023-bce4-e74fe62310b4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4"). InnerVolumeSpecName "kube-api-access-mq76x". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:03 truenas k3s[5440]: E0122 12:40:03.961485    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:03 truenas k3s[5440]: E0122 12:40:03.961502    5440 projected.go:198] Error preparing data for projected volume kube-api-access-njshv for pod cert-manager/cert-manager-8444f6f86b-gcmtm: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:03 truenas k3s[5440]: E0122 12:40:03.961551    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv podName:89472a65-6558-4aa6-aa8e-59a5692fd061 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:05.961537093 +0100 CET m=+14.372174962 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-njshv" (UniqueName: "kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv") pod "cert-manager-8444f6f86b-gcmtm" (UID: "89472a65-6558-4aa6-aa8e-59a5692fd061") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:03 truenas k3s[5440]: E0122 12:40:03.962862    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:04.962842968 +0100 CET m=+13.373480846 (durationBeforeRetry 1s). Error: MountVolume.MountDevice failed for volume "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de") pod "nginx-proxy-manager-58666cd54f-pb5w8" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: attacher.MountDevice failed to create newCsiDriverClient: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:03 truenas k3s[5440]: E0122 12:40:03.963294    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:04.463276347 +0100 CET m=+12.873914219 (durationBeforeRetry 500ms). Error: MountVolume.MountDevice failed for volume "pvc-88dedeea-6d2f-448f-bafa-658c55aa954b" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b") pod "nginx-proxy-manager-58666cd54f-pb5w8" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: attacher.MountDevice failed to create newCsiDriverClient: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.060027    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.060133    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.060159    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:04.560121328 +0100 CET m=+12.970759212 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-88dedeea-6d2f-448f-bafa-658c55aa954b" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.060278    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:04.560252895 +0100 CET m=+12.970890780 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.060348    5440 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-mq76x\" (UniqueName: \"kubernetes.io/projected/12818b5c-7610-4023-bce4-e74fe62310b4-kube-api-access-mq76x\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.060372    5440 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-jcnt5\" (UniqueName: \"kubernetes.io/projected/0c60407c-b2f4-4a08-85cb-ec68308d717c-kube-api-access-jcnt5\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.060392    5440 reconciler_common.go:295] "Volume detached for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/0c60407c-b2f4-4a08-85cb-ec68308d717c-socket-dir\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.060410    5440 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-5r4tt\" (UniqueName: \"kubernetes.io/projected/4c9adf6d-9e15-4725-a0bd-0513c49e51f6-kube-api-access-5r4tt\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.060429    5440 reconciler_common.go:295] "Volume detached for volume \"devshm\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-devshm\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.060447    5440 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-mhkk9\" (UniqueName: \"kubernetes.io/projected/2106327d-69ea-48e4-8e6c-733aef332cf3-kube-api-access-mhkk9\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.160834    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-varrun\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.160893    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-shared\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.161062    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-tmp\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.161243    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-varlogs\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.161492    5440 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.161557    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs podName:12818b5c-7610-4023-bce4-e74fe62310b4 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:08.16153849 +0100 CET m=+16.572176370 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs") pod "openebs-zfs-node-cfml4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4") : object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:04 truenas k3s[5440]: W0122 12:40:04.162984    5440 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d/volumes/kubernetes.io~empty-dir/shared: clearQuota called, but quotas disabled
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.163102    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-shared" (OuterVolumeSpecName: "shared") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d"). InnerVolumeSpecName "shared". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.182730    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-varrun" (OuterVolumeSpecName: "varrun") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d"). InnerVolumeSpecName "varrun". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.182860    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-varlogs" (OuterVolumeSpecName: "varlogs") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d"). InnerVolumeSpecName "varlogs". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.183202    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-tmp" (OuterVolumeSpecName: "tmp") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d"). InnerVolumeSpecName "tmp". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.261699    5440 reconciler_common.go:295] "Volume detached for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-tmp\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.261729    5440 reconciler_common.go:295] "Volume detached for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-varlogs\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.261746    5440 reconciler_common.go:295] "Volume detached for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-varrun\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.261763    5440 reconciler_common.go:295] "Volume detached for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/7e0f78dc-2724-42c6-97a7-76f79fcb2b9d-shared\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:04 truenas k3s[5440]: time="2024-01-22T12:40:04+01:00" level=info msg="Tunnel authorizer set Kubelet Port 10250"
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.462682    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.462902    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:05.462867843 +0100 CET m=+13.873505731 (durationBeforeRetry 1s). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.563561    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:04 truenas k3s[5440]: I0122 12:40:04.563676    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.563712    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:05.563669084 +0100 CET m=+13.974306968 (durationBeforeRetry 1s). Error: UnmountVolume.TearDown failed for volume "pvc-88dedeea-6d2f-448f-bafa-658c55aa954b" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.563785    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:05.563757085 +0100 CET m=+13.974394972 (durationBeforeRetry 1s). Error: UnmountVolume.TearDown failed for volume "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.664644    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.664674    5440 projected.go:198] Error preparing data for projected volume kube-api-access-5kbbr for pod cert-manager/cert-manager-cainjector-ffb4747bb-sfktp: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:04 truenas k3s[5440]: E0122 12:40:04.664753    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr podName:32eb535e-4f44-4b9d-b331-1450dc7b1e04 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:08.66473166 +0100 CET m=+17.075369532 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-5kbbr" (UniqueName: "kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr") pod "cert-manager-cainjector-ffb4747bb-sfktp" (UID: "32eb535e-4f44-4b9d-b331-1450dc7b1e04") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:05 truenas k3s[5440]: I0122 12:40:05.470982    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:05 truenas k3s[5440]: E0122 12:40:05.471111    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:07.471075625 +0100 CET m=+15.881713501 (durationBeforeRetry 2s). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:05 truenas k3s[5440]: I0122 12:40:05.571594    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:05 truenas k3s[5440]: I0122 12:40:05.571672    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:05 truenas k3s[5440]: E0122 12:40:05.571756    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:07.571715946 +0100 CET m=+15.982353830 (durationBeforeRetry 2s). Error: UnmountVolume.TearDown failed for volume "pvc-88dedeea-6d2f-448f-bafa-658c55aa954b" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:05 truenas k3s[5440]: E0122 12:40:05.571806    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:07.571789448 +0100 CET m=+15.982427319 (durationBeforeRetry 2s). Error: UnmountVolume.TearDown failed for volume "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:05 truenas k3s[5440]: I0122 12:40:05.575654    5440 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: nfs.csi.k8s.io endpoint: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock versions: 1.0.0
Jan 22 12:40:05 truenas k3s[5440]: I0122 12:40:05.575678    5440 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: nfs.csi.k8s.io at endpoint: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
Jan 22 12:40:05 truenas k3s[5440]: E0122 12:40:05.974394    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:05 truenas k3s[5440]: E0122 12:40:05.974417    5440 projected.go:198] Error preparing data for projected volume kube-api-access-njshv for pod cert-manager/cert-manager-8444f6f86b-gcmtm: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:05 truenas k3s[5440]: E0122 12:40:05.974474    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv podName:89472a65-6558-4aa6-aa8e-59a5692fd061 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:09.974457031 +0100 CET m=+18.385094915 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-njshv" (UniqueName: "kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv") pod "cert-manager-8444f6f86b-gcmtm" (UID: "89472a65-6558-4aa6-aa8e-59a5692fd061") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:06 truenas k3s[5440]: I0122 12:40:06.574339    5440 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: smb.csi.k8s.io endpoint: /var/lib/kubelet/plugins/smb.csi.k8s.io/csi.sock versions: 1.0.0
Jan 22 12:40:06 truenas k3s[5440]: I0122 12:40:06.574380    5440 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: smb.csi.k8s.io at endpoint: /var/lib/kubelet/plugins/smb.csi.k8s.io/csi.sock
Jan 22 12:40:07 truenas k3s[5440]: I0122 12:40:07.488501    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:07 truenas k3s[5440]: E0122 12:40:07.488644    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:11.488591094 +0100 CET m=+19.899228970 (durationBeforeRetry 4s). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:07 truenas k3s[5440]: I0122 12:40:07.589641    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:07 truenas k3s[5440]: I0122 12:40:07.589794    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:07 truenas k3s[5440]: E0122 12:40:07.589872    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:11.589816866 +0100 CET m=+20.000454781 (durationBeforeRetry 4s). Error: UnmountVolume.TearDown failed for volume "pvc-88dedeea-6d2f-448f-bafa-658c55aa954b" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:07 truenas k3s[5440]: E0122 12:40:07.590040    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:11.589973572 +0100 CET m=+20.000611505 (durationBeforeRetry 4s). Error: UnmountVolume.TearDown failed for volume "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.196234    5440 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.196306    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs podName:12818b5c-7610-4023-bce4-e74fe62310b4 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.196286088 +0100 CET m=+24.606923960 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs") pod "openebs-zfs-node-cfml4" (UID: "12818b5c-7610-4023-bce4-e74fe62310b4") : object "kube-system"/"openebs-zfspv-bin" not registered
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.581003    5440 range_allocator.go:109] No Secondary Service CIDR provided. Skipping filtering out secondary service addresses.
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.581101    5440 controllermanager.go:622] Started "nodeipam"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.581294    5440 node_ipam_controller.go:155] Starting ipam controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.581318    5440 shared_informer.go:270] Waiting for caches to sync for node
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.585601    5440 core.go:207] failed to start cloud node lifecycle controller: no cloud provider provided
Jan 22 12:40:08 truenas k3s[5440]: W0122 12:40:08.585636    5440 controllermanager.go:600] Skipping "cloud-node-lifecycle"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.590343    5440 controllermanager.go:622] Started "pvc-protection"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.591077    5440 pvc_protection_controller.go:99] "Starting PVC protection controller"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.592533    5440 shared_informer.go:270] Waiting for caches to sync for PVC protection
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.596210    5440 controllermanager.go:622] Started "daemonset"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.597045    5440 daemon_controller.go:267] Starting daemon sets controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.597067    5440 shared_informer.go:270] Waiting for caches to sync for daemon sets
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.600268    5440 controllermanager.go:622] Started "job"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.601027    5440 job_controller.go:191] Starting job controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.601052    5440 shared_informer.go:270] Waiting for caches to sync for job
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.604676    5440 controllermanager.go:622] Started "csrapproving"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.604785    5440 certificate_controller.go:112] Starting certificate controller "csrapproving"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.604801    5440 shared_informer.go:270] Waiting for caches to sync for certificate-csrapproving
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.608734    5440 controllermanager.go:622] Started "ttl"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.609252    5440 ttl_controller.go:120] Starting TTL controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.609275    5440 shared_informer.go:270] Waiting for caches to sync for TTL
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.612373    5440 controllermanager.go:622] Started "ttl-after-finished"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.612566    5440 ttlafterfinished_controller.go:104] Starting TTL after finished controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.612606    5440 shared_informer.go:270] Waiting for caches to sync for TTL after finished
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.616343    5440 controllermanager.go:622] Started "endpoint"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.616501    5440 endpoints_controller.go:178] Starting endpoint controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.616523    5440 shared_informer.go:270] Waiting for caches to sync for endpoint
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.620279    5440 controllermanager.go:622] Started "replicaset"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.620562    5440 replica_set.go:201] Starting replicaset controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.620591    5440 shared_informer.go:270] Waiting for caches to sync for ReplicaSet
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.624112    5440 node_lifecycle_controller.go:492] Controller will reconcile labels.
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.624145    5440 controllermanager.go:622] Started "nodelifecycle"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.624284    5440 node_lifecycle_controller.go:527] Sending events to api server.
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.624314    5440 node_lifecycle_controller.go:538] Starting node controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.624322    5440 shared_informer.go:270] Waiting for caches to sync for taint
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.627821    5440 controllermanager.go:622] Started "pv-protection"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.628022    5440 pv_protection_controller.go:75] Starting PV protection controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.628048    5440 shared_informer.go:270] Waiting for caches to sync for PV protection
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.631370    5440 core.go:92] Failed to start service controller: WARNING: no cloud provider provided, services of type LoadBalancer will fail
Jan 22 12:40:08 truenas k3s[5440]: W0122 12:40:08.631412    5440 controllermanager.go:600] Skipping "service"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.636478    5440 controllermanager.go:622] Started "attachdetach"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.636919    5440 attach_detach_controller.go:328] Starting attach detach controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.636950    5440 shared_informer.go:270] Waiting for caches to sync for attach detach
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.640643    5440 controllermanager.go:622] Started "endpointslice"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.641038    5440 endpointslice_controller.go:257] Starting endpoint slice controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.641059    5440 shared_informer.go:270] Waiting for caches to sync for endpoint_slice
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.644580    5440 controllermanager.go:622] Started "replicationcontroller"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.644803    5440 replica_set.go:201] Starting replicationcontroller controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.644829    5440 shared_informer.go:270] Waiting for caches to sync for ReplicationController
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.671065    5440 controllermanager.go:622] Started "namespace"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.671310    5440 namespace_controller.go:195] Starting namespace controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.671340    5440 shared_informer.go:270] Waiting for caches to sync for namespace
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.676542    5440 controllermanager.go:622] Started "deployment"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.677102    5440 deployment_controller.go:154] "Starting controller" controller="deployment"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.677133    5440 shared_informer.go:270] Waiting for caches to sync for deployment
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.690401    5440 controllermanager.go:622] Started "horizontalpodautoscaling"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.690590    5440 horizontal.go:181] Starting HPA controller
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.690617    5440 shared_informer.go:270] Waiting for caches to sync for HPA
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.700810    5440 projected.go:292] Couldn't get configMap cert-manager/kube-root-ca.crt: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.700855    5440 projected.go:198] Error preparing data for projected volume kube-api-access-5kbbr for pod cert-manager/cert-manager-cainjector-ffb4747bb-sfktp: object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.700954    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr podName:32eb535e-4f44-4b9d-b331-1450dc7b1e04 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.700918992 +0100 CET m=+25.111556889 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-5kbbr" (UniqueName: "kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr") pod "cert-manager-cainjector-ffb4747bb-sfktp" (UID: "32eb535e-4f44-4b9d-b331-1450dc7b1e04") : object "cert-manager"/"kube-root-ca.crt" not registered
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.701517    5440 shared_informer.go:270] Waiting for caches to sync for resource quota
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.718025    5440 shared_informer.go:277] Caches are synced for certificate-csrsigning-kube-apiserver-client
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.718104    5440 shared_informer.go:277] Caches are synced for certificate-csrsigning-legacy-unknown
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.727179    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432050
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.731703    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432050
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.743315    5440 shared_informer.go:270] Waiting for caches to sync for garbage collector
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.745730    5440 shared_informer.go:277] Caches are synced for ReplicationController
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.776130    5440 shared_informer.go:277] Caches are synced for namespace
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.792182    5440 shared_informer.go:277] Caches are synced for HPA
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.792449    5440 shared_informer.go:277] Caches are synced for crt configmap
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.792966    5440 shared_informer.go:277] Caches are synced for PVC protection
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.796664    5440 shared_informer.go:277] Caches are synced for ClusterRoleAggregator
Jan 22 12:40:08 truenas k3s[5440]: W0122 12:40:08.798631    5440 actual_state_of_world.go:541] Failed to update statusUpdateNeeded field in actual state of world: Failed to set statusUpdateNeeded to needed true, because nodeName="ix-truenas" does not exist
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.798894    5440 shared_informer.go:277] Caches are synced for GC
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799052    5440 gc_controller.go:207] "Garbage collecting pods" numPods=13
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799094    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/csi-smb-controller-7fbbb8fb6f-7wj2x"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799104    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/coredns-59b4f5bbd5-rxc2v"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799192    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="cert-manager/cert-manager-8444f6f86b-gcmtm"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799266    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/csi-smb-controller-7fbbb8fb6f-cmdzk"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799327    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/snapshot-controller-546868dfb4-jrg5t"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799336    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="cert-manager/cert-manager-webhook-545bd5d7d8-kchdl"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799386    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="cert-manager/cert-manager-cainjector-ffb4747bb-sfktp"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799396    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/snapshot-controller-546868dfb4-thslt"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799445    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="prometheus-operator/prometheus-operator-5dcffb7cb8-zjvqh"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799451    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/openebs-zfs-controller-0"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799094    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/csi-nfs-controller-7b74694749-tfvgv"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799515    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/snapshot-controller-546868dfb4-cq7q7"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.799504    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/csi-nfs-controller-7b74694749-skm24"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.800668    5440 shared_informer.go:277] Caches are synced for service account
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.801191    5440 shared_informer.go:277] Caches are synced for job
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.805872    5440 shared_informer.go:277] Caches are synced for certificate-csrapproving
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.809344    5440 shared_informer.go:277] Caches are synced for TTL
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.810001    5440 shared_informer.go:277] Caches are synced for certificate-csrsigning-kubelet-client
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.810241    5440 shared_informer.go:277] Caches are synced for certificate-csrsigning-kubelet-serving
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.813782    5440 shared_informer.go:277] Caches are synced for TTL after finished
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.816623    5440 shared_informer.go:277] Caches are synced for endpoint
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.822053    5440 shared_informer.go:277] Caches are synced for endpoint_slice_mirroring
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.824569    5440 shared_informer.go:277] Caches are synced for taint
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.824800    5440 node_lifecycle_controller.go:1438] Initializing eviction metric for zone:
Jan 22 12:40:08 truenas k3s[5440]: W0122 12:40:08.825469    5440 node_lifecycle_controller.go:1053] Missing timestamp for Node ix-truenas. Assuming now as a timestamp.
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.825536    5440 node_lifecycle_controller.go:1254] Controller detected that zone  is now in state Normal.
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.825913    5440 shared_informer.go:277] Caches are synced for cronjob
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.826225    5440 event.go:294] "Event occurred" object="ix-truenas" fieldPath="" kind="Node" apiVersion="v1" type="Normal" reason="RegisteredNode" message="Node ix-truenas event: Registered Node ix-truenas in Controller"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.828139    5440 taint_manager.go:206] "Starting NoExecuteTaintManager"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.828182    5440 taint_manager.go:211] "Sending events to api server"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.830111    5440 shared_informer.go:277] Caches are synced for ephemeral
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.841227    5440 shared_informer.go:277] Caches are synced for endpoint_slice
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.845808    5440 shared_informer.go:277] Caches are synced for stateful set
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.859762    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860031    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="csi-provisioner"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860055    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="12818b5c-7610-4023-bce4-e74fe62310b4" containerName="csi-node-driver-registrar"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860072    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="csi-resizer"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860086    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="a0678a1d-5503-40ad-814c-1fcdb3a2c2cb" containerName="nginx"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860100    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="d3e9e993-7ad4-424f-98a3-c02c58c0b8e8" containerName="coredns"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860112    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="c50f201d-9781-4ced-82da-8a9e72317ab2" containerName="intel-gpu-initcontainer"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860124    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="0c60407c-b2f4-4a08-85cb-ec68308d717c" containerName="csi-snapshotter"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860136    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="c50f201d-9781-4ced-82da-8a9e72317ab2" containerName="intel-gpu-plugin"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860148    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="a0678a1d-5503-40ad-814c-1fcdb3a2c2cb" containerName="nextcloud"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860161    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="0d1b468d-7739-4ce5-9926-f9ddc02840cd" containerName="adguard-home"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860173    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="b328df6e-3da7-4d11-947d-86d09a12d2cc" containerName="nextcloud-cronjob"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860184    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="d3cf2959-6304-4036-a8ce-d3cd4a82bf59" containerName="nextcloud-postgres"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860197    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="91f314cd-666f-4394-a370-8ffa1a3af32e" containerName="lb-tcp-443"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860208    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="10c443c5-3839-46a5-b255-537ad4376a4d" containerName="prometheus-operator"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860220    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="89472a65-6558-4aa6-aa8e-59a5692fd061" containerName="cert-manager-controller"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860232    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="99153dad-afdd-4dd5-a92a-589029ee105e" containerName="cert-manager-webhook"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860246    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="5d7fc30d-4e17-422a-a53f-6dc95b471cda" containerName="homarr-init-01-permissions"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860258    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="4c9adf6d-9e15-4725-a0bd-0513c49e51f6" containerName="snapshot-controller"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860270    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="ac6a8298-a65d-4813-a172-f4a34d69516a" containerName="lb-tcp-10582"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860281    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="csi-snapshotter"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860292    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="0c60407c-b2f4-4a08-85cb-ec68308d717c" containerName="liveness-probe"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860303    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="12818b5c-7610-4023-bce4-e74fe62310b4" containerName="openebs-zfs-plugin"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860315    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="openebs-zfs-plugin"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860326    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" containerName="nginx-proxy-manager-system-mariadb-wait"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860337    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="611cb983-5c93-4188-bf0f-28649b87a074" containerName="smb"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860348    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="32eb535e-4f44-4b9d-b331-1450dc7b1e04" containerName="cert-manager-cainjector"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860359    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="1a4f0cdf-d32b-4b4c-b4c7-ebe804ce1731" containerName="snapshot-controller"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860370    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="a0678a1d-5503-40ad-814c-1fcdb3a2c2cb" containerName="init-postgresdb"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860383    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="5d7fc30d-4e17-422a-a53f-6dc95b471cda" containerName="homarr"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860397    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="snapshot-controller"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860410    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" containerName="nginx-proxy-manager"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860424    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="0c60407c-b2f4-4a08-85cb-ec68308d717c" containerName="nfs"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860437    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="43e7445c-bec5-40b6-935e-1f0dcfcb02e1" containerName="nginx-proxy-manager-mariadb"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860450    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="0c60407c-b2f4-4a08-85cb-ec68308d717c" containerName="csi-provisioner"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860463    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="9ce058fc-5961-4e15-b014-dd945da4b9c0" containerName="lb-tcp-80"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860476    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="611cb983-5c93-4188-bf0f-28649b87a074" containerName="csi-provisioner"
Jan 22 12:40:08 truenas k3s[5440]: E0122 12:40:08.860488    5440 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="611cb983-5c93-4188-bf0f-28649b87a074" containerName="liveness-probe"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860863    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="611cb983-5c93-4188-bf0f-28649b87a074" containerName="csi-provisioner"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860884    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="0c60407c-b2f4-4a08-85cb-ec68308d717c" containerName="nfs"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860898    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="c50f201d-9781-4ced-82da-8a9e72317ab2" containerName="intel-gpu-plugin"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860910    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="611cb983-5c93-4188-bf0f-28649b87a074" containerName="smb"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860922    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="611cb983-5c93-4188-bf0f-28649b87a074" containerName="liveness-probe"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860935    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="4c9adf6d-9e15-4725-a0bd-0513c49e51f6" containerName="snapshot-controller"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860949    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="43e7445c-bec5-40b6-935e-1f0dcfcb02e1" containerName="nginx-proxy-manager-mariadb"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860962    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="9ce058fc-5961-4e15-b014-dd945da4b9c0" containerName="lb-tcp-80"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860974    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="91f314cd-666f-4394-a370-8ffa1a3af32e" containerName="lb-tcp-443"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860984    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="12818b5c-7610-4023-bce4-e74fe62310b4" containerName="openebs-zfs-plugin"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.860994    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="snapshot-controller"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861005    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="b328df6e-3da7-4d11-947d-86d09a12d2cc" containerName="nextcloud-cronjob"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861018    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="12818b5c-7610-4023-bce4-e74fe62310b4" containerName="csi-node-driver-registrar"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861028    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="0c60407c-b2f4-4a08-85cb-ec68308d717c" containerName="csi-provisioner"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861039    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="c50f201d-9781-4ced-82da-8a9e72317ab2" containerName="intel-gpu-initcontainer"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861050    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="csi-resizer"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861064    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="d3cf2959-6304-4036-a8ce-d3cd4a82bf59" containerName="nextcloud-postgres"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861074    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="a0678a1d-5503-40ad-814c-1fcdb3a2c2cb" containerName="nginx"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861084    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="csi-snapshotter"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861095    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="0c60407c-b2f4-4a08-85cb-ec68308d717c" containerName="liveness-probe"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861105    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="5d7fc30d-4e17-422a-a53f-6dc95b471cda" containerName="homarr"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861117    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="5d7fc30d-4e17-422a-a53f-6dc95b471cda" containerName="homarr-init-01-permissions"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861128    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="32eb535e-4f44-4b9d-b331-1450dc7b1e04" containerName="cert-manager-cainjector"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861138    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="d3e9e993-7ad4-424f-98a3-c02c58c0b8e8" containerName="coredns"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861149    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="89472a65-6558-4aa6-aa8e-59a5692fd061" containerName="cert-manager-controller"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861159    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" containerName="nginx-proxy-manager"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861170    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="ac6a8298-a65d-4813-a172-f4a34d69516a" containerName="lb-tcp-10582"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861181    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="10c443c5-3839-46a5-b255-537ad4376a4d" containerName="prometheus-operator"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861190    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="0d1b468d-7739-4ce5-9926-f9ddc02840cd" containerName="adguard-home"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861203    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" containerName="nginx-proxy-manager-system-mariadb-wait"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861214    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="a0678a1d-5503-40ad-814c-1fcdb3a2c2cb" containerName="nextcloud"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861225    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="csi-provisioner"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861235    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="1a4f0cdf-d32b-4b4c-b4c7-ebe804ce1731" containerName="snapshot-controller"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861247    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="7868b1d3-b2e7-444f-adde-748acaceb01d" containerName="openebs-zfs-plugin"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861257    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="99153dad-afdd-4dd5-a92a-589029ee105e" containerName="cert-manager-webhook"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861266    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="a0678a1d-5503-40ad-814c-1fcdb3a2c2cb" containerName="init-postgresdb"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861277    5440 memory_manager.go:346] "RemoveStaleState removing state" podUID="0c60407c-b2f4-4a08-85cb-ec68308d717c" containerName="csi-snapshotter"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.861443    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.871216    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.871383    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.871544    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.873182    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.873348    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.873462    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.883457    5440 shared_informer.go:277] Caches are synced for node
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.883518    5440 range_allocator.go:167] Sending events to api server.
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.883548    5440 range_allocator.go:171] Starting range CIDR allocator
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.883557    5440 shared_informer.go:270] Waiting for caches to sync for cidrallocator
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.883572    5440 shared_informer.go:277] Caches are synced for cidrallocator
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.888405    5440 event.go:294] "Event occurred" object="kube-system/openebs-zfs-controller" fieldPath="" kind="StatefulSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="create Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.894084    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.895183    5440 shared_informer.go:277] Caches are synced for persistent volume
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.897327    5440 shared_informer.go:277] Caches are synced for daemon sets
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.899136    5440 event.go:294] "Event occurred" object="kube-system/openebs-zfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Warning" reason="FailedDaemonPod" message="Found failed daemon pod kube-system/openebs-zfs-node-cfml4 on node ix-truenas, will try to kill it"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.901451    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/cdd22586-f645-4925-854e-c3aab74b685c-tmp\") pod \"homarr-fb47944ff-ts4kt\" (UID: \"cdd22586-f645-4925-854e-c3aab74b685c\") " pod="ix-homarr/homarr-fb47944ff-ts4kt"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.901500    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"icons\" (UniqueName: \"kubernetes.io/host-path/cdd22586-f645-4925-854e-c3aab74b685c-icons\") pod \"homarr-fb47944ff-ts4kt\" (UID: \"cdd22586-f645-4925-854e-c3aab74b685c\") " pod="ix-homarr/homarr-fb47944ff-ts4kt"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.901583    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"configs\" (UniqueName: \"kubernetes.io/host-path/cdd22586-f645-4925-854e-c3aab74b685c-configs\") pod \"homarr-fb47944ff-ts4kt\" (UID: \"cdd22586-f645-4925-854e-c3aab74b685c\") " pod="ix-homarr/homarr-fb47944ff-ts4kt"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.904967    5440 shared_informer.go:277] Caches are synced for disruption
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.910730    5440 event.go:294] "Event occurred" object="kube-system/svclb-nginx-proxy-manager-websecure-b7c8312a" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: svclb-nginx-proxy-manager-websecure-b7c8312a-rjgr9"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.915936    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.920738    5440 shared_informer.go:277] Caches are synced for ReplicaSet
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.924606    5440 event.go:294] "Event occurred" object="kube-system/svclb-nginx-proxy-manager-9216e761" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: svclb-nginx-proxy-manager-9216e761-62wsv"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.927671    5440 shared_informer.go:277] Caches are synced for expand
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.928213    5440 shared_informer.go:277] Caches are synced for PV protection
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.930842    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.937092    5440 event.go:294] "Event occurred" object="kube-system/snapshot-controller-546868dfb4" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: snapshot-controller-546868dfb4-t7f9j"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.940593    5440 event.go:294] "Event occurred" object="kube-system/openebs-zfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulDelete" message="Deleted pod: openebs-zfs-node-cfml4"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.940708    5440 event.go:294] "Event occurred" object="kube-system/csi-smb-controller-7fbbb8fb6f" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: csi-smb-controller-7fbbb8fb6f-cwb2s"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.944940    5440 event.go:294] "Event occurred" object="kube-system/csi-nfs-controller-7b74694749" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: csi-nfs-controller-7b74694749-kndwf"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.948010    5440 shared_informer.go:277] Caches are synced for attach detach
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.950053    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.951423    5440 event.go:294] "Event occurred" object="kube-system/svclb-nginx-proxy-manager-web-8ae46abf" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: svclb-nginx-proxy-manager-web-8ae46abf-47frf"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.951972    5440 event.go:294] "Event occurred" object="kube-system/snapshot-controller-546868dfb4" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: snapshot-controller-546868dfb4-4tzj7"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.959528    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.971048    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.972298    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.973273    5440 event.go:294] "Event occurred" object="kube-system/intel-gpu-plugin" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: intel-gpu-plugin-56k8h"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.973852    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.979324    5440 shared_informer.go:277] Caches are synced for deployment
Jan 22 12:40:08 truenas k3s[5440]: I0122 12:40:08.984591    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.015875    5440 event.go:294] "Event occurred" object="kube-system/openebs-zfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: openebs-zfs-node-ztdfw"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.051535    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.631142    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=0c60407c-b2f4-4a08-85cb-ec68308d717c path="/var/lib/kubelet/pods/0c60407c-b2f4-4a08-85cb-ec68308d717c/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.633446    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=10c443c5-3839-46a5-b255-537ad4376a4d path="/var/lib/kubelet/pods/10c443c5-3839-46a5-b255-537ad4376a4d/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.635237    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=12818b5c-7610-4023-bce4-e74fe62310b4 path="/var/lib/kubelet/pods/12818b5c-7610-4023-bce4-e74fe62310b4/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.636988    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=1a4f0cdf-d32b-4b4c-b4c7-ebe804ce1731 path="/var/lib/kubelet/pods/1a4f0cdf-d32b-4b4c-b4c7-ebe804ce1731/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.641239    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=2106327d-69ea-48e4-8e6c-733aef332cf3 path="/var/lib/kubelet/pods/2106327d-69ea-48e4-8e6c-733aef332cf3/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.643005    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=32eb535e-4f44-4b9d-b331-1450dc7b1e04 path="/var/lib/kubelet/pods/32eb535e-4f44-4b9d-b331-1450dc7b1e04/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.644707    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=4c9adf6d-9e15-4725-a0bd-0513c49e51f6 path="/var/lib/kubelet/pods/4c9adf6d-9e15-4725-a0bd-0513c49e51f6/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.646513    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=57e50dd4-f745-4e15-97be-36cdf31d2d66 path="/var/lib/kubelet/pods/57e50dd4-f745-4e15-97be-36cdf31d2d66/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.648018    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=611cb983-5c93-4188-bf0f-28649b87a074 path="/var/lib/kubelet/pods/611cb983-5c93-4188-bf0f-28649b87a074/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.649491    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=6edd7d70-c2f6-4b22-a160-6433ec8d858b path="/var/lib/kubelet/pods/6edd7d70-c2f6-4b22-a160-6433ec8d858b/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.650817    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=7868b1d3-b2e7-444f-adde-748acaceb01d path="/var/lib/kubelet/pods/7868b1d3-b2e7-444f-adde-748acaceb01d/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.652202    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=89472a65-6558-4aa6-aa8e-59a5692fd061 path="/var/lib/kubelet/pods/89472a65-6558-4aa6-aa8e-59a5692fd061/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.654780    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=99153dad-afdd-4dd5-a92a-589029ee105e path="/var/lib/kubelet/pods/99153dad-afdd-4dd5-a92a-589029ee105e/volumes"
Jan 22 12:40:09 truenas k3s[5440]: I0122 12:40:09.656510    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=f7e9b80f-779c-43d5-9f81-5dc9902cb086 path="/var/lib/kubelet/pods/f7e9b80f-779c-43d5-9f81-5dc9902cb086/volumes"
Jan 22 12:40:10 truenas k3s[5440]: I0122 12:40:10.002187    5440 shared_informer.go:277] Caches are synced for resource quota
Jan 22 12:40:10 truenas k3s[5440]: I0122 12:40:10.043780    5440 shared_informer.go:277] Caches are synced for garbage collector
Jan 22 12:40:10 truenas k3s[5440]: I0122 12:40:10.054278    5440 shared_informer.go:277] Caches are synced for resource quota
Jan 22 12:40:10 truenas k3s[5440]: I0122 12:40:10.123523    5440 shared_informer.go:277] Caches are synced for garbage collector
Jan 22 12:40:10 truenas k3s[5440]: I0122 12:40:10.123555    5440 garbagecollector.go:163] Garbage collector: all resource monitors have synced. Proceeding to collect garbage
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.530382    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:11 truenas k3s[5440]: E0122 12:40:11.530602    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.530546498 +0100 CET m=+27.941184399 (durationBeforeRetry 8s). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.631029    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:11 truenas k3s[5440]: E0122 12:40:11.631229    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.631162216 +0100 CET m=+28.041800129 (durationBeforeRetry 8s). Error: UnmountVolume.TearDown failed for volume "pvc-88dedeea-6d2f-448f-bafa-658c55aa954b" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.631302    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:11 truenas k3s[5440]: E0122 12:40:11.631450    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de podName:7e0f78dc-2724-42c6-97a7-76f79fcb2b9d nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.631407205 +0100 CET m=+28.042045106 (durationBeforeRetry 8s). Error: UnmountVolume.TearDown failed for volume "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.732414    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-njshv\" (UniqueName: \"kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv\") pod \"89472a65-6558-4aa6-aa8e-59a5692fd061\" (UID: \"89472a65-6558-4aa6-aa8e-59a5692fd061\") "
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.732727    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"nextcloud-data\" (UniqueName: \"kubernetes.io/host-path/69515bdc-e252-4875-b2ea-fb5088da2ac3-nextcloud-data\") pod \"nextcloud-848cdcb98d-dmtrz\" (UID: \"69515bdc-e252-4875-b2ea-fb5088da2ac3\") " pod="ix-nextcloud/nextcloud-848cdcb98d-dmtrz"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.732887    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"nginx-configuration\" (UniqueName: \"kubernetes.io/configmap/69515bdc-e252-4875-b2ea-fb5088da2ac3-nginx-configuration\") pod \"nextcloud-848cdcb98d-dmtrz\" (UID: \"69515bdc-e252-4875-b2ea-fb5088da2ac3\") " pod="ix-nextcloud/nextcloud-848cdcb98d-dmtrz"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.732930    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-h6rgd\" (UniqueName: \"kubernetes.io/projected/776d86de-3791-4530-bb23-5b9b27966e0a-kube-api-access-h6rgd\") pod \"cert-manager-webhook-545bd5d7d8-kf2sp\" (UID: \"776d86de-3791-4530-bb23-5b9b27966e0a\") " pod="cert-manager/cert-manager-webhook-545bd5d7d8-kf2sp"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.732988    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/8267c80e-8af2-4396-87cb-4a88c76f3956-varlogs\") pod \"nginx-proxy-manager-mariadb-6b689fccbb-xw6gd\" (UID: \"8267c80e-8af2-4396-87cb-4a88c76f3956\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-mariadb-6b689fccbb-xw6gd"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733053    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-xs49j\" (UniqueName: \"kubernetes.io/projected/17f7d6e2-e33c-4b09-918d-8c5e08254285-kube-api-access-xs49j\") pod \"cert-manager-cainjector-ffb4747bb-8mnsh\" (UID: \"17f7d6e2-e33c-4b09-918d-8c5e08254285\") " pod="cert-manager/cert-manager-cainjector-ffb4747bb-8mnsh"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733120    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cert-secret-volume\" (UniqueName: \"kubernetes.io/secret/69515bdc-e252-4875-b2ea-fb5088da2ac3-cert-secret-volume\") pod \"nextcloud-848cdcb98d-dmtrz\" (UID: \"69515bdc-e252-4875-b2ea-fb5088da2ac3\") " pod="ix-nextcloud/nextcloud-848cdcb98d-dmtrz"
Jan 22 12:40:11 truenas k3s[5440]: E0122 12:40:11.733217    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.233195013 +0100 CET m=+20.643832899 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733224    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/8267c80e-8af2-4396-87cb-4a88c76f3956-shared\") pod \"nginx-proxy-manager-mariadb-6b689fccbb-xw6gd\" (UID: \"8267c80e-8af2-4396-87cb-4a88c76f3956\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-mariadb-6b689fccbb-xw6gd"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733308    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/8267c80e-8af2-4396-87cb-4a88c76f3956-varrun\") pod \"nginx-proxy-manager-mariadb-6b689fccbb-xw6gd\" (UID: \"8267c80e-8af2-4396-87cb-4a88c76f3956\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-mariadb-6b689fccbb-xw6gd"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733365    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"nextcloud-configuration\" (UniqueName: \"kubernetes.io/configmap/69515bdc-e252-4875-b2ea-fb5088da2ac3-nextcloud-configuration\") pod \"nextcloud-848cdcb98d-dmtrz\" (UID: \"69515bdc-e252-4875-b2ea-fb5088da2ac3\") " pod="ix-nextcloud/nextcloud-848cdcb98d-dmtrz"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733453    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-dx558\" (UniqueName: \"kubernetes.io/projected/b9cf08fb-e370-4bcd-a235-b81e191f43d7-kube-api-access-dx558\") pod \"prometheus-operator-5dcffb7cb8-2w5r6\" (UID: \"b9cf08fb-e370-4bcd-a235-b81e191f43d7\") " pod="prometheus-operator/prometheus-operator-5dcffb7cb8-2w5r6"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733536    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-9qzj5\" (UniqueName: \"kubernetes.io/projected/9787184a-50d7-4922-b62b-f8e816d41ee4-kube-api-access-9qzj5\") pod \"cert-manager-8444f6f86b-pcclz\" (UID: \"9787184a-50d7-4922-b62b-f8e816d41ee4\") " pod="cert-manager/cert-manager-8444f6f86b-pcclz"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733661    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"work\" (UniqueName: \"kubernetes.io/host-path/c70586b0-36c2-4021-a233-a20a6d66ca67-work\") pod \"adguard-home-78766c98d5-fp67l\" (UID: \"c70586b0-36c2-4021-a233-a20a6d66ca67\") " pod="ix-adguard-home/adguard-home-78766c98d5-fp67l"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733731    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"devshm\" (UniqueName: \"kubernetes.io/empty-dir/8267c80e-8af2-4396-87cb-4a88c76f3956-devshm\") pod \"nginx-proxy-manager-mariadb-6b689fccbb-xw6gd\" (UID: \"8267c80e-8af2-4396-87cb-4a88c76f3956\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-mariadb-6b689fccbb-xw6gd"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733791    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/8267c80e-8af2-4396-87cb-4a88c76f3956-tmp\") pod \"nginx-proxy-manager-mariadb-6b689fccbb-xw6gd\" (UID: \"8267c80e-8af2-4396-87cb-4a88c76f3956\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-mariadb-6b689fccbb-xw6gd"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733850    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-xjq6b\" (UniqueName: \"kubernetes.io/projected/69515bdc-e252-4875-b2ea-fb5088da2ac3-kube-api-access-xjq6b\") pod \"nextcloud-848cdcb98d-dmtrz\" (UID: \"69515bdc-e252-4875-b2ea-fb5088da2ac3\") " pod="ix-nextcloud/nextcloud-848cdcb98d-dmtrz"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733931    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"conf\" (UniqueName: \"kubernetes.io/host-path/c70586b0-36c2-4021-a233-a20a6d66ca67-conf\") pod \"adguard-home-78766c98d5-fp67l\" (UID: \"c70586b0-36c2-4021-a233-a20a6d66ca67\") " pod="ix-adguard-home/adguard-home-78766c98d5-fp67l"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.733993    5440 reconciler_common.go:295] "Volume detached for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/12818b5c-7610-4023-bce4-e74fe62310b4-chroot-zfs\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.734029    5440 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-5kbbr\" (UniqueName: \"kubernetes.io/projected/32eb535e-4f44-4b9d-b331-1450dc7b1e04-kube-api-access-5kbbr\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.735136    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv" (OuterVolumeSpecName: "kube-api-access-njshv") pod "89472a65-6558-4aa6-aa8e-59a5692fd061" (UID: "89472a65-6558-4aa6-aa8e-59a5692fd061"). InnerVolumeSpecName "kube-api-access-njshv". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.834808    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.834912    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-l2mzr\" (UniqueName: \"kubernetes.io/projected/379fedfd-07ec-4998-90f0-c20084450fa3-kube-api-access-l2mzr\") pod \"intel-gpu-plugin-56k8h\" (UID: \"379fedfd-07ec-4998-90f0-c20084450fa3\") " pod="kube-system/intel-gpu-plugin-56k8h"
Jan 22 12:40:11 truenas k3s[5440]: E0122 12:40:11.835008    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.334968012 +0100 CET m=+20.745605910 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.835109    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-5n9c2\" (UniqueName: \"kubernetes.io/projected/865493fb-8f80-4b69-97f0-5950123a2fd1-kube-api-access-5n9c2\") pod \"snapshot-controller-546868dfb4-t7f9j\" (UID: \"865493fb-8f80-4b69-97f0-5950123a2fd1\") " pod="kube-system/snapshot-controller-546868dfb4-t7f9j"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.835304    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"devfs\" (UniqueName: \"kubernetes.io/host-path/379fedfd-07ec-4998-90f0-c20084450fa3-devfs\") pod \"intel-gpu-plugin-56k8h\" (UID: \"379fedfd-07ec-4998-90f0-c20084450fa3\") " pod="kube-system/intel-gpu-plugin-56k8h"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.835354    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/50d317cf-3356-4bbd-a7d9-200032db5545-encr-keys\") pod \"openebs-zfs-node-ztdfw\" (UID: \"50d317cf-3356-4bbd-a7d9-200032db5545\") " pod="kube-system/openebs-zfs-node-ztdfw"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.835394    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/50d317cf-3356-4bbd-a7d9-200032db5545-host-root\") pod \"openebs-zfs-node-ztdfw\" (UID: \"50d317cf-3356-4bbd-a7d9-200032db5545\") " pod="kube-system/openebs-zfs-node-ztdfw"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.835459    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-zz6md\" (UniqueName: \"kubernetes.io/projected/18bf6bd1-5d07-4bc5-a161-3d9fece77d7b-kube-api-access-zz6md\") pod \"csi-nfs-controller-7b74694749-kndwf\" (UID: \"18bf6bd1-5d07-4bc5-a161-3d9fece77d7b\") " pod="kube-system/csi-nfs-controller-7b74694749-kndwf"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.835750    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kubeletsockets\" (UniqueName: \"kubernetes.io/host-path/379fedfd-07ec-4998-90f0-c20084450fa3-kubeletsockets\") pod \"intel-gpu-plugin-56k8h\" (UID: \"379fedfd-07ec-4998-90f0-c20084450fa3\") " pod="kube-system/intel-gpu-plugin-56k8h"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.835793    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"nfd-source-hooks\" (UniqueName: \"kubernetes.io/host-path/379fedfd-07ec-4998-90f0-c20084450fa3-nfd-source-hooks\") pod \"intel-gpu-plugin-56k8h\" (UID: \"379fedfd-07ec-4998-90f0-c20084450fa3\") " pod="kube-system/intel-gpu-plugin-56k8h"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.835826    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-7sz2x\" (UniqueName: \"kubernetes.io/projected/3de3ffec-3840-4eb5-8d77-c983dec3970a-kube-api-access-7sz2x\") pod \"openebs-zfs-controller-0\" (UID: \"3de3ffec-3840-4eb5-8d77-c983dec3970a\") " pod="kube-system/openebs-zfs-controller-0"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836102    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/18bf6bd1-5d07-4bc5-a161-3d9fece77d7b-pods-mount-dir\") pod \"csi-nfs-controller-7b74694749-kndwf\" (UID: \"18bf6bd1-5d07-4bc5-a161-3d9fece77d7b\") " pod="kube-system/csi-nfs-controller-7b74694749-kndwf"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836162    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/50d317cf-3356-4bbd-a7d9-200032db5545-registration-dir\") pod \"openebs-zfs-node-ztdfw\" (UID: \"50d317cf-3356-4bbd-a7d9-200032db5545\") " pod="kube-system/openebs-zfs-node-ztdfw"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836220    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-l6ddq\" (UniqueName: \"kubernetes.io/projected/50d317cf-3356-4bbd-a7d9-200032db5545-kube-api-access-l6ddq\") pod \"openebs-zfs-node-ztdfw\" (UID: \"50d317cf-3356-4bbd-a7d9-200032db5545\") " pod="kube-system/openebs-zfs-node-ztdfw"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836263    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/b6fb13d7-35bd-4f86-80f0-842fa14c8e82-socket-dir\") pod \"csi-smb-controller-7fbbb8fb6f-cwb2s\" (UID: \"b6fb13d7-35bd-4f86-80f0-842fa14c8e82\") " pod="kube-system/csi-smb-controller-7fbbb8fb6f-cwb2s"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836308    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-64vzx\" (UniqueName: \"kubernetes.io/projected/6ac5c6b3-e0a5-4334-90ea-05d5c529f894-kube-api-access-64vzx\") pod \"snapshot-controller-546868dfb4-4tzj7\" (UID: \"6ac5c6b3-e0a5-4334-90ea-05d5c529f894\") " pod="kube-system/snapshot-controller-546868dfb4-4tzj7"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836349    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"sysfs\" (UniqueName: \"kubernetes.io/host-path/379fedfd-07ec-4998-90f0-c20084450fa3-sysfs\") pod \"intel-gpu-plugin-56k8h\" (UID: \"379fedfd-07ec-4998-90f0-c20084450fa3\") " pod="kube-system/intel-gpu-plugin-56k8h"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836414    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/50d317cf-3356-4bbd-a7d9-200032db5545-chroot-zfs\") pod \"openebs-zfs-node-ztdfw\" (UID: \"50d317cf-3356-4bbd-a7d9-200032db5545\") " pod="kube-system/openebs-zfs-node-ztdfw"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836467    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/50d317cf-3356-4bbd-a7d9-200032db5545-plugin-dir\") pod \"openebs-zfs-node-ztdfw\" (UID: \"50d317cf-3356-4bbd-a7d9-200032db5545\") " pod="kube-system/openebs-zfs-node-ztdfw"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836526    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/50d317cf-3356-4bbd-a7d9-200032db5545-pods-mount-dir\") pod \"openebs-zfs-node-ztdfw\" (UID: \"50d317cf-3356-4bbd-a7d9-200032db5545\") " pod="kube-system/openebs-zfs-node-ztdfw"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836589    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-859gt\" (UniqueName: \"kubernetes.io/projected/b6fb13d7-35bd-4f86-80f0-842fa14c8e82-kube-api-access-859gt\") pod \"csi-smb-controller-7fbbb8fb6f-cwb2s\" (UID: \"b6fb13d7-35bd-4f86-80f0-842fa14c8e82\") " pod="kube-system/csi-smb-controller-7fbbb8fb6f-cwb2s"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836734    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/18bf6bd1-5d07-4bc5-a161-3d9fece77d7b-socket-dir\") pod \"csi-nfs-controller-7b74694749-kndwf\" (UID: \"18bf6bd1-5d07-4bc5-a161-3d9fece77d7b\") " pod="kube-system/csi-nfs-controller-7b74694749-kndwf"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836787    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/50d317cf-3356-4bbd-a7d9-200032db5545-device-dir\") pod \"openebs-zfs-node-ztdfw\" (UID: \"50d317cf-3356-4bbd-a7d9-200032db5545\") " pod="kube-system/openebs-zfs-node-ztdfw"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836825    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/3de3ffec-3840-4eb5-8d77-c983dec3970a-socket-dir\") pod \"openebs-zfs-controller-0\" (UID: \"3de3ffec-3840-4eb5-8d77-c983dec3970a\") " pod="kube-system/openebs-zfs-controller-0"
Jan 22 12:40:11 truenas k3s[5440]: I0122 12:40:11.836871    5440 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-njshv\" (UniqueName: \"kubernetes.io/projected/89472a65-6558-4aa6-aa8e-59a5692fd061-kube-api-access-njshv\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:11 truenas k3s[5440]: E0122 12:40:11.938791    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.438774112 +0100 CET m=+20.849411982 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.039871    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.040032    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.539995782 +0100 CET m=+20.950633662 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.141450    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.641433525 +0100 CET m=+21.052071394 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.241926    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.242090    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.742042812 +0100 CET m=+21.152680703 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.242606    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.74259176 +0100 CET m=+21.153229644 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.343603    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.343793    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.843747126 +0100 CET m=+21.254385019 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.344207    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.844192274 +0100 CET m=+21.254830158 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.444542    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.444581    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.94453478 +0100 CET m=+21.355172664 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.445025    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:12.945009389 +0100 CET m=+21.355647271 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.545414    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.545591    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.045546434 +0100 CET m=+21.456184318 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.546046    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.046030729 +0100 CET m=+21.456668617 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.646016    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.646175    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.146133791 +0100 CET m=+21.556771672 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.646959    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.146942097 +0100 CET m=+21.557579979 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.747855    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.748278    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.24822319 +0100 CET m=+21.658861077 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.849082    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.349061743 +0100 CET m=+21.759699628 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.854712    5440 request.go:690] Waited for 1.017324493s due to client-side throttling, not priority and fairness, request: POST:https://127.0.0.1:6443/api/v1/namespaces/ix-nextcloud/serviceaccounts/default/token
Jan 22 12:40:12 truenas k3s[5440]: I0122 12:40:12.950138    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.950283    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.450243041 +0100 CET m=+21.860880929 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:12 truenas k3s[5440]: E0122 12:40:12.950681    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.450665785 +0100 CET m=+21.861303667 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: I0122 12:40:13.050630    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.051041    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.551000333 +0100 CET m=+21.961638213 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.152253    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.652231595 +0100 CET m=+22.062869467 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: I0122 12:40:13.253678    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.254037    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.75399039 +0100 CET m=+22.164628282 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.254510    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.754494433 +0100 CET m=+22.165132314 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: I0122 12:40:13.355593    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.355958    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.855913949 +0100 CET m=+22.266551834 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.356488    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.856473589 +0100 CET m=+22.267111496 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: I0122 12:40:13.456782    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.456881    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.956842138 +0100 CET m=+22.367480022 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.457313    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:13.9572969 +0100 CET m=+22.367934781 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: I0122 12:40:13.557899    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.558320    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.058277312 +0100 CET m=+22.468915196 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: I0122 12:40:13.630858    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=89472a65-6558-4aa6-aa8e-59a5692fd061 path="/var/lib/kubelet/pods/89472a65-6558-4aa6-aa8e-59a5692fd061/volumes"
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.658986    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.158966147 +0100 CET m=+22.569604033 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: I0122 12:40:13.759819    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.759984    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.259941042 +0100 CET m=+22.670578940 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.760412    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.260396162 +0100 CET m=+22.671034033 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: I0122 12:40:13.861524    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.862082    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.362044356 +0100 CET m=+22.772682240 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:13 truenas k3s[5440]: E0122 12:40:13.970506    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.470485093 +0100 CET m=+22.881122964 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.071052    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.071227    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.571172007 +0100 CET m=+22.981809893 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.072078    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.572059491 +0100 CET m=+22.982697377 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.173542    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.173783    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.673731377 +0100 CET m=+23.084369265 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.174321    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.674306955 +0100 CET m=+23.084944839 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.275146    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.275294    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.775248303 +0100 CET m=+23.185886185 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.275676    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.775665158 +0100 CET m=+23.186303030 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.376916    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.377188    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.877146371 +0100 CET m=+23.287784255 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.478210    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:14.978191688 +0100 CET m=+23.388829568 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.580520    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.580789    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.080708422 +0100 CET m=+23.491346307 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.691468    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.191446886 +0100 CET m=+23.602084772 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.791912    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.792296    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.292252127 +0100 CET m=+23.702890003 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: E0122 12:40:14.900287    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.400265988 +0100 CET m=+23.810903857 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.912499    5440 event.go:294] "Event occurred" object="kube-system/coredns-59b4f5bbd5" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: coredns-59b4f5bbd5-9l2gw"
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.931095    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.934145    5440 controller.go:615] quota admission added evaluator for: endpointslices.discovery.k8s.io
Jan 22 12:40:14 truenas k3s[5440]: W0122 12:40:14.979827    5440 endpointslice_controller.go:302] Error syncing endpoint slices for service "kube-system/kube-dns", retrying. Error: EndpointSlice informer cache is out of date
Jan 22 12:40:14 truenas k3s[5440]: I0122 12:40:14.985329    5440 controller.go:615] quota admission added evaluator for: endpoints
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.002013    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.004179    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.504126044 +0100 CET m=+23.914764236 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.008062    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.508032721 +0100 CET m=+23.918670605 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.108994    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.109119    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/620043ec-9e47-4a26-bbaf-6e486d21674e-config-volume\") pod \"coredns-59b4f5bbd5-9l2gw\" (UID: \"620043ec-9e47-4a26-bbaf-6e486d21674e\") " pod="kube-system/coredns-59b4f5bbd5-9l2gw"
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.109169    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"custom-config-volume\" (UniqueName: \"kubernetes.io/configmap/620043ec-9e47-4a26-bbaf-6e486d21674e-custom-config-volume\") pod \"coredns-59b4f5bbd5-9l2gw\" (UID: \"620043ec-9e47-4a26-bbaf-6e486d21674e\") " pod="kube-system/coredns-59b4f5bbd5-9l2gw"
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.109214    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-mjhhw\" (UniqueName: \"kubernetes.io/projected/620043ec-9e47-4a26-bbaf-6e486d21674e-kube-api-access-mjhhw\") pod \"coredns-59b4f5bbd5-9l2gw\" (UID: \"620043ec-9e47-4a26-bbaf-6e486d21674e\") " pod="kube-system/coredns-59b4f5bbd5-9l2gw"
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.109380    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.609341906 +0100 CET m=+24.019979782 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.210577    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.710553869 +0100 CET m=+24.121191761 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.317420    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.317621    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.817577507 +0100 CET m=+24.228215387 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.418681    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:15.918659889 +0100 CET m=+24.329297761 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.519298    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.519534    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.019490821 +0100 CET m=+24.430128704 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.620386    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.120365746 +0100 CET m=+24.531003630 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.720835    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.720993    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.220939636 +0100 CET m=+24.631577524 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.721551    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.221536617 +0100 CET m=+24.632174500 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: I0122 12:40:15.821626    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.821938    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.321899197 +0100 CET m=+24.732537079 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:15 truenas k3s[5440]: E0122 12:40:15.929163    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.429133598 +0100 CET m=+24.839771483 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.030637    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.030992    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.530939952 +0100 CET m=+24.941577838 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.031837    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.531817756 +0100 CET m=+24.942455642 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.132811    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.133108    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.63306715 +0100 CET m=+25.043705034 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.133714    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.633697589 +0100 CET m=+25.044335473 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.234108    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.234426    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.734389342 +0100 CET m=+25.145027224 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.237719    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.737696035 +0100 CET m=+25.148333906 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.338828    5440 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-postgres-774595cffb" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-postgres-774595cffb-qxhcr"
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.342261    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.342475    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.842433841 +0100 CET m=+25.253071730 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.353556    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.445084    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"postgres-backup\" (UniqueName: \"kubernetes.io/host-path/7a6aa2de-a975-4294-a884-84d4f234fe1e-postgres-backup\") pod \"nextcloud-postgres-774595cffb-qxhcr\" (UID: \"7a6aa2de-a975-4294-a884-84d4f234fe1e\") " pod="ix-nextcloud/nextcloud-postgres-774595cffb-qxhcr"
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.446905    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-pgpzz\" (UniqueName: \"kubernetes.io/projected/7a6aa2de-a975-4294-a884-84d4f234fe1e-kube-api-access-pgpzz\") pod \"nextcloud-postgres-774595cffb-qxhcr\" (UID: \"7a6aa2de-a975-4294-a884-84d4f234fe1e\") " pod="ix-nextcloud/nextcloud-postgres-774595cffb-qxhcr"
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.447148    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"postgres-data\" (UniqueName: \"kubernetes.io/host-path/7a6aa2de-a975-4294-a884-84d4f234fe1e-postgres-data\") pod \"nextcloud-postgres-774595cffb-qxhcr\" (UID: \"7a6aa2de-a975-4294-a884-84d4f234fe1e\") " pod="ix-nextcloud/nextcloud-postgres-774595cffb-qxhcr"
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.447659    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:16.947633174 +0100 CET m=+25.358271048 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.548578    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.549108    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.049056503 +0100 CET m=+25.459694391 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.554957    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.053684701 +0100 CET m=+25.464322589 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.654166    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.654558    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.154517336 +0100 CET m=+25.565155221 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.756501    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.256478994 +0100 CET m=+25.667116879 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: I0122 12:40:16.861110    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.861556    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.361514318 +0100 CET m=+25.772152206 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:16 truenas k3s[5440]: E0122 12:40:16.964117    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.464093886 +0100 CET m=+25.874731767 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: I0122 12:40:17.064372    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.064739    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.564688427 +0100 CET m=+25.975326315 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.166807    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.666785906 +0100 CET m=+26.077423785 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: I0122 12:40:17.268866    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.269137    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.769118883 +0100 CET m=+26.179756769 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.273909    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.773890617 +0100 CET m=+26.184528482 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: I0122 12:40:17.379432    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.379642    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.879600988 +0100 CET m=+26.290238873 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.480305    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:17.980285756 +0100 CET m=+26.390923632 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: I0122 12:40:17.580424    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.581142    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.081090492 +0100 CET m=+26.491728378 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.582901    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.082877872 +0100 CET m=+26.493515749 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: I0122 12:40:17.682984    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.683591    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.183566255 +0100 CET m=+26.594204139 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.696167    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.196147974 +0100 CET m=+26.606785849 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: I0122 12:40:17.700838    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="prometheus-operator/prometheus-operator-5dcffb7cb8-2w5r6" podStartSLOduration=297.700547355 pod.CreationTimestamp="2024-01-22 12:35:20 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:17.27379377 +0100 CET m=+25.684431661" watchObservedRunningTime="2024-01-22 12:40:17.700547355 +0100 CET m=+26.111185234"
Jan 22 12:40:17 truenas k3s[5440]: I0122 12:40:17.796648    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.804346    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.302364981 +0100 CET m=+26.713002865 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.807905    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.307880757 +0100 CET m=+26.718518632 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:17 truenas k3s[5440]: I0122 12:40:17.908806    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:17 truenas k3s[5440]: E0122 12:40:17.909102    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.409058739 +0100 CET m=+26.819696622 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.011062    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.511041658 +0100 CET m=+26.921679539 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.127905    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.128435    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.628382054 +0100 CET m=+27.039019937 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.229780    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.729757506 +0100 CET m=+27.140395376 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.330017    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.330243    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.830195247 +0100 CET m=+27.240833127 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.431814    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:18.931788733 +0100 CET m=+27.342426609 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.465290    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="cert-manager/cert-manager-cainjector-ffb4747bb-8mnsh" podStartSLOduration=295.465223976 pod.CreationTimestamp="2024-01-22 12:35:23 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:18.463276112 +0100 CET m=+26.873913994" watchObservedRunningTime="2024-01-22 12:40:18.465223976 +0100 CET m=+26.875861852"
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.532719    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.533140    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.033089207 +0100 CET m=+27.443727082 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.634136    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.134119376 +0100 CET m=+27.544757242 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.734543    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.734922    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.234879848 +0100 CET m=+27.645517732 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.742359    5440 controller.go:615] quota admission added evaluator for: zfsnodes.zfs.openebs.io
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.837746    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.337716383 +0100 CET m=+27.748354266 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.881358    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/svclb-nginx-proxy-manager-web-8ae46abf-47frf" podStartSLOduration=10.881306571 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:18.876691768 +0100 CET m=+27.287329655" watchObservedRunningTime="2024-01-22 12:40:18.881306571 +0100 CET m=+27.291944454"
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.897720    5440 event.go:294] "Event occurred" object="ix-nginx-proxy-manager/nginx-proxy-manager-web" fieldPath="" kind="Service" apiVersion="v1" type="Normal" reason="UpdatedLoadBalancer" message="Updated LoadBalancer with new IPs: [] -> [192.168.0.100]"
Jan 22 12:40:18 truenas k3s[5440]: I0122 12:40:18.937600    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:18 truenas k3s[5440]: E0122 12:40:18.937835    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.437786556 +0100 CET m=+27.848424444 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: E0122 12:40:19.039516    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.539494841 +0100 CET m=+27.950132709 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.140138    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:19 truenas k3s[5440]: E0122 12:40:19.140335    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.640288463 +0100 CET m=+28.050926355 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: E0122 12:40:19.141879    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.641857193 +0100 CET m=+28.052495079 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.242603    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:19 truenas k3s[5440]: E0122 12:40:19.243017    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.742978438 +0100 CET m=+28.153616320 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.265491    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/svclb-nginx-proxy-manager-9216e761-62wsv" podStartSLOduration=11.265438616 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:19.265199908 +0100 CET m=+27.675837804" watchObservedRunningTime="2024-01-22 12:40:19.265438616 +0100 CET m=+27.676076495"
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.281775    5440 event.go:294] "Event occurred" object="ix-nginx-proxy-manager/nginx-proxy-manager" fieldPath="" kind="Service" apiVersion="v1" type="Normal" reason="UpdatedLoadBalancer" message="Updated LoadBalancer with new IPs: [] -> [192.168.0.100]"
Jan 22 12:40:19 truenas k3s[5440]: E0122 12:40:19.344502    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.844481445 +0100 CET m=+28.255119316 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.445061    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:19 truenas k3s[5440]: E0122 12:40:19.445209    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.945165042 +0100 CET m=+28.355802929 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: E0122 12:40:19.445634    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName: nodeName:}" failed. No retries permitted until 2024-01-22 12:40:19.945620403 +0100 CET m=+28.356258281 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" (UID: "8267c80e-8af2-4396-87cb-4a88c76f3956") : kubernetes.io/csi: mounter.SetUpAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.553676    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:19 truenas k3s[5440]: E0122 12:40:19.553893    5440 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2 podName:43e7445c-bec5-40b6-935e-1f0dcfcb02e1 nodeName:}" failed. No retries permitted until 2024-01-22 12:40:20.053847474 +0100 CET m=+28.464485349 (durationBeforeRetry 500ms). Error: UnmountVolume.TearDown failed for volume "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (UniqueName: "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1") : kubernetes.io/csi: Unmounter.TearDownAt failed to get CSI client: driver name zfs.csi.openebs.io not found in the list of registered CSI drivers
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.590652    5440 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: zfs.csi.openebs.io endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock versions: 1.0.0
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.590701    5440 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: zfs.csi.openebs.io at endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.654358    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.654464    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\" (UID: \"7e0f78dc-2724-42c6-97a7-76f79fcb2b9d\") "
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.672746    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432050
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.680823    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de" (OuterVolumeSpecName: "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d"). InnerVolumeSpecName "pvc-977837ff-b237-4192-b75b-2f7c7c77f1de". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.683239    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b" (OuterVolumeSpecName: "pvc-88dedeea-6d2f-448f-bafa-658c55aa954b") pod "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d" (UID: "7e0f78dc-2724-42c6-97a7-76f79fcb2b9d"). InnerVolumeSpecName "pvc-88dedeea-6d2f-448f-bafa-658c55aa954b". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.754904    5440 reconciler_common.go:295] "Volume detached for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:19 truenas k3s[5440]: I0122 12:40:19.754948    5440 reconciler_common.go:295] "Volume detached for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:40:20 truenas k3s[5440]: I0122 12:40:20.260757    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2\") pod \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\" (UID: \"43e7445c-bec5-40b6-935e-1f0dcfcb02e1\") "
Jan 22 12:40:20 truenas k3s[5440]: I0122 12:40:20.271174    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/csi/zfs.csi.openebs.io^pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2" (OuterVolumeSpecName: "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2") pod "43e7445c-bec5-40b6-935e-1f0dcfcb02e1" (UID: "43e7445c-bec5-40b6-935e-1f0dcfcb02e1"). InnerVolumeSpecName "pvc-5d5fe13a-e03d-4ae5-b7c8-875e0c4261c2". PluginName "kubernetes.io/csi", VolumeGidValue ""
Jan 22 12:40:20 truenas k3s[5440]: I0122 12:40:20.415905    5440 server.go:144] "Got registration request from device plugin with resource" resourceName="gpu.intel.com/i915"
Jan 22 12:40:20 truenas k3s[5440]: I0122 12:40:20.673148    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-homarr/homarr-fb47944ff-ts4kt" podStartSLOduration=302.673077638 pod.CreationTimestamp="2024-01-22 12:35:18 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:20.66523767 +0100 CET m=+29.075875555" watchObservedRunningTime="2024-01-22 12:40:20.673077638 +0100 CET m=+29.083715515"
Jan 22 12:40:23 truenas k3s[5440]: I0122 12:40:23.860382    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="cert-manager/cert-manager-8444f6f86b-pcclz" podStartSLOduration=304.860338953 pod.CreationTimestamp="2024-01-22 12:35:19 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:23.860289192 +0100 CET m=+32.270927074" watchObservedRunningTime="2024-01-22 12:40:23.860338953 +0100 CET m=+32.270976836"
Jan 22 12:40:24 truenas k3s[5440]: I0122 12:40:24.264511    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-adguard-home/adguard-home-78766c98d5-fp67l" podStartSLOduration=307.264457513 pod.CreationTimestamp="2024-01-22 12:35:17 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:24.26326808 +0100 CET m=+32.673905964" watchObservedRunningTime="2024-01-22 12:40:24.264457513 +0100 CET m=+32.675095392"
Jan 22 12:40:26 truenas k3s[5440]: I0122 12:40:26.681027    5440 event.go:294] "Event occurred" object="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:26 truenas k3s[5440]: I0122 12:40:26.689073    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:40:27 truenas k3s[5440]: I0122 12:40:27.507475    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:27 truenas k3s[5440]: I0122 12:40:27.507536    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"devshm\" (UniqueName: \"kubernetes.io/empty-dir/a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9-devshm\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.114338    5440 csi_attacher.go:376] kubernetes.io/csi: attacher.MountDevice STAGE_UNSTAGE_VOLUME capability not set. Skipping MountDevice...
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.114397    5440 operation_generator.go:658] "MountVolume.MountDevice succeeded for volume \"pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-977837ff-b237-4192-b75b-2f7c7c77f1de\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") device mount path \"/var/lib/kubelet/plugins/kubernetes.io/csi/zfs.csi.openebs.io/9db8a171bc38a73779bbd462f873ba61f3201c7471771242617ba986a0272077/globalmount\"" pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.515751    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared\" (UniqueName: \"kubernetes.io/empty-dir/a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9-shared\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.515809    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varrun\" (UniqueName: \"kubernetes.io/empty-dir/a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9-varrun\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.515914    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.515949    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9-tmp\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.515974    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlogs\" (UniqueName: \"kubernetes.io/empty-dir/a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9-varlogs\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") " pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.844284    5440 gc_controller.go:207] "Garbage collecting pods" numPods=3
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.844335    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-nextcloud/nextcloud-848cdcb98d-h598x"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.844345    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/coredns-59b4f5bbd5-vnstd"
Jan 22 12:40:28 truenas k3s[5440]: I0122 12:40:28.844369    5440 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-adguard-home/adguard-home-78766c98d5-zmzk9"
Jan 22 12:40:29 truenas k3s[5440]: I0122 12:40:29.123932    5440 csi_attacher.go:376] kubernetes.io/csi: attacher.MountDevice STAGE_UNSTAGE_VOLUME capability not set. Skipping MountDevice...
Jan 22 12:40:29 truenas k3s[5440]: I0122 12:40:29.123970    5440 operation_generator.go:658] "MountVolume.MountDevice succeeded for volume \"pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\" (UniqueName: \"kubernetes.io/csi/zfs.csi.openebs.io^pvc-88dedeea-6d2f-448f-bafa-658c55aa954b\") pod \"nginx-proxy-manager-58666cd54f-47lwx\" (UID: \"a9a6ee08-fcd1-48d8-a01f-e0c88b4718d9\") device mount path \"/var/lib/kubelet/plugins/kubernetes.io/csi/zfs.csi.openebs.io/37c1765910a65106d0e6d30b3c797c7097507fddf1a9f5f1dd7510a73d68dc2d/globalmount\"" pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx"
Jan 22 12:40:29 truenas k3s[5440]: I0122 12:40:29.630702    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=0d1b468d-7739-4ce5-9926-f9ddc02840cd path="/var/lib/kubelet/pods/0d1b468d-7739-4ce5-9926-f9ddc02840cd/volumes"
Jan 22 12:40:29 truenas k3s[5440]: I0122 12:40:29.631743    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=a0678a1d-5503-40ad-814c-1fcdb3a2c2cb path="/var/lib/kubelet/pods/a0678a1d-5503-40ad-814c-1fcdb3a2c2cb/volumes"
Jan 22 12:40:29 truenas k3s[5440]: I0122 12:40:29.633339    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=d3e9e993-7ad4-424f-98a3-c02c58c0b8e8 path="/var/lib/kubelet/pods/d3e9e993-7ad4-424f-98a3-c02c58c0b8e8/volumes"
Jan 22 12:40:29 truenas k3s[5440]: I0122 12:40:29.865478    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="cert-manager/cert-manager-webhook-545bd5d7d8-kf2sp" podStartSLOduration=310.865437375 pod.CreationTimestamp="2024-01-22 12:35:19 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:29.864724422 +0100 CET m=+38.275362305" watchObservedRunningTime="2024-01-22 12:40:29.865437375 +0100 CET m=+38.276075242"
Jan 22 12:40:30 truenas k3s[5440]: I0122 12:40:30.661691    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/svclb-nginx-proxy-manager-websecure-b7c8312a-rjgr9" podStartSLOduration=22.66164334 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:30.660292216 +0100 CET m=+39.070930108" watchObservedRunningTime="2024-01-22 12:40:30.66164334 +0100 CET m=+39.072281225"
Jan 22 12:40:30 truenas k3s[5440]: I0122 12:40:30.670093    5440 event.go:294] "Event occurred" object="ix-nginx-proxy-manager/nginx-proxy-manager-websecure" fieldPath="" kind="Service" apiVersion="v1" type="Normal" reason="UpdatedLoadBalancer" message="Updated LoadBalancer with new IPs: [] -> [192.168.0.100]"
Jan 22 12:40:33 truenas k3s[5440]: I0122 12:40:33.471648    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-nginx-proxy-manager/nginx-proxy-manager-mariadb-6b689fccbb-xw6gd" podStartSLOduration=312.471572009 pod.CreationTimestamp="2024-01-22 12:35:21 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:33.469719128 +0100 CET m=+41.880357064" watchObservedRunningTime="2024-01-22 12:40:33.471572009 +0100 CET m=+41.882209922"
Jan 22 12:40:33 truenas k3s[5440]: I0122 12:40:33.865200    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/csi-smb-controller-7fbbb8fb6f-cwb2s" podStartSLOduration=25.865160041 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:33.862781514 +0100 CET m=+42.273419421" watchObservedRunningTime="2024-01-22 12:40:33.865160041 +0100 CET m=+42.275797916"
Jan 22 12:40:34 truenas k3s[5440]: I0122 12:40:34.268483    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/openebs-zfs-controller-0" podStartSLOduration=26.268438622 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:34.266903818 +0100 CET m=+42.677541705" watchObservedRunningTime="2024-01-22 12:40:34.268438622 +0100 CET m=+42.679076501"
Jan 22 12:40:34 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:40:34.365+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:40:34 truenas k3s[5440]: I0122 12:40:34.660807    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/snapshot-controller-546868dfb4-4tzj7" podStartSLOduration=26.660751827 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:34.660018169 +0100 CET m=+43.070656055" watchObservedRunningTime="2024-01-22 12:40:34.660751827 +0100 CET m=+43.071389708"
Jan 22 12:40:36 truenas k3s[5440]: I0122 12:40:36.065396    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/csi-nfs-controller-7b74694749-kndwf" podStartSLOduration=28.065349347 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:36.065058955 +0100 CET m=+44.475696838" watchObservedRunningTime="2024-01-22 12:40:36.065349347 +0100 CET m=+44.475987235"
Jan 22 12:40:36 truenas k3s[5440]: I0122 12:40:36.664235    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/coredns-59b4f5bbd5-9l2gw" podStartSLOduration=22.664193602 pod.CreationTimestamp="2024-01-22 12:40:14 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:36.663376829 +0100 CET m=+45.074014744" watchObservedRunningTime="2024-01-22 12:40:36.664193602 +0100 CET m=+45.074831484"
Jan 22 12:40:37 truenas k3s[5440]: I0122 12:40:37.062310    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-nextcloud/nextcloud-postgres-774595cffb-qxhcr" podStartSLOduration=21.062268441 pod.CreationTimestamp="2024-01-22 12:40:16 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:37.06211519 +0100 CET m=+45.472753075" watchObservedRunningTime="2024-01-22 12:40:37.062268441 +0100 CET m=+45.472906328"
Jan 22 12:40:37 truenas k3s[5440]: I0122 12:40:37.864017    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/intel-gpu-plugin-56k8h" podStartSLOduration=29.86396259 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:37.862952535 +0100 CET m=+46.273590427" watchObservedRunningTime="2024-01-22 12:40:37.86396259 +0100 CET m=+46.274600480"
Jan 22 12:40:38 truenas k3s[5440]: I0122 12:40:38.264128    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/openebs-zfs-node-ztdfw" podStartSLOduration=30.264076307 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:38.263400921 +0100 CET m=+46.674038799" watchObservedRunningTime="2024-01-22 12:40:38.264076307 +0100 CET m=+46.674714199"
Jan 22 12:40:38 truenas k3s[5440]: I0122 12:40:38.661311    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/snapshot-controller-546868dfb4-t7f9j" podStartSLOduration=30.661267008 pod.CreationTimestamp="2024-01-22 12:40:08 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:38.660488573 +0100 CET m=+47.071126460" watchObservedRunningTime="2024-01-22 12:40:38.661267008 +0100 CET m=+47.071904888"
Jan 22 12:40:46 truenas k3s[5440]: I0122 12:40:46.029367    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-nginx-proxy-manager/nginx-proxy-manager-58666cd54f-47lwx" podStartSLOduration=20.029326156 pod.CreationTimestamp="2024-01-22 12:40:26 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:40:46.029239004 +0100 CET m=+54.439876893" watchObservedRunningTime="2024-01-22 12:40:46.029326156 +0100 CET m=+54.439964054"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.517843    5440 scope.go:115] "RemoveContainer" containerID="6541b7ab17aee051fa72fae76e5ab412f599a8813ba60c2393b16fa01b8fa16a"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.528400    5440 scope.go:115] "RemoveContainer" containerID="ccb6a056d221467aee87e4cc20818e2ef3968cf8bab6ba2c5e4bf97d55bd9b32"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.538057    5440 scope.go:115] "RemoveContainer" containerID="b51191f296dd185bf94e835c55aeea198de55749438100b0c1283955ab89699a"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.550918    5440 scope.go:115] "RemoveContainer" containerID="a8f473f10ad3c0377aad8e3c035d6a66c80236ce99f9d9a2d3bedfcf1c295a6b"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.561515    5440 scope.go:115] "RemoveContainer" containerID="8edcfdf7f8c59c1c7b43dc0d2fc6df839944165ae37d134aa35f3430b9a0bc7d"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.573101    5440 scope.go:115] "RemoveContainer" containerID="61729ee4040392ba9aa3a9c901a7284b9c299387904a9594e45a90a1a3a5fa46"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.585035    5440 scope.go:115] "RemoveContainer" containerID="46a4904ad931582cba65b2aea4d5ed4b685bbc3053aceb7c024437bfe51e680e"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.596845    5440 scope.go:115] "RemoveContainer" containerID="cde4b84c11cd50aa9185f67847c5a42238fb64fadaefa25bc5909999e139e4ad"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.606787    5440 scope.go:115] "RemoveContainer" containerID="d9666954001b5b42b969a9b69c7815984811e05cbc7b4d712d6f84880c1eb199"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.616757    5440 scope.go:115] "RemoveContainer" containerID="edcdb4cfbc3f09ba4124ab4ec4a5baf084ea491fdb242de79b09c587c0a76571"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.627228    5440 scope.go:115] "RemoveContainer" containerID="a7735123a69aba90e891a9c65f7edee9d8dc3b94fd76235261bc13cdeb8a5c4e"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.639557    5440 scope.go:115] "RemoveContainer" containerID="5aa39096038113b62b28771626ac0d09f44080c08e428fc87a28b47e7615bae6"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.649812    5440 scope.go:115] "RemoveContainer" containerID="76b1699aaf9d794b4d9c9b23b77fc09d4eb5329e5efcd7249e2f7596cf4e0a79"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.659888    5440 scope.go:115] "RemoveContainer" containerID="01c883d949d1ba6d94ed67b34645451dc69f50416eee44a5c35b9497996c5609"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.673023    5440 scope.go:115] "RemoveContainer" containerID="568c22d63a775bd94078a83cb2a34c39a8a28855908c9b70e3aa5def2236f8c3"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.682816    5440 scope.go:115] "RemoveContainer" containerID="5c60a099fdf0e979cd4e501f7d7976a8d38771af11cd6a6c5d306e195c159178"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.694045    5440 scope.go:115] "RemoveContainer" containerID="b37c0256e1d3b4147b2f15d20f2107d4d3825969d01a2bc753a103c885cf678f"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.704463    5440 scope.go:115] "RemoveContainer" containerID="1ea4edecd5a73b25c4ee571d93df081eb17982db36520f5a40527f3e05161070"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.714774    5440 scope.go:115] "RemoveContainer" containerID="22ae6787035f4f3bf9dd10bc03a58a51ea4a8dbc5892b5249b132ed5c915b872"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.724711    5440 scope.go:115] "RemoveContainer" containerID="b4b20912fcb8833b0ec31366bf26517b46d877bf395cb4cc8a832fb9dce1f8f7"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.734606    5440 scope.go:115] "RemoveContainer" containerID="2ab4dbffd0b0fe40b03ba7aafd41cf66f5d4e485bd369a55d314e0f55b0b18e8"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.744162    5440 scope.go:115] "RemoveContainer" containerID="102f96667ff5eafe9a616e29a956f9175d87c5e53aa52544d1385d2d05c85d90"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.753870    5440 scope.go:115] "RemoveContainer" containerID="0c535afd469d85535c85902c48634f3d16c08a874cc0553bd96fefbcd5568888"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.764230    5440 scope.go:115] "RemoveContainer" containerID="864e22d9803950c64d48c90d49727d014e3cf8fde2b5acc9414f2d5367947c96"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.773800    5440 scope.go:115] "RemoveContainer" containerID="7c745281b0298b8ffeb26d05a2b6f30d7658a7fb11be5b5ac043c410da9718ef"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.786888    5440 scope.go:115] "RemoveContainer" containerID="8c9b0589dff2873754553a4a1172e9048dbe29b418542da9b7bfe74975f7b234"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.797435    5440 scope.go:115] "RemoveContainer" containerID="fda7f1604cfd7ca48ab5efd3b664a78037e0244eb58a56216533036017ab8ebb"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.809097    5440 scope.go:115] "RemoveContainer" containerID="e3bcff389b987c60d85b71d82b6cbe38d05d74eb5cca759278339f132f222d9d"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.820544    5440 scope.go:115] "RemoveContainer" containerID="981300e1865ea48bbf6cb060c55a8ebf564f1ef5d95e354c8e38910522f11702"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.830163    5440 scope.go:115] "RemoveContainer" containerID="4cf3044e5ceddc75ce1bdde95c0f59cce8825d43512d684d3989c8c7d5f1a468"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.841042    5440 scope.go:115] "RemoveContainer" containerID="f6d68cb0cfe20fd4194a3a79a254f9224d6ce926cf13ddeca631a81476ff1b32"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.853150    5440 scope.go:115] "RemoveContainer" containerID="9e9c6f2e7510111f23f5184259238494afa3e34fb93ec3b1218dc79500ab1958"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.864253    5440 scope.go:115] "RemoveContainer" containerID="1ddc4190e35a5411f2dd1f5b375d52d6f012893b632e9fcddbc83ca982a9e8c2"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.875980    5440 scope.go:115] "RemoveContainer" containerID="3d5b5160c0aad55683a7bb1355c9f9894d783011d8c52fb06cb5ab82b51781bc"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.889502    5440 scope.go:115] "RemoveContainer" containerID="6617ddf7ff5f01f802bceae8e00f1c251f5bcce1b84e8840600d29fb8f351359"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.899655    5440 scope.go:115] "RemoveContainer" containerID="a639400dfc5ca2dfaa6fb4e6a8137dc7fa08c353806a9b57ba834cdc9564b3c7"
Jan 22 12:40:59 truenas k3s[5440]: I0122 12:40:59.911040    5440 scope.go:115] "RemoveContainer" containerID="fb620d9f352a3577206d1118119deb683eeb21e67009e4082a3f74c723054c7f"
Jan 22 12:41:10 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:41:10.904+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:41:14 truenas k3s[5440]: I0122 12:41:14.124096    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-nextcloud/nextcloud-848cdcb98d-dmtrz" podStartSLOduration=356.124055294 pod.CreationTimestamp="2024-01-22 12:35:18 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:41:14.123597783 +0100 CET m=+82.534235674" watchObservedRunningTime="2024-01-22 12:41:14.124055294 +0100 CET m=+82.534693171"
Jan 22 12:41:44 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:41:44.239+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:42:19 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:42:19.275+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:42:59 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:42:59.872+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:43:39 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:43:39.248+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:44:12 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:44:12.008+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:44:47 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:44:47.991+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:44:52 truenas k3s[5440]: time="2024-01-22T12:44:52+01:00" level=info msg="COMPACT compactRev=23339159 targetCompactRev=23340159 currentRev=23341605"
Jan 22 12:44:52 truenas k3s[5440]: time="2024-01-22T12:44:52+01:00" level=info msg="COMPACT deleted 1000 rows from 1000 revisions in 31.314459ms - compacted to 23340159/23341605"
Jan 22 12:44:52 truenas k3s[5440]: time="2024-01-22T12:44:52+01:00" level=info msg="COMPACT compactRev=23340159 targetCompactRev=23340487 currentRev=23341605"
Jan 22 12:44:52 truenas k3s[5440]: time="2024-01-22T12:44:52+01:00" level=info msg="COMPACT deleted 252 rows from 328 revisions in 15.510119ms - compacted to 23340487/23341605"
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.118458    5440 controller.go:615] quota admission added evaluator for: jobs.batch
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.125152    5440 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created job nextcloud-cronjob-28432065"
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.125475    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.149887    5440 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-28432065" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-cronjob-28432065-kb2ps"
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.150461    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.158469    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.160109    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.162488    5440 topology_manager.go:210] "Topology Admit Handler"
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.180422    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.263818    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-rwfjs\" (UniqueName: \"kubernetes.io/projected/4ddc974f-3c8f-41c2-8c7a-a26c55057870-kube-api-access-rwfjs\") pod \"nextcloud-cronjob-28432065-kb2ps\" (UID: \"4ddc974f-3c8f-41c2-8c7a-a26c55057870\") " pod="ix-nextcloud/nextcloud-cronjob-28432065-kb2ps"
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.263942    5440 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"nextcloud-data\" (UniqueName: \"kubernetes.io/host-path/4ddc974f-3c8f-41c2-8c7a-a26c55057870-nextcloud-data\") pod \"nextcloud-cronjob-28432065-kb2ps\" (UID: \"4ddc974f-3c8f-41c2-8c7a-a26c55057870\") " pod="ix-nextcloud/nextcloud-cronjob-28432065-kb2ps"
Jan 22 12:45:00 truenas k3s[5440]: I0122 12:45:00.604558    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:02 truenas k3s[5440]: I0122 12:45:02.813657    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:02 truenas k3s[5440]: I0122 12:45:02.815391    5440 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-nextcloud/nextcloud-cronjob-28432065-kb2ps" podStartSLOduration=2.815305957 pod.CreationTimestamp="2024-01-22 12:45:00 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-01-22 12:45:02.813099113 +0100 CET m=+311.223737018" watchObservedRunningTime="2024-01-22 12:45:02.815305957 +0100 CET m=+311.225943853"
Jan 22 12:45:03 truenas k3s[5440]: I0122 12:45:03.822358    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:18 truenas k3s[5440]: I0122 12:45:18.856839    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:19 truenas k3s[5440]: I0122 12:45:19.861796    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:24 truenas k3s[5440]: I0122 12:45:24.736466    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-rwfjs\" (UniqueName: \"kubernetes.io/projected/4ddc974f-3c8f-41c2-8c7a-a26c55057870-kube-api-access-rwfjs\") pod \"4ddc974f-3c8f-41c2-8c7a-a26c55057870\" (UID: \"4ddc974f-3c8f-41c2-8c7a-a26c55057870\") "
Jan 22 12:45:24 truenas k3s[5440]: I0122 12:45:24.736518    5440 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"nextcloud-data\" (UniqueName: \"kubernetes.io/host-path/4ddc974f-3c8f-41c2-8c7a-a26c55057870-nextcloud-data\") pod \"4ddc974f-3c8f-41c2-8c7a-a26c55057870\" (UID: \"4ddc974f-3c8f-41c2-8c7a-a26c55057870\") "
Jan 22 12:45:24 truenas k3s[5440]: I0122 12:45:24.740262    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/4ddc974f-3c8f-41c2-8c7a-a26c55057870-kube-api-access-rwfjs" (OuterVolumeSpecName: "kube-api-access-rwfjs") pod "4ddc974f-3c8f-41c2-8c7a-a26c55057870" (UID: "4ddc974f-3c8f-41c2-8c7a-a26c55057870"). InnerVolumeSpecName "kube-api-access-rwfjs". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jan 22 12:45:24 truenas k3s[5440]: W0122 12:45:24.740470    5440 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/4ddc974f-3c8f-41c2-8c7a-a26c55057870/volume-subpaths/nextcloud-data/nextcloud-cronjob/0" is not a mountpoint, deleting
Jan 22 12:45:24 truenas k3s[5440]: W0122 12:45:24.743409    5440 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/4ddc974f-3c8f-41c2-8c7a-a26c55057870/volume-subpaths/nextcloud-data/nextcloud-cronjob/1" is not a mountpoint, deleting
Jan 22 12:45:24 truenas k3s[5440]: W0122 12:45:24.745781    5440 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/4ddc974f-3c8f-41c2-8c7a-a26c55057870/volume-subpaths/nextcloud-data/nextcloud-cronjob/2" is not a mountpoint, deleting
Jan 22 12:45:24 truenas k3s[5440]: W0122 12:45:24.747685    5440 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/4ddc974f-3c8f-41c2-8c7a-a26c55057870/volume-subpaths/nextcloud-data/nextcloud-cronjob/3" is not a mountpoint, deleting
Jan 22 12:45:24 truenas k3s[5440]: W0122 12:45:24.749170    5440 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/4ddc974f-3c8f-41c2-8c7a-a26c55057870/volume-subpaths/nextcloud-data/nextcloud-cronjob/4" is not a mountpoint, deleting
Jan 22 12:45:24 truenas k3s[5440]: W0122 12:45:24.751527    5440 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/4ddc974f-3c8f-41c2-8c7a-a26c55057870/volume-subpaths/nextcloud-data/nextcloud-cronjob/5" is not a mountpoint, deleting
Jan 22 12:45:24 truenas k3s[5440]: W0122 12:45:24.753178    5440 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/4ddc974f-3c8f-41c2-8c7a-a26c55057870/volume-subpaths/nextcloud-data/nextcloud-cronjob/6" is not a mountpoint, deleting
Jan 22 12:45:24 truenas k3s[5440]: I0122 12:45:24.753655    5440 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/4ddc974f-3c8f-41c2-8c7a-a26c55057870-nextcloud-data" (OuterVolumeSpecName: "nextcloud-data") pod "4ddc974f-3c8f-41c2-8c7a-a26c55057870" (UID: "4ddc974f-3c8f-41c2-8c7a-a26c55057870"). InnerVolumeSpecName "nextcloud-data". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jan 22 12:45:24 truenas k3s[5440]: I0122 12:45:24.837009    5440 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-rwfjs\" (UniqueName: \"kubernetes.io/projected/4ddc974f-3c8f-41c2-8c7a-a26c55057870-kube-api-access-rwfjs\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:45:24 truenas k3s[5440]: I0122 12:45:24.837059    5440 reconciler_common.go:295] "Volume detached for volume \"nextcloud-data\" (UniqueName: \"kubernetes.io/host-path/4ddc974f-3c8f-41c2-8c7a-a26c55057870-nextcloud-data\") on node \"ix-truenas\" DevicePath \"\""
Jan 22 12:45:24 truenas k3s[5440]: I0122 12:45:24.868276    5440 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="c17d5294bfe1a26093de6740205581252ce6a3b03830b9a8455da61b28fd3607"
Jan 22 12:45:24 truenas k3s[5440]: I0122 12:45:24.892410    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.898931    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.907833    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.914061    5440 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob-28432065" fieldPath="" kind="Job" apiVersion="batch/v1" type="Normal" reason="Completed" message="Job completed"
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.914092    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.914303    5440 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-28432065, status: Complete"
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.920266    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432065
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.921937    5440 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SuccessfulDelete" message="Deleted job nextcloud-cronjob-28432050"
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.922440    5440 job_controller.go:514] enqueueing job ix-nextcloud/nextcloud-cronjob-28432050
Jan 22 12:45:25 truenas k3s[5440]: I0122 12:45:25.928319    5440 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cronjob" fieldPath="" kind="CronJob" apiVersion="batch/v1" type="Normal" reason="SawCompletedJob" message="Saw completed job: nextcloud-cronjob-28432065, status: Complete"
Jan 22 12:45:25 truenas k3s[5440]: E0122 12:45:25.934552    5440 cronjob_controllerv2.go:166] error syncing CronJobController ix-nextcloud/nextcloud-cronjob, requeuing: Operation cannot be fulfilled on cronjobs.batch "nextcloud-cronjob": the object has been modified; please apply your changes to the latest version and try again
Jan 22 12:45:26 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:45:26.385+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:45:27 truenas k3s[5440]: I0122 12:45:27.634193    5440 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=b328df6e-3da7-4d11-947d-86d09a12d2cc path="/var/lib/kubelet/pods/b328df6e-3da7-4d11-947d-86d09a12d2cc/volumes"
Jan 22 12:45:58 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:45:58.014+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:46:36 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:46:36.509+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:47:07 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:47:07.934+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:47:48 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:47:48.646+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:48:24 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:48:24.234+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:49:07 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:49:07.635+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:49:47 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:49:47.243+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:49:52 truenas k3s[5440]: time="2024-01-22T12:49:52+01:00" level=info msg="COMPACT compactRev=23340487 targetCompactRev=23341191 currentRev=23342191"
Jan 22 12:49:52 truenas k3s[5440]: time="2024-01-22T12:49:52+01:00" level=info msg="COMPACT deleted 446 rows from 704 revisions in 32.507522ms - compacted to 23341191/23342191"
Jan 22 12:49:52 truenas k3s[5440]: time="2024-01-22T12:49:52+01:00" level=info msg="COMPACT revision 23341191 has already been compacted"
Jan 22 12:50:25 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:50:25.934+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:51:03 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:51:03.192+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:51:38 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:51:38.075+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:52:14 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:52:14.454+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:52:52 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:52:52.418+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:53:30 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:53:30.010+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:54:02 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:54:02.978+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:54:39 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:54:39.063+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:54:52 truenas k3s[5440]: time="2024-01-22T12:54:52+01:00" level=info msg="COMPACT compactRev=23341191 targetCompactRev=23341745 currentRev=23342745"
Jan 22 12:54:52 truenas k3s[5440]: time="2024-01-22T12:54:52+01:00" level=info msg="COMPACT deleted 543 rows from 554 revisions in 23.179018ms - compacted to 23341745/23342745"
Jan 22 12:55:13 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:55:13.238+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 22 12:55:51 truenas k3s[5440]: {"level":"warn","ts":"2024-01-22T12:55:51.212+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a55a40/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
