root@truenas[/var/log]# tail -500 k3s_daemon.log Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Kube API server is now running" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="k3s is up and running" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Applying CRD addons.k3s.cattle.io" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Applying CRD helmcharts.helm.cattle.io" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Applying CRD helmchartconfigs.helm.cattle.io" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Writing static file: /mnt/SmallPool/ix-applications/k3s/server/static/charts/traefik-crd-10.14.100.tgz" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Writing static file: /mnt/SmallPool/ix-applications/k3s/server/static/charts/traefik-10.14.100.tgz" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Writing manifest: /mnt/SmallPool/ix-applications/k3s/server/manifests/rolebindings.yaml" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Writing manifest: /mnt/SmallPool/ix-applications/k3s/server/manifests/coredns.yaml" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Writing manifest: /mnt/SmallPool/ix-applications/k3s/server/manifests/multus-daemonset.yaml" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Writing manifest: /mnt/SmallPool/ix-applications/k3s/server/manifests/zfs-operator.yaml" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Writing manifest: /mnt/SmallPool/ix-applications/k3s/server/manifests/kuberouter.yaml" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Starting k3s.cattle.io/v1, Kind=Addon controller" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Creating deploy event broadcaster" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"coredns\", UID:\"8b4c3498-a218-4ffc-940f-fa663c503e8f\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"271\", FieldPath:\"\"}): type: 'Normal' reason: 'ApplyingManifest' Applying manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/coredns.yaml\"" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Creating svccontroller event broadcaster" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Cluster dns configmap already exists" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Labels and annotations have been set successfully on node: ix-truenas" Nov 2 17:44:01 truenas k3s[7306]: I1102 17:44:01.755592 7306 controller.go:611] quota admission added evaluator for: deployments.apps Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"coredns\", UID:\"8b4c3498-a218-4ffc-940f-fa663c503e8f\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"271\", FieldPath:\"\"}): type: 'Normal' reason: 'AppliedManifest' Applied manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/coredns.yaml\"" Nov 2 17:44:01 truenas k3s[7306]: I1102 17:44:01.785779 7306 controller.go:611] quota admission added evaluator for: addons.k3s.cattle.io Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"kuberouter\", UID:\"65588532-a38b-4af9-9f68-e90c6e0e1a2b\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"280\", FieldPath:\"\"}): type: 'Normal' reason: 'ApplyingManifest' Applying manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/kuberouter.yaml\"" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"kuberouter\", UID:\"65588532-a38b-4af9-9f68-e90c6e0e1a2b\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"280\", FieldPath:\"\"}): type: 'Normal' reason: 'AppliedManifest' Applied manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/kuberouter.yaml\"" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"multus-daemonset\", UID:\"a38d0deb-518d-471c-a3a5-c5befc6bfa33\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"291\", FieldPath:\"\"}): type: 'Normal' reason: 'ApplyingManifest' Applying manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/multus-daemonset.yaml\"" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChart controller" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChartConfig controller" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"multus-daemonset\", UID:\"a38d0deb-518d-471c-a3a5-c5befc6bfa33\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"291\", FieldPath:\"\"}): type: 'Normal' reason: 'AppliedManifest' Applied manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/multus-daemonset.yaml\"" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"rolebindings\", UID:\"7120f591-2b7e-454a-b8d3-0cf06c6c510f\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"300\", FieldPath:\"\"}): type: 'Normal' reason: 'ApplyingManifest' Applying manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/rolebindings.yaml\"" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Starting apps/v1, Kind=DaemonSet controller" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Starting apps/v1, Kind=Deployment controller" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"rolebindings\", UID:\"7120f591-2b7e-454a-b8d3-0cf06c6c510f\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"300\", FieldPath:\"\"}): type: 'Normal' reason: 'AppliedManifest' Applied manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/rolebindings.yaml\"" Nov 2 17:44:01 truenas k3s[7306]: time="2022-11-02T17:44:01Z" level=info msg="Starting rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"zfs-operator\", UID:\"3bfb6bd3-dcd0-431d-92f8-9d5b9fe390b8\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"354\", FieldPath:\"\"}): type: 'Normal' reason: 'ApplyingManifest' Applying manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/zfs-operator.yaml\"" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Starting batch/v1, Kind=Job controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Starting /v1, Kind=ServiceAccount controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Starting /v1, Kind=Pod controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Starting /v1, Kind=Service controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Starting /v1, Kind=Endpoints controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Starting /v1, Kind=Node controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Starting /v1, Kind=ConfigMap controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Starting /v1, Kind=Secret controller" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Connecting to proxy" url="wss://127.0.0.1:6443/v1-k3s/connect" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Updating TLS secret for k3s-serving (count: 11): map[listener.cattle.io/cn-0.0.0.0:0.0.0.0 listener.cattle.io/cn-127.0.0.1:127.0.0.1 listener.cattle.io/cn-172.17.0.1:172.17.0.1 listener.cattle.io/cn-192.168.0.237:192.168.0.237 listener.cattle.io/cn-__1-f16284:::1 listener.cattle.io/cn-ix-truenas:ix-truenas listener.cattle.io/cn-kubernetes:kubernetes listener.cattle.io/cn-kubernetes.default:kubernetes.default listener.cattle.io/cn-kubernetes.default.svc:kubernetes.default.svc listener.cattle.io/cn-kubernetes.default.svc.cluster.local:kubernetes.default.svc.cluster.local listener.cattle.io/cn-localhost:localhost listener.cattle.io/fingerprint:SHA1=AA82F2D80C14E9D649DE637CA470BE4D62301A90]" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Event(v1.ObjectReference{Kind:\"Addon\", Namespace:\"kube-system\", Name:\"zfs-operator\", UID:\"3bfb6bd3-dcd0-431d-92f8-9d5b9fe390b8\", APIVersion:\"k3s.cattle.io/v1\", ResourceVersion:\"354\", FieldPath:\"\"}): type: 'Normal' reason: 'AppliedManifest' Applied manifest at \"/mnt/SmallPool/ix-applications/k3s/server/manifests/zfs-operator.yaml\"" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.381306 7306 serving.go:355] Generated self-signed cert in-memory Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Handling backend connection request [ix-truenas]" Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="Running kubelet --address=0.0.0.0 --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --client-ca-file=/mnt/SmallPool/ix-applications/k3s/agent/client-ca.crt --cluster-dns=172.17.0.10 --cluster-domain=cluster.local --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ix-truenas --kubeconfig=/mnt/SmallPool/ix-applications/k3s/agent/kubelet.kubeconfig --max-pods=250 --network-plugin=cni --node-ip=0.0.0.0 --node-labels= --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/mnt/SmallPool/ix-applications/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/etc/resolv.conf --tls-cert-file=/mnt/SmallPool/ix-applications/k3s/agent/serving-kubelet.crt --tls-private-key-file=/mnt/SmallPool/ix-applications/k3s/agent/serving-kubelet.key" Nov 2 17:44:02 truenas k3s[7306]: Flag --network-plugin has been deprecated, will be removed along with dockershim. Nov 2 17:44:02 truenas k3s[7306]: time="2022-11-02T17:44:02Z" level=info msg="labels have already set on node: ix-truenas" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.625362 7306 serving.go:355] Generated self-signed cert in-memory Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.653734 7306 server.go:442] "Kubelet version" kubeletVersion="v1.23.5+k3s-fbfa51e5-dirty" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.659563 7306 dynamic_cafile_content.go:156] "Starting controller" name="client-ca-bundle::/mnt/SmallPool/ix-applications/k3s/agent/client-ca.crt" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.854482 7306 server.go:693] "--cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.856469 7306 container_manager_linux.go:281] "Container manager verified user specified cgroup-root exists" cgroupRoot=[] Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.856727 7306 container_manager_linux.go:286] "Creating Container Manager object based on Node Config" nodeConfig={RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:true NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:nodefs.available Operator:LessThan Value:{Quantity: Percentage:0.05} GracePeriod:0s MinReclaim:} {Signal:imagefs.available Operator:LessThan Value:{Quantity: Percentage:0.05} GracePeriod:0s MinReclaim:}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUManagerPolicyOptions:map[] ExperimentalTopologyManagerScope:container ExperimentalCPUManagerReconcilePeriod:10s ExperimentalMemoryManagerPolicy:None ExperimentalMemoryManagerReservedMemory:[] ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none} Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.858355 7306 topology_manager.go:133] "Creating topology manager with policy per scope" topologyPolicyName="none" topologyScopeName="container" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.858519 7306 container_manager_linux.go:321] "Creating device plugin manager" devicePluginEnabled=true Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.866813 7306 state_mem.go:36] "Initialized new in-memory state store" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.868161 7306 kubelet.go:313] "Using dockershim is deprecated, please consider using a full-fledged CRI implementation" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.874543 7306 client.go:80] "Connecting to docker on the dockerEndpoint" endpoint="unix:///var/run/docker.sock" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.875066 7306 client.go:99] "Start docker client with request timeout" timeout="2m0s" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.905300 7306 docker_service.go:571] "Hairpin mode is set but kubenet is not enabled, falling back to HairpinVeth" hairpinMode=promiscuous-bridge Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.905836 7306 docker_service.go:243] "Hairpin mode is set" hairpinMode=hairpin-veth Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.906562 7306 cni.go:240] "Unable to update cni config" err="no networks found in /etc/cni/net.d" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.938334 7306 cni.go:240] "Unable to update cni config" err="no networks found in /etc/cni/net.d" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.938998 7306 docker_service.go:258] "Docker cri networking managed by the network plugin" networkPluginName="cni" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.939572 7306 cni.go:240] "Unable to update cni config" err="no networks found in /etc/cni/net.d" Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.960781 7306 docker_service.go:264] "Docker Info" dockerInfo=&{ID:UKA6:O2OD:7F3K:JB56:7VBC:7UNR:HM35:5YFD:HG4L:DD5D:LY5I:HO6T Containers:22 ContainersRunning:0 ContainersPaused:0 ContainersStopped:22 Images:9 Driver:zfs DriverStatus:[[Zpool SmallPool] [Zpool Health ONLINE] [Parent Dataset SmallPool/ix-applications/docker] [Space Used By Parent 297463808] [Space Available 591528284160] [Parent Quota no] [Compression lz4]] SystemStatus:[] Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:[] Log:[awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:true KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6tables:true Debug:false NFd:23 OomKillDisable:true NGoroutines:41 SystemTime:2022-11-02T17:44:02.94081183Z LoggingDriver:json-file CgroupDriver:cgroupfs CgroupVersion:1 NEventsListener:0 KernelVersion:5.10.142+truenas OperatingSystem:Debian GNU/Linux 11 (bullseye) OSVersion:11 OSType:linux Architecture:x86_64 IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:0xc000251960 NCPU:8 MemTotal:16732246016 GenericResources:[] DockerRootDir:/mnt/SmallPool/ix-applications/docker HTTPProxy: HTTPSProxy: NoProxy: Name:truenas Labels:[] ExperimentalBuild:false ServerVersion:dev ClusterStore: ClusterAdvertise: Runtimes:map[io.containerd.runc.v2:{Path:runc Args:[] Shim:} io.containerd.runtime.v1.linux:{Path:runc Args:[] Shim:} runc:{Path:runc Args:[] Shim:}] DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:[] Nodes:0 Managers:0 Cluster: Warnings:[]} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:7b11cfaabd73bb80907dd23182b9347b4245eb5d Expected:7b11cfaabd73bb80907dd23182b9347b4245eb5d} RuncCommit:{ID:v1.0.2-0-g52b36a2 Expected:v1.0.2-0-g52b36a2} InitCommit:{ID:de40ad0 Expected:de40ad0} SecurityOptions:[name=apparmor] ProductLicense: DefaultAddressPools:[] Warnings:[]} Nov 2 17:44:02 truenas k3s[7306]: I1102 17:44:02.960817 7306 docker_service.go:279] "Setting cgroupDriver" cgroupDriver="cgroupfs" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.039641 7306 kubelet.go:416] "Attempting to sync node with API server" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.039720 7306 kubelet.go:278] "Adding static pod path" path="/mnt/SmallPool/ix-applications/k3s/agent/pod-manifests" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.040338 7306 kubelet.go:289] "Adding apiserver pod source" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.049280 7306 apiserver.go:42] "Waiting for node sync before watching apiserver pods" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.056414 7306 apiserver.go:52] "Watching apiserver" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.098195 7306 kuberuntime_manager.go:249] "Container runtime initialized" containerRuntime="docker" version="dev" apiVersion="1.41.0" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.125641 7306 server.go:1231] "Started kubelet" Nov 2 17:44:03 truenas k3s[7306]: E1102 17:44:03.126357 7306 kubelet.go:1351] "Image garbage collection failed once. Stats initialization may not have completed yet" err="failed to get imageFs info: unable to find data in memory cache" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.127242 7306 server.go:150] "Starting to listen" address="0.0.0.0" port=10250 Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.134592 7306 server.go:410] "Adding debug handlers to kubelet server" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.143892 7306 scope.go:110] "RemoveContainer" containerID="c096730de4b00bbb2632072b8eb458506f30f8bd3afbd1f77bf2aacb51306da4" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.154749 7306 fs_resource_analyzer.go:67] "Starting FS ResourceAnalyzer" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.159242 7306 volume_manager.go:291] "Starting Kubelet Volume Manager" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.160048 7306 desired_state_of_world_populator.go:147] "Desired state populator starts to run" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.182457 7306 controller.go:611] quota admission added evaluator for: leases.coordination.k8s.io Nov 2 17:44:03 truenas k3s[7306]: E1102 17:44:03.198150 7306 kubelet.go:2347] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.267155 7306 kuberuntime_manager.go:1105] "Updating runtime config through cri with podcidr" CIDR="172.16.0.0/16" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.271474 7306 docker_service.go:364] "Docker cri received runtime config" runtimeConfig="&RuntimeConfig{NetworkConfig:&NetworkConfig{PodCidr:172.16.0.0/16,},}" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.271842 7306 kubelet_network.go:76] "Updating Pod CIDR" originalPodCIDR="" newPodCIDR="172.16.0.0/16" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.321043 7306 kubelet_node_status.go:70] "Attempting to register node" node="ix-truenas" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.336489 7306 kubelet_node_status.go:108] "Node was previously registered" node="ix-truenas" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.336963 7306 kubelet_node_status.go:73] "Successfully registered node" node="ix-truenas" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.372117 7306 setters.go:578] "Node became not ready" node="ix-truenas" condition={Type:Ready Status:False LastHeartbeatTime:2022-11-02 17:44:03.372027573 +0000 GMT m=+35.000309116 LastTransitionTime:2022-11-02 17:44:03.372027573 +0000 GMT m=+35.000309116 Reason:KubeletNotReady Message:[container runtime status check may not have completed yet, PLEG is not healthy: pleg has yet to be successful, container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized]} Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.529542 7306 cpu_manager.go:213] "Starting CPU manager" policy="none" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.531514 7306 cpu_manager.go:214] "Reconciling" reconcilePeriod="10s" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.531704 7306 state_mem.go:36] "Initialized new in-memory state store" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.556066 7306 state_mem.go:88] "Updated default CPUSet" cpuSet="" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.556302 7306 state_mem.go:96] "Updated CPUSet assignments" assignments=map[] Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.556439 7306 policy_none.go:49] "None policy: Start" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.568139 7306 kubelet_network_linux.go:57] "Initialized protocol iptables rules." protocol=IPv4 Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.569173 7306 memory_manager.go:168] "Starting memorymanager" policy="None" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.569843 7306 state_mem.go:35] "Initializing new in-memory state store" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.603388 7306 state_mem.go:75] "Updated machine memory state" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.628079 7306 controllermanager.go:196] Version: v1.23.5+k3s-fbfa51e5-dirty Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.634987 7306 secure_serving.go:200] Serving securely on 127.0.0.1:10257 Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.635244 7306 tlsconfig.go:240] "Starting DynamicServingCertificateController" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.636024 7306 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.636179 7306 shared_informer.go:240] Waiting for caches to sync for RequestHeaderAuthRequestController Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.636548 7306 configmap_cafile_content.go:201] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.636691 7306 shared_informer.go:240] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.636965 7306 configmap_cafile_content.go:201] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.636990 7306 shared_informer.go:240] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.666538 7306 manager.go:610] "Failed to read data from checkpoint" checkpoint="kubelet_internal_checkpoint" err="checkpoint is not found" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.681677 7306 plugin_manager.go:114] "Starting Kubelet Plugin Manager" Nov 2 17:44:03 truenas k3s[7306]: E1102 17:44:03.703318 7306 kubelet.go:2347] "Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.801449 7306 kubelet_network_linux.go:57] "Initialized protocol iptables rules." protocol=IPv6 Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.802209 7306 status_manager.go:159] "Starting to sync pod status with apiserver" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.802494 7306 kubelet.go:1977] "Starting kubelet main sync loop" Nov 2 17:44:03 truenas k3s[7306]: E1102 17:44:03.802856 7306 kubelet.go:2001] "Skipping pod synchronization" err="PLEG is not healthy: pleg has yet to be successful" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.820053 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"e9828288cd6b67602c86e04d805accd4ede1172cb67cc014f232632d0cdbd97d\"" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.822215 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"6c7af75a15477e5a98670274d8dde604322540d7b921993301a5f3226b9f1c44\"" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.836467 7306 shared_informer.go:247] Caches are synced for RequestHeaderAuthRequestController Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.837381 7306 shared_informer.go:247] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.837532 7306 shared_informer.go:247] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.903420 7306 topology_manager.go:200] "Topology Admit Handler" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.905122 7306 topology_manager.go:200] "Topology Admit Handler" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.905348 7306 topology_manager.go:200] "Topology Admit Handler" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.992759 7306 server.go:139] "Starting Kubernetes Scheduler" version="v1.23.5+k3s-fbfa51e5-dirty" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.998116 7306 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.998142 7306 shared_informer.go:240] Waiting for caches to sync for RequestHeaderAuthRequestController Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.998185 7306 configmap_cafile_content.go:201] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.998198 7306 shared_informer.go:240] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.998469 7306 secure_serving.go:200] Serving securely on 127.0.0.1:10259 Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.998127 7306 configmap_cafile_content.go:201] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.998765 7306 shared_informer.go:240] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.998960 7306 tlsconfig.go:240] "Starting DynamicServingCertificateController" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999053 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/3c079ed1-e6fb-4ae1-92f5-ff24371acbd2-pods-mount-dir\") pod \"openebs-zfs-node-s9qqv\" (UID: \"3c079ed1-e6fb-4ae1-92f5-ff24371acbd2\") " pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999126 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-jbcfq\" (UniqueName: \"kubernetes.io/projected/3c079ed1-e6fb-4ae1-92f5-ff24371acbd2-kube-api-access-jbcfq\") pod \"openebs-zfs-node-s9qqv\" (UID: \"3c079ed1-e6fb-4ae1-92f5-ff24371acbd2\") " pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999188 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/eb786c1d-1288-40a1-9e7c-1d8da9188f4d-config-volume\") pod \"coredns-d76bd69b-rqdx5\" (UID: \"eb786c1d-1288-40a1-9e7c-1d8da9188f4d\") " pod="kube-system/coredns-d76bd69b-rqdx5" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999247 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"custom-config-volume\" (UniqueName: \"kubernetes.io/configmap/eb786c1d-1288-40a1-9e7c-1d8da9188f4d-custom-config-volume\") pod \"coredns-d76bd69b-rqdx5\" (UID: \"eb786c1d-1288-40a1-9e7c-1d8da9188f4d\") " pod="kube-system/coredns-d76bd69b-rqdx5" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999308 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/3c079ed1-e6fb-4ae1-92f5-ff24371acbd2-encr-keys\") pod \"openebs-zfs-node-s9qqv\" (UID: \"3c079ed1-e6fb-4ae1-92f5-ff24371acbd2\") " pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999363 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/3c079ed1-e6fb-4ae1-92f5-ff24371acbd2-registration-dir\") pod \"openebs-zfs-node-s9qqv\" (UID: \"3c079ed1-e6fb-4ae1-92f5-ff24371acbd2\") " pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999414 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/3c079ed1-e6fb-4ae1-92f5-ff24371acbd2-host-root\") pod \"openebs-zfs-node-s9qqv\" (UID: \"3c079ed1-e6fb-4ae1-92f5-ff24371acbd2\") " pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999465 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/3c079ed1-e6fb-4ae1-92f5-ff24371acbd2-plugin-dir\") pod \"openebs-zfs-node-s9qqv\" (UID: \"3c079ed1-e6fb-4ae1-92f5-ff24371acbd2\") " pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999521 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/fde5b1cd-da44-4a8e-9375-5fc09fc7972f-socket-dir\") pod \"openebs-zfs-controller-0\" (UID: \"fde5b1cd-da44-4a8e-9375-5fc09fc7972f\") " pod="kube-system/openebs-zfs-controller-0" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999575 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-2znzq\" (UniqueName: \"kubernetes.io/projected/fde5b1cd-da44-4a8e-9375-5fc09fc7972f-kube-api-access-2znzq\") pod \"openebs-zfs-controller-0\" (UID: \"fde5b1cd-da44-4a8e-9375-5fc09fc7972f\") " pod="kube-system/openebs-zfs-controller-0" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999626 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-ttr9r\" (UniqueName: \"kubernetes.io/projected/eb786c1d-1288-40a1-9e7c-1d8da9188f4d-kube-api-access-ttr9r\") pod \"coredns-d76bd69b-rqdx5\" (UID: \"eb786c1d-1288-40a1-9e7c-1d8da9188f4d\") " pod="kube-system/coredns-d76bd69b-rqdx5" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999679 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/3c079ed1-e6fb-4ae1-92f5-ff24371acbd2-device-dir\") pod \"openebs-zfs-node-s9qqv\" (UID: \"3c079ed1-e6fb-4ae1-92f5-ff24371acbd2\") " pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999734 7306 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/3c079ed1-e6fb-4ae1-92f5-ff24371acbd2-chroot-zfs\") pod \"openebs-zfs-node-s9qqv\" (UID: \"3c079ed1-e6fb-4ae1-92f5-ff24371acbd2\") " pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:03 truenas k3s[7306]: I1102 17:44:03.999774 7306 reconciler.go:157] "Reconciler: start to sync state" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.099066 7306 shared_informer.go:247] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.099072 7306 shared_informer.go:247] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.099094 7306 shared_informer.go:247] Caches are synced for RequestHeaderAuthRequestController Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.185329 7306 shared_informer.go:240] Waiting for caches to sync for tokens Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.194597 7306 controllermanager.go:605] Started "cronjob" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.195726 7306 cronjob_controllerv2.go:132] "Starting cronjob controller v2" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.195756 7306 shared_informer.go:240] Waiting for caches to sync for cronjob Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.199186 7306 node_lifecycle_controller.go:377] Sending events to api server. Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.199910 7306 taint_manager.go:163] "Sending events to api server" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.200059 7306 node_lifecycle_controller.go:505] Controller will reconcile labels. Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.200335 7306 controllermanager.go:605] Started "nodelifecycle" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.200480 7306 node_lifecycle_controller.go:539] Starting node controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.200503 7306 shared_informer.go:240] Waiting for caches to sync for taint Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.204774 7306 controllermanager.go:605] Started "endpoint" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.204858 7306 endpoints_controller.go:193] Starting endpoint controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.204876 7306 shared_informer.go:240] Waiting for caches to sync for endpoint Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.212041 7306 controllermanager.go:605] Started "disruption" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.212473 7306 disruption.go:363] Starting disruption controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.212494 7306 shared_informer.go:240] Waiting for caches to sync for disruption Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.215894 7306 controllermanager.go:605] Started "ttl" Nov 2 17:44:04 truenas k3s[7306]: W1102 17:44:04.215919 7306 controllermanager.go:570] "bootstrapsigner" is disabled Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.216158 7306 ttl_controller.go:121] Starting TTL controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.216177 7306 shared_informer.go:240] Waiting for caches to sync for TTL Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.220645 7306 controllermanager.go:605] Started "attachdetach" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.221681 7306 attach_detach_controller.go:328] Starting attach detach controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.221703 7306 shared_informer.go:240] Waiting for caches to sync for attach detach Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.224691 7306 controllermanager.go:605] Started "clusterrole-aggregation" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.224936 7306 clusterroleaggregation_controller.go:194] Starting ClusterRoleAggregator Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.224960 7306 shared_informer.go:240] Waiting for caches to sync for ClusterRoleAggregator Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.228587 7306 controllermanager.go:605] Started "ttl-after-finished" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.228650 7306 ttlafterfinished_controller.go:109] Starting TTL after finished controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.228667 7306 shared_informer.go:240] Waiting for caches to sync for TTL after finished Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.232600 7306 controllermanager.go:605] Started "root-ca-cert-publisher" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.232780 7306 publisher.go:107] Starting root CA certificate configmap publisher Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.232800 7306 shared_informer.go:240] Waiting for caches to sync for crt configmap Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.257815 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="e9828288cd6b67602c86e04d805accd4ede1172cb67cc014f232632d0cdbd97d" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.257858 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="6c7af75a15477e5a98670274d8dde604322540d7b921993301a5f3226b9f1c44" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.257923 7306 scope.go:110] "RemoveContainer" containerID="53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" Nov 2 17:44:04 truenas k3s[7306]: E1102 17:44:04.260609 7306 pod_workers.go:949] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=fde5b1cd-da44-4a8e-9375-5fc09fc7972f Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.262841 7306 controllermanager.go:605] Started "horizontalpodautoscaling" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.263089 7306 horizontal.go:168] Starting HPA controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.263112 7306 shared_informer.go:240] Waiting for caches to sync for HPA Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.272488 7306 controllermanager.go:605] Started "pvc-protection" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.273095 7306 pvc_protection_controller.go:103] "Starting PVC protection controller" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.273135 7306 shared_informer.go:240] Waiting for caches to sync for PVC protection Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.276828 7306 controllermanager.go:605] Started "pv-protection" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.277663 7306 pv_protection_controller.go:79] Starting PV protection controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.277685 7306 shared_informer.go:240] Waiting for caches to sync for PV protection Nov 2 17:44:04 truenas k3s[7306]: E1102 17:44:04.284814 7306 core.go:92] Failed to start service controller: WARNING: no cloud provider provided, services of type LoadBalancer will fail Nov 2 17:44:04 truenas k3s[7306]: W1102 17:44:04.284842 7306 controllermanager.go:583] Skipping "service" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.286052 7306 shared_informer.go:247] Caches are synced for tokens Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.295583 7306 garbagecollector.go:146] Starting garbage collector controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.295626 7306 shared_informer.go:240] Waiting for caches to sync for garbage collector Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.295755 7306 graph_builder.go:289] GraphBuilder running Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.296130 7306 controllermanager.go:605] Started "garbagecollector" Nov 2 17:44:04 truenas k3s[7306]: W1102 17:44:04.296442 7306 core.go:226] configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes. Nov 2 17:44:04 truenas k3s[7306]: W1102 17:44:04.296465 7306 controllermanager.go:583] Skipping "route" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.300180 7306 controllermanager.go:605] Started "persistentvolume-expander" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.300413 7306 expand_controller.go:342] Starting expand controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.300450 7306 shared_informer.go:240] Waiting for caches to sync for expand Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.303347 7306 controllermanager.go:605] Started "podgc" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.303875 7306 gc_controller.go:89] Starting GC controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.304135 7306 shared_informer.go:240] Waiting for caches to sync for GC Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.340040 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="bfc6341c81c17d32989f332359586a9b9c5da06055769632c55577ed90c044c4" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.340083 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="a3e16336bbaa1a3d2941df52c3030dc58bd4de669e9ff1085aec9044fc94d5b4" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.340329 7306 controllermanager.go:605] Started "namespace" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.340589 7306 namespace_controller.go:200] Starting namespace controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.340609 7306 shared_informer.go:240] Waiting for caches to sync for namespace Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.344369 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"coredns-d76bd69b-rqdx5_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"43cff5ff65a4501017ffcc6c6ff5d6ec6370730c006d80a53464f96686c9f13b\"" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.344526 7306 controllermanager.go:605] Started "csrcleaner" Nov 2 17:44:04 truenas k3s[7306]: W1102 17:44:04.344544 7306 controllermanager.go:570] "tokencleaner" is disabled Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.345271 7306 cleaner.go:82] Starting CSR cleaner controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.347639 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"coredns-d76bd69b-rqdx5_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"547375f47425b4a03d55db03ec887a3b35c15c21516c4019925e8acb344ce3fd\"" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.349228 7306 controllermanager.go:605] Started "endpointslice" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.349431 7306 endpointslice_controller.go:257] Starting endpoint slice controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.349452 7306 shared_informer.go:240] Waiting for caches to sync for endpoint_slice Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.353631 7306 controllermanager.go:605] Started "job" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.353750 7306 job_controller.go:186] Starting job controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.353767 7306 shared_informer.go:240] Waiting for caches to sync for job Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.358169 7306 node_lifecycle_controller.go:77] Sending events to api server Nov 2 17:44:04 truenas k3s[7306]: E1102 17:44:04.358224 7306 core.go:212] failed to start cloud node lifecycle controller: no cloud provider provided Nov 2 17:44:04 truenas k3s[7306]: W1102 17:44:04.358240 7306 controllermanager.go:583] Skipping "cloud-node-lifecycle" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.361228 7306 controllermanager.go:605] Started "ephemeral-volume" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.361364 7306 controller.go:170] Starting ephemeral volume controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.361383 7306 shared_informer.go:240] Waiting for caches to sync for ephemeral Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.364228 7306 controllermanager.go:605] Started "serviceaccount" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.364491 7306 serviceaccounts_controller.go:117] Starting service account controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.364512 7306 shared_informer.go:240] Waiting for caches to sync for service account Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.393733 7306 controllermanager.go:605] Started "replicationcontroller" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.394200 7306 replica_set.go:186] Starting replicationcontroller controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.394238 7306 shared_informer.go:240] Waiting for caches to sync for ReplicationController Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.442478 7306 controllermanager.go:605] Started "statefulset" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.442618 7306 stateful_set.go:147] Starting stateful set controller Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.442634 7306 shared_informer.go:240] Waiting for caches to sync for stateful set Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.481249 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="43cff5ff65a4501017ffcc6c6ff5d6ec6370730c006d80a53464f96686c9f13b" Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.481296 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="547375f47425b4a03d55db03ec887a3b35c15c21516c4019925e8acb344ce3fd" Nov 2 17:44:04 truenas k3s[7306]: E1102 17:44:04.481415 7306 pod_workers.go:949] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-d76bd69b-rqdx5" podUID=eb786c1d-1288-40a1-9e7c-1d8da9188f4d Nov 2 17:44:04 truenas k3s[7306]: I1102 17:44:04.490686 7306 node_ipam_controller.go:91] Sending events to api server. Nov 2 17:44:05 truenas k3s[7306]: I1102 17:44:05.493549 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"e9828288cd6b67602c86e04d805accd4ede1172cb67cc014f232632d0cdbd97d\"" Nov 2 17:44:05 truenas k3s[7306]: I1102 17:44:05.495524 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"6c7af75a15477e5a98670274d8dde604322540d7b921993301a5f3226b9f1c44\"" Nov 2 17:44:05 truenas k3s[7306]: E1102 17:44:05.520552 7306 pod_workers.go:949] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-d76bd69b-rqdx5" podUID=eb786c1d-1288-40a1-9e7c-1d8da9188f4d Nov 2 17:44:05 truenas k3s[7306]: E1102 17:44:05.520828 7306 pod_workers.go:949] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=fde5b1cd-da44-4a8e-9375-5fc09fc7972f Nov 2 17:44:05 truenas k3s[7306]: I1102 17:44:05.657979 7306 scope.go:110] "RemoveContainer" containerID="53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5" Nov 2 17:44:06 truenas k3s[7306]: time="2022-11-02T17:44:06Z" level=info msg="Stopped tunnel to 127.0.0.1:6443" Nov 2 17:44:06 truenas k3s[7306]: time="2022-11-02T17:44:06Z" level=info msg="Proxy done" err="context canceled" url="wss://127.0.0.1:6443/v1-k3s/connect" Nov 2 17:44:06 truenas k3s[7306]: time="2022-11-02T17:44:06Z" level=info msg="Connecting to proxy" url="wss://192.168.0.237:6443/v1-k3s/connect" Nov 2 17:44:06 truenas k3s[7306]: time="2022-11-02T17:44:06Z" level=info msg="error in remotedialer server [400]: websocket: close 1006 (abnormal closure): unexpected EOF" Nov 2 17:44:06 truenas k3s[7306]: time="2022-11-02T17:44:06Z" level=info msg="Handling backend connection request [ix-truenas]" Nov 2 17:44:06 truenas k3s[7306]: I1102 17:44:06.553253 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"e9828288cd6b67602c86e04d805accd4ede1172cb67cc014f232632d0cdbd97d\"" Nov 2 17:44:06 truenas k3s[7306]: I1102 17:44:06.555188 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"6c7af75a15477e5a98670274d8dde604322540d7b921993301a5f3226b9f1c44\"" Nov 2 17:44:06 truenas k3s[7306]: I1102 17:44:06.580970 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="c096730de4b00bbb2632072b8eb458506f30f8bd3afbd1f77bf2aacb51306da4" Nov 2 17:44:06 truenas k3s[7306]: E1102 17:44:06.581154 7306 pod_workers.go:949] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-d76bd69b-rqdx5" podUID=eb786c1d-1288-40a1-9e7c-1d8da9188f4d Nov 2 17:44:06 truenas k3s[7306]: E1102 17:44:06.581474 7306 pod_workers.go:949] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=fde5b1cd-da44-4a8e-9375-5fc09fc7972f Nov 2 17:44:07 truenas k3s[7306]: I1102 17:44:07.069752 7306 scope.go:110] "RemoveContainer" containerID="3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" Nov 2 17:44:07 truenas k3s[7306]: I1102 17:44:07.594541 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"e9828288cd6b67602c86e04d805accd4ede1172cb67cc014f232632d0cdbd97d\"" Nov 2 17:44:07 truenas k3s[7306]: I1102 17:44:07.596168 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"6c7af75a15477e5a98670274d8dde604322540d7b921993301a5f3226b9f1c44\"" Nov 2 17:44:07 truenas k3s[7306]: I1102 17:44:07.617839 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" Nov 2 17:44:07 truenas k3s[7306]: E1102 17:44:07.618187 7306 pod_workers.go:949] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/openebs-zfs-controller-0" podUID=fde5b1cd-da44-4a8e-9375-5fc09fc7972f Nov 2 17:44:07 truenas k3s[7306]: E1102 17:44:07.803208 7306 pod_workers.go:949] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized" pod="kube-system/coredns-d76bd69b-rqdx5" podUID=eb786c1d-1288-40a1-9e7c-1d8da9188f4d Nov 2 17:44:08 truenas k3s[7306]: I1102 17:44:08.298447 7306 scope.go:110] "RemoveContainer" containerID="53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" Nov 2 17:44:08 truenas k3s[7306]: E1102 17:44:08.308915 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" containerID="53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" Nov 2 17:44:08 truenas k3s[7306]: E1102 17:44:08.309841 7306 kuberuntime_gc.go:146] "Failed to remove container" err="failed to get container status \"53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f\": rpc error: code = Unknown desc = Error: No such container: 53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" containerID="53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" Nov 2 17:44:08 truenas k3s[7306]: I1102 17:44:08.309892 7306 scope.go:110] "RemoveContainer" containerID="3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" Nov 2 17:44:08 truenas k3s[7306]: E1102 17:44:08.323684 7306 remote_runtime.go:505] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143\": Error response from daemon: removal of container 3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143 is already in progress" containerID="3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" Nov 2 17:44:08 truenas k3s[7306]: E1102 17:44:08.323741 7306 kuberuntime_gc.go:146] "Failed to remove container" err="rpc error: code = Unknown desc = failed to remove container \"3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143\": Error response from daemon: removal of container 3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143 is already in progress" containerID="3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" Nov 2 17:44:08 truenas k3s[7306]: I1102 17:44:08.323770 7306 scope.go:110] "RemoveContainer" containerID="68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d" Nov 2 17:44:08 truenas k3s[7306]: I1102 17:44:08.644268 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5" Nov 2 17:44:09 truenas k3s[7306]: I1102 17:44:09.655906 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"e9828288cd6b67602c86e04d805accd4ede1172cb67cc014f232632d0cdbd97d\"" Nov 2 17:44:09 truenas k3s[7306]: I1102 17:44:09.658592 7306 docker_sandbox.go:402] "Failed to read pod IP from plugin/docker" err="networkPlugin cni failed on the status hook for pod \"openebs-zfs-controller-0_kube-system\": CNI failed to retrieve network namespace path: cannot find network namespace for the terminated container \"6c7af75a15477e5a98670274d8dde604322540d7b921993301a5f3226b9f1c44\"" Nov 2 17:44:09 truenas k3s[7306]: I1102 17:44:09.788850 7306 scope.go:110] "RemoveContainer" containerID="68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d" Nov 2 17:44:09 truenas k3s[7306]: E1102 17:44:09.793094 7306 remote_runtime.go:505] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d\": Error response from daemon: removal of container 68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d is already in progress" containerID="68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d" Nov 2 17:44:09 truenas k3s[7306]: I1102 17:44:09.793158 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d} err="rpc error: code = Unknown desc = failed to remove container \"68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d\": Error response from daemon: removal of container 68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d is already in progress" Nov 2 17:44:09 truenas k3s[7306]: I1102 17:44:09.793187 7306 scope.go:110] "RemoveContainer" containerID="408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" Nov 2 17:44:09 truenas k3s[7306]: I1102 17:44:09.848411 7306 cni.go:334] "CNI failed to retrieve network namespace path" err="cannot find network namespace for the terminated container \"43cff5ff65a4501017ffcc6c6ff5d6ec6370730c006d80a53464f96686c9f13b\"" Nov 2 17:44:09 truenas k3s[7306]: I1102 17:44:09.852610 7306 cni.go:334] "CNI failed to retrieve network namespace path" err="cannot find network namespace for the terminated container \"e9828288cd6b67602c86e04d805accd4ede1172cb67cc014f232632d0cdbd97d\"" Nov 2 17:44:10 truenas k3s[7306]: I1102 17:44:10.009776 7306 cni.go:334] "CNI failed to retrieve network namespace path" err="cannot find network namespace for the terminated container \"6c7af75a15477e5a98670274d8dde604322540d7b921993301a5f3226b9f1c44\"" Nov 2 17:44:10 truenas k3s[7306]: I1102 17:44:10.020322 7306 cni.go:334] "CNI failed to retrieve network namespace path" err="cannot find network namespace for the terminated container \"547375f47425b4a03d55db03ec887a3b35c15c21516c4019925e8acb344ce3fd\"" Nov 2 17:44:10 truenas k3s[7306]: I1102 17:44:10.710377 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" Nov 2 17:44:11 truenas k3s[7306]: I1102 17:44:11.067313 7306 scope.go:110] "RemoveContainer" containerID="408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" Nov 2 17:44:11 truenas k3s[7306]: E1102 17:44:11.071675 7306 remote_runtime.go:505] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6\": Error response from daemon: removal of container 408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6 is already in progress" containerID="408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" Nov 2 17:44:11 truenas k3s[7306]: E1102 17:44:11.071731 7306 kuberuntime_gc.go:146] "Failed to remove container" err="rpc error: code = Unknown desc = failed to remove container \"408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6\": Error response from daemon: removal of container 408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6 is already in progress" containerID="408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" Nov 2 17:44:11 truenas k3s[7306]: I1102 17:44:11.071760 7306 scope.go:110] "RemoveContainer" containerID="770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e" Nov 2 17:44:11 truenas k3s[7306]: I1102 17:44:11.739267 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.355453 7306 scope.go:110] "RemoveContainer" containerID="c096730de4b00bbb2632072b8eb458506f30f8bd3afbd1f77bf2aacb51306da4" Nov 2 17:44:12 truenas k3s[7306]: E1102 17:44:12.356441 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: c096730de4b00bbb2632072b8eb458506f30f8bd3afbd1f77bf2aacb51306da4" containerID="c096730de4b00bbb2632072b8eb458506f30f8bd3afbd1f77bf2aacb51306da4" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.356508 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:c096730de4b00bbb2632072b8eb458506f30f8bd3afbd1f77bf2aacb51306da4} err="failed to get container status \"c096730de4b00bbb2632072b8eb458506f30f8bd3afbd1f77bf2aacb51306da4\": rpc error: code = Unknown desc = Error: No such container: c096730de4b00bbb2632072b8eb458506f30f8bd3afbd1f77bf2aacb51306da4" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.356538 7306 scope.go:110] "RemoveContainer" containerID="53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.363223 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f} err="failed to get container status \"53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f\": rpc error: code = Unknown desc = Error: No such container: 53639a059c0c489f3f3c19c73e597234c5fd0de225b5f85b3ab07b16e9481e7f" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.363266 7306 scope.go:110] "RemoveContainer" containerID="3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" Nov 2 17:44:12 truenas k3s[7306]: E1102 17:44:12.365354 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" containerID="3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.365444 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143} err="failed to get container status \"3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143\": rpc error: code = Unknown desc = Error: No such container: 3f0227eced2bd2295c8534054dd963066f6f062a505cf9b178ddea81456e0143" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.365474 7306 scope.go:110] "RemoveContainer" containerID="68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d" Nov 2 17:44:12 truenas k3s[7306]: E1102 17:44:12.366601 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d" containerID="68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.366660 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d} err="failed to get container status \"68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d\": rpc error: code = Unknown desc = Error: No such container: 68a1d94e783ee09efd33881bb9a3807eb3d4fc5e9c1b70f7f04e82261f73932d" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.366685 7306 scope.go:110] "RemoveContainer" containerID="408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" Nov 2 17:44:12 truenas k3s[7306]: E1102 17:44:12.368573 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" containerID="408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.368956 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6} err="failed to get container status \"408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6\": rpc error: code = Unknown desc = Error: No such container: 408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.368986 7306 scope.go:110] "RemoveContainer" containerID="770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e" Nov 2 17:44:12 truenas k3s[7306]: E1102 17:44:12.375778 7306 remote_runtime.go:505] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e\": Error response from daemon: removal of container 770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e is already in progress" containerID="770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.375853 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e} err="rpc error: code = Unknown desc = failed to remove container \"770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e\": Error response from daemon: removal of container 770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e is already in progress" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.375891 7306 scope.go:110] "RemoveContainer" containerID="53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5" Nov 2 17:44:12 truenas k3s[7306]: E1102 17:44:12.376871 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5" containerID="53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.376936 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5} err="failed to get container status \"53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5\": rpc error: code = Unknown desc = Error: No such container: 53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.376960 7306 scope.go:110] "RemoveContainer" containerID="770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e" Nov 2 17:44:12 truenas k3s[7306]: E1102 17:44:12.383058 7306 remote_runtime.go:505] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e\": Error response from daemon: removal of container 770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e is already in progress" containerID="770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.383122 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e} err="rpc error: code = Unknown desc = failed to remove container \"770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e\": Error response from daemon: removal of container 770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e is already in progress" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.383150 7306 scope.go:110] "RemoveContainer" containerID="53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.384279 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5} err="failed to get container status \"53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5\": rpc error: code = Unknown desc = Error: No such container: 53449f387730c9ba11199e33fed40cb3a13d70d8bd3920a643da3502b39c8aa5" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.384310 7306 scope.go:110] "RemoveContainer" containerID="f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" Nov 2 17:44:12 truenas k3s[7306]: I1102 17:44:12.771176 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="408847f75ccc4a939d1bd222390c740850bbc317a03e0adefaf07dc878aa6ec6" Nov 2 17:44:13 truenas k3s[7306]: I1102 17:44:13.713441 7306 scope.go:110] "RemoveContainer" containerID="f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" Nov 2 17:44:13 truenas k3s[7306]: E1102 17:44:13.718399 7306 remote_runtime.go:505] "RemoveContainer from runtime service failed" err="rpc error: code = Unknown desc = failed to remove container \"f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3\": Error response from daemon: removal of container f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3 is already in progress" containerID="f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" Nov 2 17:44:13 truenas k3s[7306]: E1102 17:44:13.718456 7306 kuberuntime_gc.go:146] "Failed to remove container" err="rpc error: code = Unknown desc = failed to remove container \"f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3\": Error response from daemon: removal of container f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3 is already in progress" containerID="f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" Nov 2 17:44:13 truenas k3s[7306]: I1102 17:44:13.789138 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="770e0c4a4c2fc745317b7d5c81f247c69eb097b634cbfd2e730b72031c037d2e" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.529347 7306 range_allocator.go:83] Sending events to api server. Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.529856 7306 range_allocator.go:111] No Service CIDR provided. Skipping filtering out service addresses. Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.529878 7306 range_allocator.go:117] No Secondary Service CIDR provided. Skipping filtering out secondary service addresses. Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.529971 7306 controllermanager.go:605] Started "nodeipam" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.530084 7306 node_ipam_controller.go:154] Starting ipam controller Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.530109 7306 shared_informer.go:240] Waiting for caches to sync for node Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.533655 7306 controllermanager.go:605] Started "persistentvolume-binder" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.534321 7306 pv_controller_base.go:310] Starting persistent volume controller Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.534344 7306 shared_informer.go:240] Waiting for caches to sync for persistent volume Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.537760 7306 controllermanager.go:605] Started "endpointslicemirroring" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.538205 7306 endpointslicemirroring_controller.go:212] Starting EndpointSliceMirroring controller Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.538230 7306 shared_informer.go:240] Waiting for caches to sync for endpoint_slice_mirroring Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.541465 7306 controllermanager.go:605] Started "daemonset" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.542068 7306 daemon_controller.go:284] Starting daemon sets controller Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.542092 7306 shared_informer.go:240] Waiting for caches to sync for daemon sets Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.545142 7306 controllermanager.go:605] Started "deployment" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.545447 7306 deployment_controller.go:153] "Starting controller" controller="deployment" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.545470 7306 shared_informer.go:240] Waiting for caches to sync for deployment Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.549289 7306 controllermanager.go:605] Started "replicaset" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.549408 7306 replica_set.go:186] Starting replicaset controller Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.549432 7306 shared_informer.go:240] Waiting for caches to sync for ReplicaSet Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.554676 7306 dynamic_serving_content.go:131] "Starting controller" name="csr-controller::/mnt/SmallPool/ix-applications/k3s/server/tls/server-ca.crt::/mnt/SmallPool/ix-applications/k3s/server/tls/server-ca.key" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.554865 7306 certificate_controller.go:118] Starting certificate controller "csrsigning-kubelet-serving" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.554894 7306 shared_informer.go:240] Waiting for caches to sync for certificate-csrsigning-kubelet-serving Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.555223 7306 certificate_controller.go:118] Starting certificate controller "csrsigning-kubelet-client" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.555249 7306 shared_informer.go:240] Waiting for caches to sync for certificate-csrsigning-kubelet-client Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.555423 7306 dynamic_serving_content.go:131] "Starting controller" name="csr-controller::/mnt/SmallPool/ix-applications/k3s/server/tls/client-ca.crt::/mnt/SmallPool/ix-applications/k3s/server/tls/client-ca.key" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.555692 7306 certificate_controller.go:118] Starting certificate controller "csrsigning-kube-apiserver-client" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.555717 7306 shared_informer.go:240] Waiting for caches to sync for certificate-csrsigning-kube-apiserver-client Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.555863 7306 dynamic_serving_content.go:131] "Starting controller" name="csr-controller::/mnt/SmallPool/ix-applications/k3s/server/tls/client-ca.crt::/mnt/SmallPool/ix-applications/k3s/server/tls/client-ca.key" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.556083 7306 controllermanager.go:605] Started "csrsigning" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.556190 7306 certificate_controller.go:118] Starting certificate controller "csrsigning-legacy-unknown" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.556207 7306 shared_informer.go:240] Waiting for caches to sync for certificate-csrsigning-legacy-unknown Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.556262 7306 dynamic_serving_content.go:131] "Starting controller" name="csr-controller::/mnt/SmallPool/ix-applications/k3s/server/tls/server-ca.crt::/mnt/SmallPool/ix-applications/k3s/server/tls/server-ca.key" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.559633 7306 controllermanager.go:605] Started "csrapproving" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.559755 7306 certificate_controller.go:118] Starting certificate controller "csrapproving" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.559774 7306 shared_informer.go:240] Waiting for caches to sync for certificate-csrapproving Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.590808 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for roles.rbac.authorization.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591213 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for network-attachment-definitions.k8s.cni.cncf.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591276 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for horizontalpodautoscalers.autoscaling Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591311 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for jobs.batch Nov 2 17:44:14 truenas k3s[7306]: W1102 17:44:14.591355 7306 shared_informer.go:504] resyncPeriod 13h44m25.249833918s is smaller than resyncCheckPeriod 15h59m47.079921374s and the informer has already started. Changing it to 15h59m47.079921374s Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591470 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for serviceaccounts Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591518 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for controllerrevisions.apps Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591573 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for addons.k3s.cattle.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591822 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for events.events.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591868 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for zfsvolumes.zfs.openebs.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591909 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for zfssnapshots.zfs.openebs.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591949 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for zfsnodes.zfs.openebs.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.591991 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for zfsrestores.zfs.openebs.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592104 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for statefulsets.apps Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592147 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for helmchartconfigs.helm.cattle.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592195 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for deployments.apps Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592235 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for helmcharts.helm.cattle.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592268 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for leases.coordination.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592311 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for volumesnapshots.snapshot.storage.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592373 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for limitranges Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592420 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for rolebindings.rbac.authorization.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592455 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for daemonsets.apps Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592492 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for replicasets.apps Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592529 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for cronjobs.batch Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592570 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for ingresses.networking.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592630 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for csistoragecapacities.storage.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592669 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for podtemplates Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592702 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for endpoints Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592738 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for poddisruptionbudgets.policy Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592772 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for endpointslices.discovery.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.592813 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for zfsbackups.zfs.openebs.io Nov 2 17:44:14 truenas k3s[7306]: W1102 17:44:14.592854 7306 shared_informer.go:504] resyncPeriod 12h3m59.596149311s is smaller than resyncCheckPeriod 15h59m47.079921374s and the informer has already started. Changing it to 15h59m47.079921374s Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.593003 7306 resource_quota_monitor.go:233] QuotaMonitor created object count evaluator for networkpolicies.networking.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.593032 7306 controllermanager.go:605] Started "resourcequota" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.593107 7306 resource_quota_controller.go:273] Starting resource quota controller Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.593123 7306 shared_informer.go:240] Waiting for caches to sync for resource quota Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.593177 7306 resource_quota_monitor.go:308] QuotaMonitor running Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.612483 7306 shared_informer.go:240] Waiting for caches to sync for resource quota Nov 2 17:44:14 truenas k3s[7306]: W1102 17:44:14.634505 7306 actual_state_of_world.go:539] Failed to update statusUpdateNeeded field in actual state of world: Failed to set statusUpdateNeeded to needed true, because nodeName="ix-truenas" does not exist Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.634778 7306 shared_informer.go:247] Caches are synced for persistent volume Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.642738 7306 shared_informer.go:247] Caches are synced for stateful set Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.643177 7306 shared_informer.go:247] Caches are synced for crt configmap Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.653407 7306 shared_informer.go:247] Caches are synced for ReplicaSet Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.656278 7306 shared_informer.go:247] Caches are synced for certificate-csrsigning-kubelet-client Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.660872 7306 shared_informer.go:247] Caches are synced for certificate-csrapproving Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.663297 7306 shared_informer.go:247] Caches are synced for job Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.667231 7306 shared_informer.go:247] Caches are synced for endpoint_slice_mirroring Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.678046 7306 shared_informer.go:247] Caches are synced for PV protection Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.679465 7306 shared_informer.go:247] Caches are synced for daemon sets Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.680372 7306 shared_informer.go:247] Caches are synced for deployment Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.681269 7306 shared_informer.go:247] Caches are synced for PVC protection Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.681405 7306 shared_informer.go:247] Caches are synced for namespace Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.702368 7306 shared_informer.go:247] Caches are synced for taint Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.702868 7306 node_lifecycle_controller.go:1397] Initializing eviction metric for zone: Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.703809 7306 shared_informer.go:247] Caches are synced for ephemeral Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.711070 7306 shared_informer.go:247] Caches are synced for GC Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.711989 7306 shared_informer.go:247] Caches are synced for certificate-csrsigning-kubelet-serving Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.712405 7306 shared_informer.go:247] Caches are synced for certificate-csrsigning-kube-apiserver-client Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.717261 7306 shared_informer.go:247] Caches are synced for ReplicationController Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.717507 7306 shared_informer.go:247] Caches are synced for expand Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.718035 7306 shared_informer.go:247] Caches are synced for endpoint_slice Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.722090 7306 taint_manager.go:187] "Starting NoExecuteTaintManager" Nov 2 17:44:14 truenas k3s[7306]: W1102 17:44:14.723714 7306 node_lifecycle_controller.go:1012] Missing timestamp for Node ix-truenas. Assuming now as a timestamp. Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.723979 7306 shared_informer.go:247] Caches are synced for disruption Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.724006 7306 disruption.go:371] Sending events to api server. Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.724481 7306 node_lifecycle_controller.go:1213] Controller detected that zone is now in state Normal. Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.728200 7306 event.go:294] "Event occurred" object="ix-truenas" kind="Node" apiVersion="v1" type="Normal" reason="RegisteredNode" message="Node ix-truenas event: Registered Node ix-truenas in Controller" Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.728491 7306 shared_informer.go:247] Caches are synced for ClusterRoleAggregator Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.729673 7306 shared_informer.go:247] Caches are synced for endpoint Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.729776 7306 shared_informer.go:247] Caches are synced for TTL Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.735235 7306 shared_informer.go:247] Caches are synced for node Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.735492 7306 range_allocator.go:173] Starting range CIDR allocator Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.735633 7306 shared_informer.go:240] Waiting for caches to sync for cidrallocator Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.735778 7306 shared_informer.go:247] Caches are synced for cidrallocator Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.735907 7306 shared_informer.go:247] Caches are synced for TTL after finished Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.736127 7306 shared_informer.go:247] Caches are synced for service account Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.736456 7306 shared_informer.go:247] Caches are synced for cronjob Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.735866 7306 shared_informer.go:247] Caches are synced for certificate-csrsigning-legacy-unknown Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.763343 7306 shared_informer.go:240] Waiting for caches to sync for garbage collector Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.773842 7306 controller.go:611] quota admission added evaluator for: endpointslices.discovery.k8s.io Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.817276 7306 controller.go:611] quota admission added evaluator for: endpoints Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.821841 7306 shared_informer.go:247] Caches are synced for attach detach Nov 2 17:44:14 truenas k3s[7306]: I1102 17:44:14.863834 7306 shared_informer.go:247] Caches are synced for HPA Nov 2 17:44:15 truenas k3s[7306]: I1102 17:44:15.093356 7306 shared_informer.go:247] Caches are synced for resource quota Nov 2 17:44:15 truenas k3s[7306]: I1102 17:44:15.113002 7306 shared_informer.go:247] Caches are synced for resource quota Nov 2 17:44:15 truenas k3s[7306]: I1102 17:44:15.363753 7306 shared_informer.go:247] Caches are synced for garbage collector Nov 2 17:44:15 truenas k3s[7306]: I1102 17:44:15.396737 7306 shared_informer.go:247] Caches are synced for garbage collector Nov 2 17:44:15 truenas k3s[7306]: I1102 17:44:15.396779 7306 garbagecollector.go:155] Garbage collector: all resource monitors have synced. Proceeding to collect garbage Nov 2 17:44:16 truenas k3s[7306]: I1102 17:44:16.169419 7306 scope.go:110] "RemoveContainer" containerID="f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" Nov 2 17:44:16 truenas k3s[7306]: E1102 17:44:16.171087 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" containerID="f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" Nov 2 17:44:16 truenas k3s[7306]: I1102 17:44:16.171161 7306 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:docker ID:f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3} err="failed to get container status \"f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3\": rpc error: code = Unknown desc = Error: No such container: f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" Nov 2 17:44:16 truenas k3s[7306]: I1102 17:44:16.841230 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="f4e25eb3a49b1d044b98b45ddca9b21242a79bae9bb2f6e202b5fc271d3e8af3" Nov 2 17:44:19 truenas k3s[7306]: E1102 17:44:19.879415 7306 kuberuntime_manager.go:1065] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" podSandboxID="00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:20 truenas k3s[7306]: E1102 17:44:20.888315 7306 kuberuntime_manager.go:1065] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" podSandboxID="00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:20 truenas k3s[7306]: E1102 17:44:20.890548 7306 kuberuntime_manager.go:1065] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" podSandboxID="00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:21 truenas k3s[7306]: E1102 17:44:21.899155 7306 kuberuntime_manager.go:1065] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" podSandboxID="00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:21 truenas k3s[7306]: E1102 17:44:21.901141 7306 kuberuntime_manager.go:1065] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" podSandboxID="00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:26 truenas k3s[7306]: I1102 17:44:26.296518 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="00373e41d3f941cd55fdb6a7f41b8230fe910efcd81841b2ce0803e75baf7152" Nov 2 17:44:28 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:44:28.205Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:44:28 truenas k3s[7306]: E1102 17:44:28.320783 7306 kuberuntime_manager.go:1065] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 6b9aa15fcb7a0c57903e3023398d69c1eca3cd5678612f08113a61fdcfa4723f" podSandboxID="6b9aa15fcb7a0c57903e3023398d69c1eca3cd5678612f08113a61fdcfa4723f" pod="kube-system/openebs-zfs-controller-0" Nov 2 17:44:29 truenas k3s[7306]: E1102 17:44:29.330211 7306 kuberuntime_manager.go:1065] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 6b9aa15fcb7a0c57903e3023398d69c1eca3cd5678612f08113a61fdcfa4723f" podSandboxID="6b9aa15fcb7a0c57903e3023398d69c1eca3cd5678612f08113a61fdcfa4723f" pod="kube-system/openebs-zfs-controller-0" Nov 2 17:44:29 truenas k3s[7306]: E1102 17:44:29.333388 7306 kuberuntime_manager.go:1065] "PodSandboxStatus of sandbox for pod" err="rpc error: code = Unknown desc = Error: No such container: 6b9aa15fcb7a0c57903e3023398d69c1eca3cd5678612f08113a61fdcfa4723f" podSandboxID="6b9aa15fcb7a0c57903e3023398d69c1eca3cd5678612f08113a61fdcfa4723f" pod="kube-system/openebs-zfs-controller-0" Nov 2 17:44:34 truenas k3s[7306]: I1102 17:44:34.007125 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="6b9aa15fcb7a0c57903e3023398d69c1eca3cd5678612f08113a61fdcfa4723f" Nov 2 17:44:34 truenas k3s[7306]: I1102 17:44:34.175827 10124 event.go:278] Event(v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"openebs-zfs-controller-0", UID:"fde5b1cd-da44-4a8e-9375-5fc09fc7972f", APIVersion:"v1", ResourceVersion:"219817", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.26/16] from ix-net Nov 2 17:44:35 truenas k3s[7306]: I1102 17:44:35.097756 7306 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="49851ec83aa05e3dc2758ce23d3492bdd0176788b0d69e831fc1fbeb316e0186" Nov 2 17:44:39 truenas k3s[7306]: I1102 17:44:39.213655 10394 event.go:278] Event(v1.ObjectReference{Kind:"Pod", Namespace:"kube-system", Name:"coredns-d76bd69b-rqdx5", UID:"eb786c1d-1288-40a1-9e7c-1d8da9188f4d", APIVersion:"v1", ResourceVersion:"219808", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.27/16] from ix-net Nov 2 17:44:44 truenas k3s[7306]: E1102 17:44:44.192356 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: ecc941f50587df13fb98c9c101ecd3a10cb6c255528fb98026ba85bbc151063e" containerID="ecc941f50587df13fb98c9c101ecd3a10cb6c255528fb98026ba85bbc151063e" Nov 2 17:44:44 truenas k3s[7306]: E1102 17:44:44.193212 7306 kuberuntime_manager.go:1079] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: ecc941f50587df13fb98c9c101ecd3a10cb6c255528fb98026ba85bbc151063e" pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:44:50 truenas k3s[7306]: E1102 17:44:50.702404 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 1c1c8271988aaf533592509f243b685f4ae2661f01132d284d325fad333021c3" containerID="1c1c8271988aaf533592509f243b685f4ae2661f01132d284d325fad333021c3" Nov 2 17:44:50 truenas k3s[7306]: E1102 17:44:50.702465 7306 kuberuntime_manager.go:1079] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 1c1c8271988aaf533592509f243b685f4ae2661f01132d284d325fad333021c3" pod="kube-system/openebs-zfs-controller-0" Nov 2 17:44:51 truenas k3s[7306]: E1102 17:44:51.721059 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 60718bc7bd2f23f74d2a19684dda54e915296413b4ad25cd525ce1eec89649fe" containerID="60718bc7bd2f23f74d2a19684dda54e915296413b4ad25cd525ce1eec89649fe" Nov 2 17:44:51 truenas k3s[7306]: E1102 17:44:51.721894 7306 kuberuntime_manager.go:1079] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 60718bc7bd2f23f74d2a19684dda54e915296413b4ad25cd525ce1eec89649fe" pod="kube-system/coredns-d76bd69b-rqdx5" Nov 2 17:45:03 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:45:03.426Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:45:05 truenas k3s[7306]: E1102 17:45:05.746372 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: bcb557bd496695fc4f883405981fb5d99cc236e3f6dabab38f75aa798a2902fa" containerID="bcb557bd496695fc4f883405981fb5d99cc236e3f6dabab38f75aa798a2902fa" Nov 2 17:45:05 truenas k3s[7306]: E1102 17:45:05.747228 7306 kuberuntime_manager.go:1079] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: bcb557bd496695fc4f883405981fb5d99cc236e3f6dabab38f75aa798a2902fa" pod="kube-system/openebs-zfs-node-s9qqv" Nov 2 17:45:08 truenas k3s[7306]: E1102 17:45:08.851242 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 858025d926c08504dea59e3f8b755a18367d8f79679ec1b547b7cfd4a524dbf3" containerID="858025d926c08504dea59e3f8b755a18367d8f79679ec1b547b7cfd4a524dbf3" Nov 2 17:45:08 truenas k3s[7306]: E1102 17:45:08.851304 7306 kuberuntime_manager.go:1079] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 858025d926c08504dea59e3f8b755a18367d8f79679ec1b547b7cfd4a524dbf3" pod="kube-system/openebs-zfs-controller-0" Nov 2 17:45:11 truenas k3s[7306]: I1102 17:45:11.114782 7306 controller.go:611] quota admission added evaluator for: zfsnodes.zfs.openebs.io Nov 2 17:45:11 truenas k3s[7306]: I1102 17:45:11.757944 7306 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: zfs.csi.openebs.io endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock versions: 1.0.0 Nov 2 17:45:11 truenas k3s[7306]: I1102 17:45:11.758008 7306 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: zfs.csi.openebs.io at endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock Nov 2 17:45:16 truenas k3s[7306]: E1102 17:45:16.000177 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: bb630d4c58ffe2c208c436afef927b51afae3a902ef7c8541c169a1c692ea36e" containerID="bb630d4c58ffe2c208c436afef927b51afae3a902ef7c8541c169a1c692ea36e" Nov 2 17:45:16 truenas k3s[7306]: E1102 17:45:16.001060 7306 kuberuntime_manager.go:1079] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: bb630d4c58ffe2c208c436afef927b51afae3a902ef7c8541c169a1c692ea36e" pod="kube-system/openebs-zfs-controller-0" Nov 2 17:45:23 truenas k3s[7306]: E1102 17:45:23.104443 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 5319a98d01fc26794d736e12be11c10a7332e2baddeb53e2994f1263d0a786cd" containerID="5319a98d01fc26794d736e12be11c10a7332e2baddeb53e2994f1263d0a786cd" Nov 2 17:45:23 truenas k3s[7306]: E1102 17:45:23.104487 7306 kuberuntime_manager.go:1079] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 5319a98d01fc26794d736e12be11c10a7332e2baddeb53e2994f1263d0a786cd" pod="kube-system/openebs-zfs-controller-0" Nov 2 17:45:30 truenas k3s[7306]: E1102 17:45:30.902452 7306 remote_runtime.go:572] "ContainerStatus from runtime service failed" err="rpc error: code = Unknown desc = Error: No such container: 424a091c35d4da943130f1feeee0f203dccfc24197862b742255d1e6a843b19b" containerID="424a091c35d4da943130f1feeee0f203dccfc24197862b742255d1e6a843b19b" Nov 2 17:45:30 truenas k3s[7306]: E1102 17:45:30.902523 7306 kuberuntime_manager.go:1079] "getPodContainerStatuses for pod failed" err="rpc error: code = Unknown desc = Error: No such container: 424a091c35d4da943130f1feeee0f203dccfc24197862b742255d1e6a843b19b" pod="kube-system/openebs-zfs-controller-0" Nov 2 17:45:41 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:45:41.022Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:46:21 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:46:21.102Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:47:06 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:47:06.065Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:47:47 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:47:47.406Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:48:17 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:48:17.534Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:48:51 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:48:51.611Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:49:26 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:49:26.053Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:50:09 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:50:09.959Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:50:53 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:50:53.680Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:51:37 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:51:37.196Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:52:19 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:52:19.893Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:53:04 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:53:04.271Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:53:38 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:53:38.587Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"} Nov 2 17:54:08 truenas k3s[7306]: {"level":"warn","ts":"2022-11-02T17:54:08.872Z","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fbc540/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}