diff --git a/53ll/values.yaml.gotmpl b/53ll/values.yaml.gotmpl index 904b29e..22af0d4 100644 --- a/53ll/values.yaml.gotmpl +++ b/53ll/values.yaml.gotmpl @@ -15,11 +15,11 @@ readinessProbe: resources: requests: cpu: 10m - ephemeral-storage: 50Mi + # Ephemeral storage is intentionally NOT requested: with Talos's single-EPHM + # layout every request here competes for the same small filesystem that + # holds container images, and the LimitRange default covers us. memory: 64Mi limits: - cpu: 500m - ephemeral-storage: 2Gi memory: 224Mi persistence: size: 1Gi diff --git a/README.md b/README.md index 69f8258..575e013 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Like the cloud but local. - just (https://just.systems/man/en/ ) ## Talos Cluster ### Hardware -- HP ProDesk 400 G3 (x3) +- HP ProDesk 400 G3 16Gib (x3) - Seagate Iron Wolf 2Tb (x2) - Western Digital Red Pro 2Tb - Sabrent HDD Docking Station (3x) diff --git a/audiobookshelf/values.yaml.gotmpl b/audiobookshelf/values.yaml.gotmpl index 6615032..29c2f7f 100644 --- a/audiobookshelf/values.yaml.gotmpl +++ b/audiobookshelf/values.yaml.gotmpl @@ -2,6 +2,14 @@ image: tag: 2.37.1 env: TZ: America/Edmonton + +# k8s-home-lab common lib: resources is top-level +resources: + requests: + cpu: 20m + memory: 128Mi + limits: + memory: 1Gi ingress: main: enabled: true diff --git a/calibre-web/values.yaml.gotmpl b/calibre-web/values.yaml.gotmpl index 3dc4610..2d316d9 100644 --- a/calibre-web/values.yaml.gotmpl +++ b/calibre-web/values.yaml.gotmpl @@ -1,6 +1,14 @@ env: TZ: America/Edmonton DOCKER_MODS: linuxserver/mods:universal-calibre + +# k8s-home-lab common lib: resources is top-level (see charts/common/templates/lib/controller/_container.tpl) +resources: + requests: + cpu: 20m + memory: 128Mi + limits: + memory: 768Mi ingress: main: enabled: true diff --git a/ceph-csi-drivers/values.yaml.gotmpl b/ceph-csi-drivers/values.yaml.gotmpl index 1a06620..c64545a 100644 --- a/ceph-csi-drivers/values.yaml.gotmpl +++ b/ceph-csi-drivers/values.yaml.gotmpl @@ -7,6 +7,10 @@ # If the Rook operator is installed in a namespace other than rook-ceph, replace # "rook-ceph" in the driver names below with your operator namespace. +# NOTE: driver.yaml renders `nodePlugin.resources. | default dict`, so +# omitting a container emits an explicit `{}` into the CR, which can wipe the +# operator's built-in sidecar defaults. Every container is therefore listed +# explicitly so the requests are deterministic and visible to the scheduler. operatorConfig: namespace: rook-ceph # namespace:operator driverSpecDefaults: @@ -14,8 +18,52 @@ operatorConfig: name: rook-csi-operator-image-set-configmap nodePlugin: priorityClassName: system-node-critical + resources: + registrar: + requests: {cpu: 5m, memory: 16Mi} + limits: {memory: 64Mi} + liveness: + requests: {cpu: 5m, memory: 16Mi} + limits: {memory: 64Mi} + logRotator: + requests: {cpu: 5m, memory: 16Mi} + limits: {memory: 64Mi} + addons: + requests: {cpu: 5m, memory: 16Mi} + limits: {memory: 64Mi} + plugin: + requests: {cpu: 20m, memory: 64Mi} + limits: {memory: 512Mi} controllerPlugin: priorityClassName: system-cluster-critical + resources: + attacher: + requests: {cpu: 10m, memory: 32Mi} + limits: {memory: 128Mi} + snapshotter: + requests: {cpu: 10m, memory: 32Mi} + limits: {memory: 128Mi} + resizer: + requests: {cpu: 10m, memory: 32Mi} + limits: {memory: 128Mi} + provisioner: + requests: {cpu: 10m, memory: 32Mi} + limits: {memory: 128Mi} + omapGenerator: + requests: {cpu: 10m, memory: 32Mi} + limits: {memory: 128Mi} + liveness: + requests: {cpu: 5m, memory: 16Mi} + limits: {memory: 64Mi} + logRotator: + requests: {cpu: 5m, memory: 16Mi} + limits: {memory: 64Mi} + addons: + requests: {cpu: 5m, memory: 16Mi} + limits: {memory: 64Mi} + plugin: + requests: {cpu: 50m, memory: 64Mi} + limits: {memory: 256Mi} drivers: rbd: diff --git a/cert-manager-hetzner-webhook/values.yaml.gotmpl b/cert-manager-hetzner-webhook/values.yaml.gotmpl index dba1e72..7aac2c1 100644 --- a/cert-manager-hetzner-webhook/values.yaml.gotmpl +++ b/cert-manager-hetzner-webhook/values.yaml.gotmpl @@ -1,5 +1,12 @@ groupName: acme.hetzner.com +resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 64Mi + certManager: namespace: cert-manager serviceAccountName: cert-manager diff --git a/cert-manager/values.yaml.gotmpl b/cert-manager/values.yaml.gotmpl index b1b560a..d88eb9f 100644 --- a/cert-manager/values.yaml.gotmpl +++ b/cert-manager/values.yaml.gotmpl @@ -5,7 +5,6 @@ resources: cpu: 10m memory: 64Mi limits: - cpu: 1 memory: 128Mi webhook: @@ -14,7 +13,6 @@ webhook: cpu: 10m memory: 32Mi limits: - cpu: 1 memory: 64Mi cainjector: @@ -23,5 +21,4 @@ cainjector: cpu: 10m memory: 64Mi limits: - cpu: 1 memory: 128Mi diff --git a/gitea/values.yaml.gotmpl b/gitea/values.yaml.gotmpl index a338003..6daba01 100644 --- a/gitea/values.yaml.gotmpl +++ b/gitea/values.yaml.gotmpl @@ -41,6 +41,15 @@ gitea: strategy: type: Recreate +# Requests make the pod visible to the scheduler; without them gitea is +# BestEffort and is the first candidate for eviction under node pressure. +resources: + requests: + cpu: 50m + memory: 256Mi + limits: + memory: 1Gi + ingress: enabled: true hosts: @@ -61,6 +70,13 @@ service: valkey: enabled: true + primary: + resources: + requests: + cpu: 10m + memory: 48Mi + limits: + memory: 256Mi valkey-cluster: enabled: false postgresql: diff --git a/helmfile.d/03-apps.yaml b/helmfile.d/03-apps.yaml index 5daeb54..d59e5e2 100644 --- a/helmfile.d/03-apps.yaml +++ b/helmfile.d/03-apps.yaml @@ -104,4 +104,11 @@ releases: chart: incngrnt/k8up-backup createNamespace: true values: - - ../k8up-backup/values.yaml.gotmpl + - ../k8up-backup/values.yaml.gotmpl + + # Namespace guardrails (default requests/limits for pods that declare none). + # Listed last so the namespaces it targets already exist; it only affects pod + # admission from here on. Run `just update` to refresh the lock file. + - name: resource-defaults + namespace: default + chart: ../resource-defaults diff --git a/immich/values.yaml.gotmpl b/immich/values.yaml.gotmpl index 2ddb890..c1febae 100644 --- a/immich/values.yaml.gotmpl +++ b/immich/values.yaml.gotmpl @@ -17,19 +17,27 @@ immich: library: existingClaim: immich-data +# NOTE: immich 0.12.0 embeds valkey as a bjw-s "common" library template +# (templates/valkey.yaml), not as a Bitnami subchart. The old `valkey.master.*` +# block was silently dropped: it set a 32Mi limit under a 64Mi request (an +# impossible pair that would have OOMKilled the container) and never reached the +# pod. The correct path is controllers.main.containers.main.*. valkey: enabled: true - master: - persistence: + persistence: + data: annotations: k8up.io/backup: 'false' - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 1 - memory: 32Mi + controllers: + main: + containers: + main: + resources: + requests: + cpu: 10m + memory: 48Mi + limits: + memory: 256Mi server: enabled: true @@ -43,7 +51,8 @@ server: cpu: 10m memory: 256Mi limits: - cpu: 1 + # No CPU limit: limits do not change scheduler bin-packing and + # only cause CFS throttling. Memory limit is what protects the node. memory: 1Gi ingress: main: @@ -70,7 +79,6 @@ machine-learning: cpu: 10m memory: 128Mi limits: - cpu: 1 memory: 2Gi diff --git a/incngrnt-web/values.yaml.gotmpl b/incngrnt-web/values.yaml.gotmpl index 2e6a97d..3c54248 100644 --- a/incngrnt-web/values.yaml.gotmpl +++ b/incngrnt-web/values.yaml.gotmpl @@ -8,8 +8,9 @@ resources: cpu: 10m memory: 32Mi limits: - cpu: 500m - memory: 32Mi + # limit == request (32Mi) left no headroom for the nginx worker; CPU limit + # removed - it does not affect scheduling, only throttles. + memory: 96Mi ingress: enabled: true diff --git a/k8up-backup/values.yaml.gotmpl b/k8up-backup/values.yaml.gotmpl index 74b7359..db465fc 100644 --- a/k8up-backup/values.yaml.gotmpl +++ b/k8up-backup/values.yaml.gotmpl @@ -12,5 +12,4 @@ resources: cpu: 10m memory: 64Mi limits: - cpu: 500m - memory: 128Mi + memory: 256Mi diff --git a/k8up/values.yaml.gotmpl b/k8up/values.yaml.gotmpl index e1eb8c0..8d75589 100644 --- a/k8up/values.yaml.gotmpl +++ b/k8up/values.yaml.gotmpl @@ -11,5 +11,7 @@ k8up: cpu: 10m memory: 64Mi limits: - cpu: 500m - memory: 64Mi + # memory limit == request meant any spike (e.g. a reconcile of many + # Schedules) OOMKilled the controller. Leave headroom; CPU is uncapped + # because CPU limits do not affect scheduling and only throttle. + memory: 192Mi diff --git a/kgnot/values.yaml.gotmpl b/kgnot/values.yaml.gotmpl index 3ba518d..1964d50 100644 --- a/kgnot/values.yaml.gotmpl +++ b/kgnot/values.yaml.gotmpl @@ -15,11 +15,11 @@ readinessProbe: resources: requests: cpu: 10m - ephemeral-storage: 50Mi + # Ephemeral storage is intentionally NOT requested: with Talos's single-EPHM + # layout every request here competes for the same small filesystem that + # holds container images, and the LimitRange default covers us. memory: 64Mi limits: - cpu: 500m - ephemeral-storage: 2Gi memory: 350Mi persistence: diff --git a/mariadb/values.yaml.gotmpl b/mariadb/values.yaml.gotmpl index 954b0a5..a582d27 100644 --- a/mariadb/values.yaml.gotmpl +++ b/mariadb/values.yaml.gotmpl @@ -8,9 +8,9 @@ primary: resources: requests: cpu: 50m - ephemeral-storage: 50Mi memory: 96Mi limits: - cpu: 1 - ephemeral-storage: 2Gi - memory: 192Mi + # ephemeral-storage request removed - see ghost values note; the LimitRange + # default/max covers it. Memory limit raised: 192Mi was tight for a 5Gi + # dataset with the default key buffer + per-connection buffers. + memory: 512Mi diff --git a/matrix-registration/values.yaml.gotmpl b/matrix-registration/values.yaml.gotmpl index c80b57f..8166ed6 100644 --- a/matrix-registration/values.yaml.gotmpl +++ b/matrix-registration/values.yaml.gotmpl @@ -9,5 +9,6 @@ resources: cpu: 10m memory: 64Mi limits: - cpu: 500m - memory: 64Mi \ No newline at end of file + # memory limit was equal to the request (64Mi) - no headroom for the + # registration flow. CPU limit removed: it has no effect on scheduling. + memory: 192Mi \ No newline at end of file diff --git a/metallb/values.yaml.gotmpl b/metallb/values.yaml.gotmpl index fee90ef..2dddc27 100644 --- a/metallb/values.yaml.gotmpl +++ b/metallb/values.yaml.gotmpl @@ -4,8 +4,7 @@ controller: cpu: 10m memory: 32Mi limits: - cpu: 500m - memory: 64Mi + memory: 128Mi speaker: resources: @@ -13,5 +12,4 @@ speaker: cpu: 10m memory: 48Mi limits: - cpu: 500m memory: 96Mi diff --git a/postgres/operator-values.yaml.gotmpl b/postgres/operator-values.yaml.gotmpl index 2a29fbd..8454991 100644 --- a/postgres/operator-values.yaml.gotmpl +++ b/postgres/operator-values.yaml.gotmpl @@ -6,5 +6,4 @@ resources: cpu: 10m memory: 64Mi limits: - cpu: 500m memory: 160Mi diff --git a/postgres/values.yaml.gotmpl b/postgres/values.yaml.gotmpl index db164ca..8113ec3 100644 --- a/postgres/values.yaml.gotmpl +++ b/postgres/values.yaml.gotmpl @@ -9,7 +9,9 @@ instances: cpu: 100m memory: 192Mi limits: - cpu: 2 + # No CPU limit: the operator's instanceCPU: 2 is what the scheduler and + # the operator's own sizing logic use; a CFS limit here only throttles + # autovacuum and checkpointing without changing pod placement. memory: 1Gi dataVolumeClaimSpec: accessModes: diff --git a/rook-ceph-cluster/values.yaml.gotmpl b/rook-ceph-cluster/values.yaml.gotmpl index 6f817b1..ce0702c 100644 --- a/rook-ceph-cluster/values.yaml.gotmpl +++ b/rook-ceph-cluster/values.yaml.gotmpl @@ -6,34 +6,54 @@ cephClusterSpec: count: 3 allowMultiplePerNode: false + # A 3-node homelab does not need an active+standby mgr pair. Each mgr costs + # 256Mi request / 704Mi limit (plus mgr-sidecar), which is pure fixed overhead + # per node and the first thing that blocks co-scheduling on small nodes. + mgr: + count: 1 + storage: useAllNodes: true useAllDevices: false deviceFilter: "^sda" config: - osd_memory_target: "1073741824" # 1GB per OSD to maintain 70% node capacity + # osd_memory_target must match the osd request/limit below: the OSD is a + # long-running cache-filling process, so an under-sized request lets the + # scheduler overpack the node and an over-sized limit lets it OOM siblings. + osd_memory_target: "1073741824" # 1GB per OSD resources: mgr: requests: cpu: 50m memory: 256Mi limits: - cpu: "1" memory: 704Mi mon: requests: cpu: 100m memory: 256Mi limits: - cpu: "1" - memory: 2Gi + # 3 mons with ~zero load; the previous 2Gi limit was 8x the request and + # made the worst-case per-node footprint look 1.75Gi worse than it is. + memory: 1Gi osd: + # Guaranteed QoS (request == limit) so the scheduler reserves what the + # OSD will actually use. CPU is left unlimited: CPU limits don't change + # bin-packing and only throttle the OSD during backfill. requests: cpu: 100m - memory: 896Mi + memory: 1Gi limits: - cpu: "1" - memory: 1280Mi + memory: 1Gi + # crashcollector is a DaemonSet (one pod per node) and was previously + # inherited from the chart with a 100m CPU request, which is a large share + # of a ProDesk's 4 cores for a pod that only reads /var/lib/rook/crash. + crashcollector: + requests: + cpu: 10m + memory: 60Mi + limits: + memory: 60Mi ingress: dashboard: diff --git a/rook-ceph/values.yaml.gotmpl b/rook-ceph/values.yaml.gotmpl index e7a821e..e747087 100644 --- a/rook-ceph/values.yaml.gotmpl +++ b/rook-ceph/values.yaml.gotmpl @@ -1,7 +1,8 @@ resources: requests: - cpu: 100m - memory: 512Mi + cpu: 50m + # Operator is control-plane-ish: it mostly idles between reconcile loops. + # 512Mi of reservation was the single largest non-OSD request in the cluster. + memory: 128Mi limits: - cpu: 1 memory: 1Gi diff --git a/synapse/values.yaml.gotmpl b/synapse/values.yaml.gotmpl index 91d085a..2a0c401 100644 --- a/synapse/values.yaml.gotmpl +++ b/synapse/values.yaml.gotmpl @@ -18,8 +18,10 @@ synapse: cpu: 10m memory: 128Mi limits: - cpu: '1' - memory: 512Mi + # No CPU limit (throttling only, no scheduling effect). Synapse is + # memory-hungry on room joins/backfill; 512Mi was the tightest limit in + # the cluster for a stateful service. + memory: 1Gi config: macaroonSecretKey: {{ requiredEnv "GOATCHAT_SYNAPSE_MACAROON_SECRET_KEY" }} diff --git a/tailscale/connector.yaml b/tailscale/connector.yaml index f5e8cd4..172f2c6 100644 --- a/tailscale/connector.yaml +++ b/tailscale/connector.yaml @@ -3,6 +3,9 @@ kind: Connector metadata: name: home-cidr spec: + # Applies the resource requests/limits defined in proxyclass-home.yaml to the + # proxy StatefulSet this Connector creates. + proxyClass: fog tags: - "tag:k8s" hostname: home-cidr diff --git a/tailscale/values.yaml.gotmpl b/tailscale/values.yaml.gotmpl index 888437e..34dd1da 100644 --- a/tailscale/values.yaml.gotmpl +++ b/tailscale/values.yaml.gotmpl @@ -2,14 +2,18 @@ operatorConfig: extraEnv: - name: PROXY_PRIORITY_CLASS_NAME value: critical + resources: + requests: + cpu: 10m + memory: 48Mi + limits: + memory: 128Mi -resources: - requests: - cpu: 10m - memory: 48Mi - limits: - cpu: 500m - memory: 64Mi +# Make the fog ProxyClass the default for proxies created from Ingress and +# LoadBalancer services so they don't land as BestEffort pods. Connectors do not +# honour this and need spec.proxyClass (see connector.yaml). +proxyConfig: + defaultProxyClass: fog oauth: clientId: {{ requiredEnv "TAILSCALE_OAUTH_CLIENT_ID" }} diff --git a/talos/machine_patch.yaml b/talos/machine_patch.yaml index 6b2c6e5..7d7a44f 100644 --- a/talos/machine_patch.yaml +++ b/talos/machine_patch.yaml @@ -1,9 +1,29 @@ machine: install: - image: ghcr.io/siderolabs/installer:v1.13.0 + image: ghcr.io/siderolabs/installer:v1.13.12 kubelet: extraArgs: - rotate-server-certificates: true + rotate-server-certificates: true + # Reserve capacity so the scheduler, not kubelet's eviction manager, + # decides how many pods a node carries. Without this, kubelet advertises + # the node's full allocatable and only protects the OS/system pods once + # memory is already critical (default: memory.available < 100Mi), which + # on a ProDesk means OOM-killing workloads. With reservations the node + # reports a smaller allocatable, so pods pack to a sustainable density. + extraConfig: + maxPods: 110 + systemReserved: + cpu: 100m + memory: 256Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 250m + memory: 512Mi + ephemeral-storage: 2Gi + evictionHard: + memory.available: 200Mi + nodefs.available: 10% + imagefs.available: 15% udev: rules: # set all disks to `none` scheduler (optimal setting for Ceph and ZFS) diff --git a/traefik/values.yaml.gotmpl b/traefik/values.yaml.gotmpl index b454ea9..51df29b 100644 --- a/traefik/values.yaml.gotmpl +++ b/traefik/values.yaml.gotmpl @@ -6,8 +6,9 @@ resources: cpu: 50m memory: 64Mi limits: - cpu: 1 - memory: 128Mi + # No CPU limit: limits do not affect scheduler placement and only throttle + # the proxy under load. Memory is what protects the node. + memory: 256Mi updateStrategy: type: RollingUpdate