increase postgres memory

.gitignore

@@ -17,4 +17,4 @@ kgnot/config.production.json
 *.pub
 
 .envrc
-.kubeconfig
+.config

53ll/values.yaml.gotmpl

@@ -48,5 +48,21 @@ ingress:
   enabled: true
   hostname: 53ll.ca
   tls: true
-  annotations:
+  extraTls:
-    "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt" 
+    - secretName: 53ll-ca-tls
+      hosts:
+        - 53ll.ca
+
+extraDeploy:
+  - apiVersion: cert-manager.io/v1
+    kind: Certificate
+    metadata:
+      name: 53ll-ca
+      namespace: ghost
+    spec:
+      secretName: 53ll-ca-tls
+      issuerRef:
+        name: letsencrypt-53ll
+        kind: ClusterIssuer
+      dnsNames:
+        - 53ll.ca

cert-manager-hetzner-webhook/values.yaml.gotmpl

@@ -0,0 +1,11 @@
+groupName: acme.hetzner.com
+
+certManager:
+  namespace: cert-manager
+  serviceAccountName: cert-manager
+
+secretName:
+  - hetzner-dns-credentials
+
+secrets:
+  apiToken: {{ requiredEnv "HETZNER_API_KEY" }}

cert-manager-issuers/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: cert-manager-issuers
+description: cert-manager ClusterIssuers for Let's Encrypt
+type: application
+version: 0.1.0

cert-manager-issuers/templates/clusterissuers.yaml

@@ -0,0 +1,88 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hetzner-dns-credentials
+  namespace: cert-manager
+type: Opaque
+stringData:
+  api-key: {{ .Values.hetzner.apiToken }}
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-incngrnt
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: {{ .Values.acme.email }}
+    privateKeySecretRef:
+      name: letsencrypt-incngrnt-private-key
+    solvers:
+      - dns01:
+          webhook:
+            groupName: acme.hetzner.com
+            solverName: hetzner
+            config:
+              secretName: hetzner-dns-credentials
+              zoneName: incngrnt.ca
+              apiUrl: https://dns.hetzner.com/api/v1
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-goatchat
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: {{ .Values.acme.email }}
+    privateKeySecretRef:
+      name: letsencrypt-goatchat-private-key
+    solvers:
+      - dns01:
+          webhook:
+            groupName: acme.hetzner.com
+            solverName: hetzner
+            config:
+              secretName: hetzner-dns-credentials
+              zoneName: goatchat.ca
+              apiUrl: https://dns.hetzner.com/api/v1
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-53ll
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: {{ .Values.acme.email }}
+    privateKeySecretRef:
+      name: letsencrypt-53ll-private-key
+    solvers:
+      - dns01:
+          webhook:
+            groupName: acme.hetzner.com
+            solverName: hetzner
+            config:
+              secretName: hetzner-dns-credentials
+              zoneName: 53ll.ca
+              apiUrl: https://dns.hetzner.com/api/v1
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-kgnot
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: {{ .Values.acme.email }}
+    privateKeySecretRef:
+      name: letsencrypt-kgnot-private-key
+    solvers:
+      - dns01:
+          webhook:
+            groupName: acme.hetzner.com
+            solverName: hetzner
+            config:
+              secretName: hetzner-dns-credentials
+              zoneName: kgnot.ca
+              apiUrl: https://dns.hetzner.com/api/v1

cert-manager-issuers/values.yaml.gotmpl

@@ -0,0 +1,5 @@
+hetzner:
+  apiToken: {{ requiredEnv "HETZNER_API_KEY" }}
+
+acme:
+  email: {{ requiredEnv "ACME_EMAIL" }}

cert-manager/values.yaml.gotmpl

@@ -0,0 +1,27 @@
+installCRDs: true
+
+resources:
+  requests:
+    cpu: 10m
+    memory: 64Mi
+  limits:
+    cpu: 1
+    memory: 128Mi
+
+webhook:
+  resources:
+    requests:
+      cpu: 10m
+      memory: 32Mi
+    limits:
+      cpu: 1
+      memory: 64Mi
+
+cainjector:
+  resources:
+    requests:
+      cpu: 10m
+      memory: 64Mi
+    limits:
+      cpu: 1
+      memory: 128Mi

gitea/values.yaml.gotmpl

@@ -48,8 +48,10 @@ ingress:
       paths:
         - path: "/"
           pathType: Prefix
-  annotations:
+  tls:
-    "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt"
+    - secretName: git-incngrnt-ca-tls
+      hosts:
+        - git.incngrnt.ca
 
 service:
   ssh:
@@ -57,18 +59,9 @@ service:
     port: 22
     clusterIP:
 
-actions:
+valkey:
   enabled: true
-  giteaRootURL: https://git.incngrnt.ca
+valkey-cluster:
-  existingSecret: gitea-runner-token
-  existingSecretKey: token
-  provisioning:
-    enabled: false
-  persistence:
-    enabled: false
-redis:
-  enabled: true
-redis-cluster:
   enabled: false
 postgresql:
   enabled: false
@@ -77,6 +70,18 @@ postgresql-ha:
 
 
 extraDeploy:
+- apiVersion: cert-manager.io/v1
+  kind: Certificate
+  metadata:
+    name: git-incngrnt-ca
+    namespace: gitea
+  spec:
+    secretName: git-incngrnt-ca-tls
+    issuerRef:
+      name: letsencrypt-incngrnt
+      kind: ClusterIssuer
+    dnsNames:
+      - git.incngrnt.ca
 - apiVersion: traefik.io/v1alpha1
   kind: IngressRouteTCP
   metadata:

helmfile.d/01-infrastructure.lock

@@ -1,16 +1,22 @@
-version: 0.170.1
+version: 1.2.3
 dependencies:
-- name: rook-ceph
+  - name: cert-manager
-  repository: https://charts.rook.io/release
+    repository: https://charts.jetstack.io
-  version: v1.18.8
+    version: v1.19.2
-- name: rook-ceph-cluster
+  - name: cert-manager-webhook-hetzner
-  repository: https://charts.rook.io/release
+    repository: https://vadimkim.github.io/cert-manager-webhook-hetzner
-  version: v1.18.8
+    version: 1.4.2
-- name: tailscale-operator
+  - name: rook-ceph
-  repository: https://pkgs.tailscale.com/helmcharts
+    repository: https://charts.rook.io/release
-  version: 1.90.9
+    version: v1.18.8
-- name: traefik
+  - name: rook-ceph-cluster
-  repository: https://traefik.github.io/charts
+    repository: https://charts.rook.io/release
-  version: 37.4.0
+    version: v1.18.8
-digest: sha256:66b2e4b590af3ee51f97d61435400977ceb8d70ddfc50d638ccfaeede79e8a6f
+  - name: tailscale-operator
-generated: "2025-12-07T13:19:28.002423348-07:00"
+    repository: https://pkgs.tailscale.com/helmcharts
+    version: 1.92.4
+  - name: traefik
+    repository: https://traefik.github.io/charts
+    version: 38.0.1
+digest: sha256:3297bc0c10765abe170881882f7daf441a4dd735ed0ee7d1f4233692e8888c3c
+generated: "2025-12-23T10:31:34.409765694-07:00"

helmfile.d/01-infrastructure.yaml

@@ -5,7 +5,12 @@ repositories:
     url: https://traefik.github.io/charts
   - name: tailscale
     url: https://pkgs.tailscale.com/helmcharts
+  - name: jetstack
+    url: https://charts.jetstack.io
+  - name: cert-manager-webhook-hetzner
+    url: https://vadimkim.github.io/cert-manager-webhook-hetzner
 
+lockFilePath: ./helmfile.d/01-infrastructure.lock
 releases:
   # networking
   - name: metallb
@@ -14,6 +19,24 @@ releases:
     chart: ../metallb
     values:
       - ../metallb/values.yaml.gotmpl
+  - name: cert-manager
+    namespace: cert-manager
+    createNamespace: true
+    chart: jetstack/cert-manager
+    values:
+      - ../cert-manager/values.yaml.gotmpl
+  - name: cert-manager-webhook-hetzner
+    namespace: cert-manager
+    createNamespace: true
+    chart: cert-manager-webhook-hetzner/cert-manager-webhook-hetzner
+    values:
+      - ../cert-manager-hetzner-webhook/values.yaml.gotmpl
+  - name: cert-manager-issuers
+    namespace: cert-manager
+    createNamespace: true
+    chart: ../cert-manager-issuers
+    values:
+      - ../cert-manager-issuers/values.yaml.gotmpl    
   - name: traefik
     namespace: traefik
     createNamespace: true

helmfile.d/02-datastore.lock

@@ -1,16 +1,16 @@
-version: 0.170.1
+version: 1.2.3
 dependencies:
-- name: k8up
+  - name: k8up
-  repository: https://k8up-io.github.io/k8up
+    repository: https://k8up-io.github.io/k8up
-  version: 4.8.6
+    version: 4.8.6
-- name: mariadb
+  - name: mariadb
-  repository: https://charts.bitnami.com/bitnami
+    repository: https://charts.bitnami.com/bitnami
-  version: 24.0.0
+    version: 24.0.2
-- name: pgo
+  - name: pgo
-  repository: git+https://github.com/grantdhunter/postgres-operator@helm?ref=main
+    repository: git+https://github.com/grantdhunter/postgres-operator@helm?ref=main
-  version: 5.8.1
+    version: 5.8.1
-- name: postgrescluster
+  - name: postgrescluster
-  repository: git+https://github.com/grantdhunter/postgres-operator@helm?ref=main
+    repository: git+https://github.com/grantdhunter/postgres-operator@helm?ref=main
-  version: 5.7.4
+    version: 5.7.4
-digest: sha256:cd960bd2adfc6d5bbfadd4d8ba745904717ba888da8dee9cde7c83ba71e5f8a4
+digest: sha256:f8989df670b3574b6d87438486b66fdaf44bc1ed379d3a98e00963a27703003a
-generated: "2025-12-07T13:19:41.655599535-07:00"
+generated: "2025-12-14T14:14:05.060998516-07:00"

helmfile.d/02-datastore.yaml

@@ -6,6 +6,7 @@ repositories:
   - name: k8up-io
     url: https://k8up-io.github.io/k8up
 
+lockFilePath: ./helmfile.d/02-datastore.lock
 releases:
   # data storage
   - name: pgo

helmfile.d/03-apps.lock

@@ -1,34 +1,37 @@
-version: 0.170.1
+version: 1.2.3
 dependencies:
-- name: ghost
+  - name: ghost
-  repository: https://charts.bitnami.com/bitnami
+    repository: https://charts.bitnami.com/bitnami
-  version: 25.0.4
+    version: 25.0.4
-- name: ghost
+  - name: ghost
-  repository: https://charts.bitnami.com/bitnami
+    repository: https://charts.bitnami.com/bitnami
-  version: 25.0.4
+    version: 25.0.4
-- name: immich
+  - name: gitea
-  repository: https://immich-app.github.io/immich-charts
+    repository: https://dl.gitea.io/charts
-  version: 0.10.3
+    version: 12.4.0
-- name: k8up-backup
+  - name: immich
-  repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
+    repository: https://immich-app.github.io/immich-charts
-  version: 0.0.3
+    version: 0.10.3
-- name: k8up-backup
+  - name: k8up-backup
-  repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
+    repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
-  version: 0.0.3
+    version: 0.0.3
-- name: k8up-backup
+  - name: k8up-backup
-  repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
+    repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
-  version: 0.0.3
+    version: 0.0.3
-- name: k8up-backup
+  - name: k8up-backup
-  repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
+    repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
-  version: 0.0.3
+    version: 0.0.3
-- name: matrix-registration
+  - name: k8up-backup
-  repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
+    repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
-  version: 0.1.0
+    version: 0.0.3
-- name: matrix-synapse
+  - name: matrix-registration
-  repository: https://ananace.gitlab.io/charts
+    repository: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
-  version: 3.12.16
+    version: 0.1.0
-- name: static-site
+  - name: matrix-synapse
-  repository: git+https://github.com/cfpb/static-site@charts?ref=main
+    repository: https://ananace.gitlab.io/charts
-  version: 0.1.1
+    version: 3.12.17
-digest: sha256:59866b3b160d35756885a2db0a3344bba48161e5ba6935350286f9a754b8b219
+  - name: static-site
-generated: "2025-11-25T20:31:24.531424306-07:00"
+    repository: git+https://github.com/cfpb/static-site@charts?ref=main
+    version: 0.1.1
+digest: sha256:b44d082b71203ca6bb4fd881d8c6ce71575db556f432bbcc46078a535c8cd9c3
+generated: "2025-12-23T10:31:37.404126839-07:00"

helmfile.d/03-apps.yaml

@@ -12,6 +12,7 @@ repositories:
   - name: incngrnt
     url: git+https://git.incngrnt.ca/grant/charts@charts?ref=main
 
+lockFilePath: ./helmfile.d/03-apps.lock
 releases:
   # goatchat matrix
   - name: goatchat
@@ -56,12 +57,12 @@ releases:
       - ../53ll/values.yaml.gotmpl
 
   # dev tools
-  # - name: gitea
+  - name: gitea
-  #   namespace: gitea
+    namespace: gitea
-  #   createNamespace: true
+    createNamespace: true
-  #   chart: gitea/gitea
+    chart: gitea/gitea
-  #   values:
+    values:
-  #     - ../gitea/values.yaml.gotmpl
+      - ../gitea/values.yaml.gotmpl
 
   # backups
   - name: ghost-backup

immich/values.yaml.gotmpl

@@ -3,7 +3,7 @@ controllers:
     containers:
       main:
         image:
-          tag: v2.3.1
+          tag: v2.4.1
 
         env:
           DB_HOSTNAME: {{ exec "kubectl" (list "-n" "immich" "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.host | base64decode }}'") }}
@@ -44,12 +44,14 @@ server:
               memory: 256Mi
             limits:
               cpu: 1
-              memory: 512Mi
+              memory: 700Mi
   ingress:
     main:
       enabled: true
-      annotations:
+      tls:
-        traefik.ingress.kubernetes.io/router.tls.certresolver: "letsencrypt"
+        - secretName: photos-incngrnt-ca-tls
+          hosts:
+            - photos.incngrnt.ca
       hosts:
         - host: photos.incngrnt.ca
           paths:
@@ -69,4 +71,6 @@ machine-learning:
               memory: 128Mi
             limits:
               cpu: 1
-              memory: 384Mi
+              memory: 1Gi
+
+

incngrnt-web/values.yaml.gotmpl

@@ -13,10 +13,13 @@ resources:
 
 ingress:
   enabled: true
-  annotations:
+  tls:
-    "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt"  
+    - secretName: incngrnt-ca-tls
+      hosts:
+        - incngrnt.ca
   hosts:
     - host: incngrnt.ca
       paths:
         - path: /
           pathType: ImplementationSpecific
+

justfile

@@ -1,11 +1,11 @@
 update:
     bws run 'helmfile deps'
 
-deploy ARGS='--output simple -i':
+deploy ARGS='':
-    bws run 'helmfile apply {{ARGS}}'
+    bws run 'helmfile apply --output simple --skip-deps {{ARGS}}'
 
 diff ARGS='':
-    bws run 'helmfile diff --output dyff {{ARGS}}'
+    bws run 'helmfile diff --output dyff  --skip-deps {{ARGS}}'
 
 cleanuppods:
     #!/bin/bash
@@ -38,3 +38,17 @@ goatchat-register-review:
     bws run 'curl -v -H '\"'Authorization: SharedSecret $GOATCHAT_REGISTRATION_ADMIN_API_SHARE_SECRET'\"' \
         -H "Content-Type: application/json" \
         https://goatchat.ca/gate/api/token' | jq
+
+refresh-client-cert:
+    #!/bin/bash
+    yq -r .machine.ca.crt controlplane.yaml | base64 -d > ca.crt
+    yq -r .machine.ca.key controlplane.yaml | base64 -d > ca.key
+    talosctl gen key --name admin           
+    talosctl gen csr --key admin.key --ip 127.0.0.1
+    talosctl gen crt --ca ca --csr admin.csr --name admin
+    yq -i '.contexts.fog.ca = "'"$(base64 -w0 ca.crt)"\
+    '" | .contexts.fog.crt = "'"$(base64 -w0 admin.crt)"\
+    '" | .contexts.fog.key = "'"$(base64 -w0 admin.key)"'"' \
+    .config/talosconfig
+
+    talosctl kubeconfig .config/kubeconfig -n 192.168.1.43

kgnot/values.yaml.gotmpl

@@ -20,7 +20,7 @@ resources:
   limits:
     cpu: 500m
     ephemeral-storage: 2Gi
-    memory: 224Mi
+    memory: 350Mi
 
 persistence:
   size: 1Gi
@@ -50,5 +50,21 @@ ingress:
   enabled: true
   hostname: kgnot.ca
   tls: true
-  annotations:
+  extraTls:
-    "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt" 
+    - secretName: kgnot-ca-tls
+      hosts:
+        - kgnot.ca
+
+extraDeploy:
+  - apiVersion: cert-manager.io/v1
+    kind: Certificate
+    metadata:
+      name: kgnot-ca
+      namespace: ghost
+    spec:
+      secretName: kgnot-ca-tls
+      issuerRef:
+        name: letsencrypt-kgnot
+        kind: ClusterIssuer
+      dnsNames:
+        - kgnot.ca

postgres/values.yaml.gotmpl

@@ -9,7 +9,7 @@ instances:
         memory: 192Mi
       limits:
         cpu: 2
-        memory: 256Mi
+        memory: 1Gi
     dataVolumeClaimSpec:
       accessModes:
       - "ReadWriteOnce"

rook-ceph-cluster/values.yaml.gotmpl

@@ -37,10 +37,27 @@ ingress:
         name: fog.incngrnt.ca
         path: /fog/ceph
         pathType: Prefix
+      tls:
+        - secretName: fog-incngrnt-ca-tls
+          hosts:
+            - fog.incngrnt.ca
       annotations:
-        "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt"
         "traefik.ingress.kubernetes.io/router.middlewares": "rook-ceph-ceph-stripprefix@kubernetescrd"
 
+extraDeploy:
+  - apiVersion: cert-manager.io/v1
+    kind: Certificate
+    metadata:
+      name: fog-ceph-incngrnt-ca
+      namespace: rook-ceph
+    spec:
+      secretName: fog-incngrnt-ca-tls
+      issuerRef:
+        name: letsencrypt-incngrnt
+        kind: ClusterIssuer
+      dnsNames:
+        - fog.incngrnt.ca
+
 cephFileSystems:
   - name: ceph-filesystem
     # see https://github.com/rook/rook/blob/master/Documentation/ceph-filesystem-crd.md#filesystem-settings for available configuration

synapse/values.yaml.gotmpl

@@ -19,7 +19,7 @@ synapse:
       memory: 128Mi
     limits:
       cpu: '1'
-      memory: 192Mi
+      memory: 256Mi
 
 config:
   macaroonSecretKey: {{ requiredEnv "GOATCHAT_SYNAPSE_MACAROON_SECRET_KEY" }}
@@ -69,8 +69,10 @@ extraConfig:
 
 ingress:
   traefikPaths: true
-  annotations:
+  tls:
-    "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt"
+    - secretName: goatchat-ca-tls
+      hosts:
+        - goatchat.ca
 
 
 persistence:

traefik/values.yaml.gotmpl

@@ -1,11 +1,5 @@
 deployment:
-  initContainers:
+  replicas: 2
-    - name: volume-permissions
-      image: busybox:latest
-      command: ["sh", "-c", "touch /data/acme.json; chmod -v 600 /data/acme.json"]
-      volumeMounts:
-        - name: data
-          mountPath: /data
 
 resources:
   requests:
@@ -16,20 +10,33 @@ resources:
     memory: 128Mi
 
 updateStrategy:
-  type: Recreate
+  type: RollingUpdate
+  rollingUpdate:
+    maxUnavailable: 1
+    maxSurge: 1
 
-env:
+podDisruptionBudget:
- - name: HETZNER_API_KEY
+  enabled: true
-   valueFrom:
+  minAvailable: 1
-     secretKeyRef:
+
-       name: hetzner-api-key
+affinity:
-       key: token
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+              - key: app.kubernetes.io/name
+                operator: In
+                values:
+                  - traefik
+          topologyKey: kubernetes.io/hostname
     
 additionalArguments:
  - "--api.basePath=/fog/traefik"
 
 persistence:
-  enabled: true
+  enabled: false
     
 logs:
   format: json    
@@ -49,7 +56,7 @@ ingressRoute:
     middlewares:
       - name: traefik-dashboard-auth
     tls:
-      certResolver: letsencrypt
+      secretName: fog-incngrnt-ca-tls
 
 ports:
   websecure:
@@ -99,12 +106,53 @@ extraObjects:
       redirectScheme:
         scheme: https
         permanent: true
+  - apiVersion: cert-manager.io/v1
+    kind: Certificate
+    metadata:
+      name: fog-incngrnt-ca
+      namespace: traefik
+    spec:
+      secretName: fog-incngrnt-ca-tls
+      issuerRef:
+        name: letsencrypt-incngrnt
+        kind: ClusterIssuer
+      dnsNames:
+        - fog.incngrnt.ca
 
-certificatesResolvers:
+# other certs
-  letsencrypt:
+  - apiVersion: cert-manager.io/v1
-    acme:
+    kind: Certificate
-      dnschallenge:
+    metadata:
-        provider: hetzner
+      name: goatchat-ca
-        delaybeforecheck: 30
+      namespace: goatchat
-      email: {{ requiredEnv "ACME_EMAIL" }}
+    spec:
-      storage: /data/acme.json 
+      secretName: goatchat-ca-tls
+      issuerRef:
+        name: letsencrypt-goatchat
+        kind: ClusterIssuer
+      dnsNames:
+        - goatchat.ca
+  - apiVersion: cert-manager.io/v1
+    kind: Certificate
+    metadata:
+      name: incngrnt-ca
+      namespace: incngrnt-web
+    spec:
+      secretName: incngrnt-ca-tls
+      issuerRef:
+        name: letsencrypt-incngrnt
+        kind: ClusterIssuer
+      dnsNames:
+        - incngrnt.ca
+  - apiVersion: cert-manager.io/v1
+    kind: Certificate
+    metadata:
+      name: photos-incngrnt-ca
+      namespace: immich
+    spec:
+      secretName: photos-incngrnt-ca-tls
+      issuerRef:
+        name: letsencrypt-incngrnt
+        kind: ClusterIssuer
+      dnsNames:
+        - photos.incngrnt.ca