Add monitoring stack

.gitignore

@@ -13,3 +13,5 @@ talos/secrets.yaml
 kgnot/config.production.json
 53ll/config.production.json
 
+*.key
+*.pub

grafana/alloy_values.yaml

@@ -0,0 +1,23 @@
+alloy:
+  configMap:
+    content: |-
+      logging {
+      	level  = "info"
+      	format = "logfmt"
+      }
+      
+      discovery.kubernetes "pods" {
+        role = "pod"
+      }
+      
+      loki.source.kubernetes "pods" {
+        targets    = discovery.kubernetes.pods.targets
+        forward_to = [loki.write.loki.receiver]
+
+      }
+      
+      loki.write "loki" {
+         endpoint {
+           url = "http://loki.grafana.svc.cluster.local:3100/loki/api/v1/push"
+         }
+       }

grafana/grafana_values.yaml

@@ -0,0 +1,9 @@
+ingress:
+  enabled: true
+  hosts:
+    - watcher.incngrnt.ca
+  annotations:
+    "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt"
+
+persistence:
+  enabled: true    

grafana/loki_values.yaml

@@ -0,0 +1,70 @@
+loki:
+  commonConfig:
+    replication_factor: 1
+  schemaConfig:
+    configs:
+      - from: "2024-04-01"
+        store: tsdb
+        object_store: s3
+        schema: v13
+        index:
+          prefix: loki_index_
+          period: 24h
+  pattern_ingester:
+      enabled: true
+  limits_config:
+    allow_structured_metadata: true
+    volume_enabled: true
+  ruler:
+    enable_api: true
+  auth_enabled: false
+
+resultsCache:
+  resources:
+    request:
+      cpu: 100ms
+      memory: 500Mi
+    limits:
+      memory: 500Mi
+chunksCache:
+  resources:
+    request:
+      cpu: 100ms
+      memory: 500Mi
+    limits:
+      memory: 500Mi
+minio:
+  enabled: true
+
+      
+deploymentMode: SingleBinary
+
+singleBinary:
+  replicas: 1
+
+# Zero out replica counts of other deployment modes
+backend:
+  replicas: 0
+read:
+  replicas: 0
+write:
+  replicas: 0
+
+ingester:
+  replicas: 0
+querier:
+  replicas: 0
+queryFrontend:
+  replicas: 0
+queryScheduler:
+  replicas: 0
+distributor:
+  replicas: 0
+compactor:
+  replicas: 0
+indexGateway:
+  replicas: 0
+bloomCompactor:
+  replicas: 0
+bloomGateway:
+  replicas: 0

grafana/prometheus_values.yaml

@@ -0,0 +1,3 @@
+rometheus-node-exporter:
+  rbac:
+    pspEnabled: true

helmfile.lock

@@ -1,17 +1,26 @@
 version: 0.170.1
 dependencies:
+- name: alloy
+  repository: https://grafana.github.io/helm-charts
+  version: 0.12.5
 - name: ghost
   repository: https://charts.bitnami.com/bitnami
-  version: 22.1.19
+  version: 22.2.0
 - name: ghost
   repository: https://charts.bitnami.com/bitnami
-  version: 22.1.19
+  version: 22.2.0
 - name: gitea
   repository: https://dl.gitea.io/charts
   version: 11.0.0
+- name: grafana
+  repository: https://grafana.github.io/helm-charts
+  version: 8.10.3
 - name: k8up
   repository: https://k8up-io.github.io/k8up
   version: 4.8.4
+- name: loki
+  repository: https://grafana.github.io/helm-charts
+  version: 6.28.0
 - name: mariadb
   repository: https://charts.bitnami.com/bitnami
   version: 20.4.1
@@ -24,6 +33,9 @@ dependencies:
 - name: postgrescluster
   repository: git+https://github.com/grantdhunter/postgres-operator@helm?ref=main
   version: 5.7.2
+- name: prometheus
+  repository: https://prometheus-community.github.io/helm-charts
+  version: 27.5.1
 - name: rook-ceph
   repository: https://charts.rook.io/release
   version: v1.16.5
@@ -36,5 +48,5 @@ dependencies:
 - name: traefik
   repository: https://traefik.github.io/charts
   version: 34.4.1
-digest: sha256:14c9bc504c5060f8bbce5ec9a8df737a19a7216428a31c1cb34ef5c6648e45c5
-generated: "2025-03-12T21:27:22.529913117-06:00"
+digest: sha256:b28767f0ec4d8549e0b1de7446f3468555a2a67bf88d2b554e9a12c2de723d2d
+generated: "2025-03-15T14:34:49.001292746-06:00"

helmfile.yaml

@@ -19,6 +19,10 @@ repositories:
     url: https://pkgs.tailscale.com/helmcharts
   - name: gitea
     url: https://dl.gitea.io/charts
+  - name: grafana
+    url: https://grafana.github.io/helm-charts
+  - name: prometheus-community
+    url: https://prometheus-community.github.io/helm-charts
 releases:
   # networking
   - name: metallb
@@ -52,6 +56,8 @@ releases:
     namespace: rook-ceph
     createNamespace: true
     chart: rook-release/rook-ceph
+    values:
+      - ./rook-ceph/values.yaml
   - name: rook-ceph-cluster
     namespace: rook-ceph
     createNamespace: true
@@ -83,6 +89,36 @@ releases:
     setString:
       - name: auth.rootPassword
         value: {{ requiredEnv "MARIADB_ROOT_PASSWORD" }}
+
+# monitoring
+  - name: grafana
+    namespace: grafana
+    createNamespace: true
+    chart: grafana/grafana
+    values:
+      - grafana/grafana_values.yaml
+    setString:
+      - name: adminPassword
+        value: VYHEKk0Q9KfqQ3UpTx8oc4InrXlUQivUuEeGU8LJ
+  - name: prometheus
+    namespace: grafana
+    createNamespace: true
+    chart: prometheus-community/prometheus
+    values:
+      - grafana/prometheus_values.yaml
+  - name: loki
+    namespace: grafana
+    createNamespace: true
+    chart: grafana/loki
+    values:
+      - grafana/loki_values.yaml
+  - name: alloy
+    namespace: grafana
+    createNamespace: true
+    chart: grafana/alloy
+    values:
+      - grafana/alloy_values.yaml
+      
   # goatchat matrix
   - name: goatchat
     namespace: goatchat
@@ -172,3 +208,17 @@ releases:
       - name: repoPassword
         value:  {{ requiredEnv "k8UP_REPO_PASSWORD" }}
  
+  - name: gitea-backup
+    namespace: gitea
+    chart: ./k8up-backup
+    createNamespace: true    
+    values:
+      - ./k8up-backup/values_override.yaml
+    setString:
+      - name: credentials.id
+        value:  {{ requiredEnv "HETZNER_S3_ACCESS_KEY" }}
+      - name: credentials.key
+        value:  {{ requiredEnv "HETZNER_S3_ACCESS_SECRET" }}
+      - name: repoPassword
+        value:  {{ requiredEnv "k8UP_REPO_PASSWORD" }}
+ 

rook-ceph/values.yaml

@@ -0,0 +1,4 @@
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi

traefik/values.yaml

@@ -19,6 +19,12 @@ additionalArguments:
 
 persistence:
   enabled: true
+    
+logs:
+  format: json    
+  access:
+    enabled: true
+    format: json    
 
 ingressRoute:
   dashboard: