Fix monitoring, remove limits from immich

grafana/alloy_values.yaml

@@ -9,15 +9,121 @@ alloy:
       discovery.kubernetes "pods" {
         role = "pod"
       }
+      discovery.kubernetes "nodes" {
+        role = "node"
+      }
       
-      loki.source.kubernetes "pods" {
+      discovery.relabel "pods" {
         targets = discovery.kubernetes.pods.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_container_name"]
+          separator     = "/"
+          target_label  = "deployment_name"
+          action        = "replace"
+        }
+      }
+      loki.source.kubernetes "pods" {
+        targets    = discovery.relabel.pods.output
+        forward_to = [loki.process.process.receiver]
+      }
+      loki.process "process" {
         forward_to = [loki.write.loki.receiver]
 
+        stage.drop {
+          older_than          = "1h"
+          drop_counter_reason = "too old"
+        }
+        stage.match { 
+          selector = "{instance=~\".*\"}"
+          stage.json {
+            expressions = {
+              level = "\"level\"",
+            }
+          }
+          stage.labels {
+            values = { 
+              level = "level",
+            }
+          }
+        }
+        stage.label_drop {
+          values = [ "job", "service_name" ]
+        }
       }      
-      
       loki.write "loki" {
          endpoint {
-           url = "http://loki.grafana.svc.cluster.local:3100/loki/api/v1/push"
+           url = "http://grafana-loki-distributor:3100/loki/api/v1/push"
+         }
+       }
+       
+      discovery.relabel "metrics" {
+        targets = discovery.kubernetes.pods.targets
+        rule {
+          source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port"]
+          target_label  = "__meta_kubernetes_pod_container_port_number"
+          action = "keepequal"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_container_port_number"]
+          regex = ""
+          action = "drop"
+        }    
+        rule {
+          source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path",]
+          target_label  = "__metrics_path__"
+          separator = ""
+          action = "replace"
+        }        
+      }
+      prometheus.scrape "metrics" {
+        clustering {
+            enabled = true
+        }
+        targets    = discovery.relabel.metrics.output
+        forward_to = [prometheus.remote_write.metrics.receiver]
+        scrape_interval = "30s"
+      }
+      discovery.relabel "pods_metrics" {
+        targets = discovery.kubernetes.nodes.targets
+        rule {
+          replacement  = "kubernetes.default.svc:443"
+          target_label = "__address__"
+        }
+        rule {
+          regex         = "(.+)"
+          replacement   = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
+          source_labels = ["__meta_kubernetes_node_name"]
+          target_label  = "__metrics_path__"
+        }
+      }
+      prometheus.scrape "pods_metrics" {
+        clustering {
+            enabled = true
+        }
+        targets      = discovery.relabel.pods_metrics.output
+        job_name     = "integrations/kubernetes/kubelet"
+        scheme       = "https"
+        honor_labels = true
+        forward_to = [prometheus.remote_write.metrics.receiver]
+        bearer_token_file = "/run/secrets/kubernetes.io/serviceaccount/token"
+        tls_config {
+          insecure_skip_verify = true
+          server_name          = "kubernetes"
+        }
+        scrape_interval = "30s"
+      }
+      prometheus.exporter.unix "os_metrics" { }
+      prometheus.scrape "os_metrics" {
+        clustering {
+            enabled = true
+        }
+        targets    = prometheus.exporter.unix.os_metrics.targets
+        forward_to = [prometheus.remote_write.metrics.receiver]
+        scrape_interval = "30s"
+      }
+      prometheus.remote_write "metrics" {
+        endpoint {
+          url = "http://grafana-mimir-nginx/api/v1/push"
         }
       }

grafana/values.yaml

@@ -0,0 +1,8 @@
+grafana:
+  ingress:
+    enabled: true
+    hosts:
+      - watcher.incngrnt.ca
+    annotations:
+      "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt"
+

helmfile.lock

@@ -5,25 +5,22 @@ dependencies:
   version: 0.12.5
 - name: ghost
   repository: https://charts.bitnami.com/bitnami
-  version: 22.2.0
+  version: 22.2.1
 - name: ghost
   repository: https://charts.bitnami.com/bitnami
-  version: 22.2.0
+  version: 22.2.1
 - name: gitea
   repository: https://dl.gitea.io/charts
   version: 11.0.0
-- name: grafana
-  repository: https://grafana.github.io/helm-charts
-  version: 8.10.4
 - name: immich
   repository: https://immich-app.github.io/immich-charts
   version: 0.9.1
 - name: k8up
   repository: https://k8up-io.github.io/k8up
   version: 4.8.4
-- name: loki
+- name: lgtm-distributed
   repository: https://grafana.github.io/helm-charts
-  version: 6.28.0
+  version: 2.1.0
 - name: mariadb
   repository: https://charts.bitnami.com/bitnami
   version: 20.4.1
@@ -36,9 +33,6 @@ dependencies:
 - name: postgrescluster
   repository: git+https://github.com/grantdhunter/postgres-operator@helm?ref=main
   version: 5.7.2
-- name: prometheus
-  repository: https://prometheus-community.github.io/helm-charts
-  version: 27.5.1
 - name: rook-ceph
   repository: https://charts.rook.io/release
   version: v1.16.5
@@ -54,5 +48,5 @@ dependencies:
 - name: traefik
   repository: https://traefik.github.io/charts
   version: 34.4.1
-digest: sha256:d9f95a5155811ec4c166a0d351902b6a4d69cecbb9e466aca3b31721fd0fa7c9
+digest: sha256:e2c65297653c325106e0a9055ed7a59f9c801035089e1684934ceef50a032ac8
-generated: "2025-03-16T22:05:07.757175164-06:00"
+generated: "2025-03-17T21:13:03.161466198-06:00"

helmfile.yaml

@@ -97,24 +97,9 @@ releases:
   - name: grafana
     namespace: grafana
     createNamespace: true
-    chart: grafana/grafana
+    chart: grafana/lgtm-distributed
     values:
-      - grafana/grafana_values.yaml
+      - grafana/values.yaml
-    setString:
-      - name: adminPassword
-        value: VYHEKk0Q9KfqQ3UpTx8oc4InrXlUQivUuEeGU8LJ
-  - name: prometheus
-    namespace: grafana
-    createNamespace: true
-    chart: prometheus-community/prometheus
-    values:
-      - grafana/prometheus_values.yaml
-  - name: loki
-    namespace: grafana
-    createNamespace: true
-    chart: grafana/loki
-    values:
-      - grafana/loki_values.yaml
   - name: alloy
     namespace: grafana
     createNamespace: true

immich/values.yaml.gotmpl

@@ -1,8 +1,8 @@
 env:
-  DB_HOSTNAME: {{ exec "kubectl" (list "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.host | base64decode }}'") }}
+  DB_HOSTNAME: {{ exec "kubectl" (list "-n" "immich" "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.host | base64decode }}'") }}
-  DB_USERNAME: {{ exec "kubectl" (list "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.user | base64decode }}'") }}
+  DB_USERNAME: {{ exec "kubectl" (list "-n" "immich" "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.user | base64decode }}'") }}
-  DB_DATABASE_NAME: {{ exec "kubectl" (list "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.dbname | base64decode }}'") }}
+  DB_DATABASE_NAME: {{ exec "kubectl" (list "-n" "immich" "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.dbname | base64decode }}'") }}
-  DB_PASSWORD: {{ exec "kubectl" (list "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.password | base64decode }}'") }}
+  DB_PASSWORD: {{ exec "kubectl" (list "-n" "immich" "get" "secrets" "postgres-pguser-immich" "-ogo-template='{{.data.password | base64decode }}'") }}
   DB_VECTOR_EXTENSION: pgvector
 
 immich:
@@ -12,7 +12,9 @@ immich:
   
 redis:
   enabled: true
-
+  master:
+    resources:
+      limits: 
 
 server:
   ingress:

incngrnt-web/values.yaml

@@ -1,7 +1,7 @@
 init:
   method: wget
   wget:
-    url: https://git.incngrnt.ca/grant/incngrnt/releases/download/v0.0.7/v0.0.7.tar
+    url: https://git.incngrnt.ca/grant/incngrnt/releases/download/v0.0.8/v0.0.8.tar
 
 ingress:
   enabled: true

traefik/values.yaml

@@ -47,7 +47,9 @@ ports:
   websecure:
     middlewares:
       - traefik-rate-limit@kubernetescrd
-
+  web:
+    middlewares:
+      - traefik-redirectscheme@kubernetescrd      
   ssh:
     port: 2222
     expose:
@@ -81,6 +83,14 @@ extraObjects:
       rateLimit:
         average: 50
         burst: 100
+  - apiVersion: traefik.io/v1alpha1
+    kind: Middleware
+    metadata:
+      name: redirectscheme
+    spec:
+      redirectScheme:
+        scheme: https
+        permanent: true
 
 certificatesResolvers:
   letsencrypt: