Fix monitoring, remove limits from immich

grafana/alloy_values.yaml

@@ -9,15 +9,121 @@ alloy:
       discovery.kubernetes "pods" {
         role = "pod"
       }
-      
-      loki.source.kubernetes "pods" {
-        targets    = discovery.kubernetes.pods.targets
-        forward_to = [loki.write.loki.receiver]
-
+      discovery.kubernetes "nodes" {
+        role = "node"
       }
       
+      discovery.relabel "pods" {
+        targets = discovery.kubernetes.pods.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_container_name"]
+          separator     = "/"
+          target_label  = "deployment_name"
+          action        = "replace"
+        }
+      }
+      loki.source.kubernetes "pods" {
+        targets    = discovery.relabel.pods.output
+        forward_to = [loki.process.process.receiver]
+      }
+      loki.process "process" {
+        forward_to = [loki.write.loki.receiver]
+
+        stage.drop {
+          older_than          = "1h"
+          drop_counter_reason = "too old"
+        }
+        stage.match { 
+          selector = "{instance=~\".*\"}"
+          stage.json {
+            expressions = {
+              level = "\"level\"",
+            }
+          }
+          stage.labels {
+            values = { 
+              level = "level",
+            }
+          }
+        }
+        stage.label_drop {
+          values = [ "job", "service_name" ]
+        }
+      }      
       loki.write "loki" {
          endpoint {
-           url = "http://loki.grafana.svc.cluster.local:3100/loki/api/v1/push"
+           url = "http://grafana-loki-distributor:3100/loki/api/v1/push"
          }
        }
+       
+      discovery.relabel "metrics" {
+        targets = discovery.kubernetes.pods.targets
+        rule {
+          source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port"]
+          target_label  = "__meta_kubernetes_pod_container_port_number"
+          action = "keepequal"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_container_port_number"]
+          regex = ""
+          action = "drop"
+        }    
+        rule {
+          source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path",]
+          target_label  = "__metrics_path__"
+          separator = ""
+          action = "replace"
+        }        
+      }
+      prometheus.scrape "metrics" {
+        clustering {
+            enabled = true
+        }
+        targets    = discovery.relabel.metrics.output
+        forward_to = [prometheus.remote_write.metrics.receiver]
+        scrape_interval = "30s"
+      }
+      discovery.relabel "pods_metrics" {
+        targets = discovery.kubernetes.nodes.targets
+        rule {
+          replacement  = "kubernetes.default.svc:443"
+          target_label = "__address__"
+        }
+        rule {
+          regex         = "(.+)"
+          replacement   = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
+          source_labels = ["__meta_kubernetes_node_name"]
+          target_label  = "__metrics_path__"
+        }
+      }
+      prometheus.scrape "pods_metrics" {
+        clustering {
+            enabled = true
+        }
+        targets      = discovery.relabel.pods_metrics.output
+        job_name     = "integrations/kubernetes/kubelet"
+        scheme       = "https"
+        honor_labels = true
+        forward_to = [prometheus.remote_write.metrics.receiver]
+        bearer_token_file = "/run/secrets/kubernetes.io/serviceaccount/token"
+        tls_config {
+          insecure_skip_verify = true
+          server_name          = "kubernetes"
+        }
+        scrape_interval = "30s"
+      }
+      prometheus.exporter.unix "os_metrics" { }
+      prometheus.scrape "os_metrics" {
+        clustering {
+            enabled = true
+        }
+        targets    = prometheus.exporter.unix.os_metrics.targets
+        forward_to = [prometheus.remote_write.metrics.receiver]
+        scrape_interval = "30s"
+      }
+      prometheus.remote_write "metrics" {
+        endpoint {
+          url = "http://grafana-mimir-nginx/api/v1/push"
+        }
+      }

grafana/values.yaml

@@ -0,0 +1,8 @@
+grafana:
+  ingress:
+    enabled: true
+    hosts:
+      - watcher.incngrnt.ca
+    annotations:
+      "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt"
+