add ksm metrics

grafana/alloy_values.yaml

@@ -1,4 +1,6 @@
 alloy:
+  clustering:
+    enabled: true
   configMap:
     content: |-
       logging {
@@ -122,8 +124,76 @@ alloy:
         forward_to = [prometheus.remote_write.metrics.receiver]
         scrape_interval = "30s"
       }
+
+      discovery.kubernetes "kube_state_metrics" {
+        role = "endpoints"
+
+        selectors {
+          role = "endpoints"
+          label = "app.kubernetes.io/name=kube-state-metrics"
+        }
+        namespaces {
+          names = ["grafana"]
+        }
+      }
+
+      discovery.relabel "kube_state_metrics" {
+        targets = discovery.kubernetes.kube_state_metrics.targets
+
+        // only keep targets with a matching port name
+        rule {
+          source_labels = ["__meta_kubernetes_endpoint_port_name"]
+          regex = "http"
+          action = "keep"
+        }
+
+        rule {
+          action = "replace"
+          replacement = "kubernetes"
+          target_label = "source"
+        }
+
+      }
+
+      prometheus.scrape "kube_state_metrics" {
+        targets = discovery.relabel.kube_state_metrics.output
+        job_name = "integrations/kubernetes/kube-state-metrics"
+        scrape_interval = "30s"
+        scheme = "http"
+        bearer_token_file = ""
+        tls_config {
+          insecure_skip_verify = true
+        }
+
+        clustering {
+          enabled = true
+        }
+        forward_to = [prometheus.relabel.kube_state_metrics.receiver]
+      }
+
+      prometheus.relabel "kube_state_metrics" {
+        max_cache_size = 100000
+        rule {
+          source_labels = ["__name__"]
+          regex = "up|scrape_samples_scraped|kube_configmap_info|kube_configmap_metadata_resource_version|kube_daemonset.*|kube_deployment_metadata_generation|kube_deployment_spec_replicas|kube_deployment_status_condition|kube_deployment_status_observed_generation|kube_deployment_status_replicas_available|kube_deployment_status_replicas_updated|kube_horizontalpodautoscaler_spec_max_replicas|kube_horizontalpodautoscaler_spec_min_replicas|kube_horizontalpodautoscaler_status_current_replicas|kube_horizontalpodautoscaler_status_desired_replicas|kube_job.*|kube_namespace_status_phase|kube_node.*|kube_persistentvolume_status_phase|kube_persistentvolumeclaim_access_mode|kube_persistentvolumeclaim_info|kube_persistentvolumeclaim_labels|kube_persistentvolumeclaim_resource_requests_storage_bytes|kube_persistentvolumeclaim_status_phase|kube_pod_container_info|kube_pod_container_resource_limits|kube_pod_container_resource_requests|kube_pod_container_status_last_terminated_reason|kube_pod_container_status_restarts_total|kube_pod_container_status_waiting_reason|kube_pod_info|kube_pod_owner|kube_pod_spec_volumes_persistentvolumeclaims_info|kube_pod_start_time|kube_pod_status_phase|kube_pod_status_reason|kube_replicaset.*|kube_resourcequota|kube_secret_metadata_resource_version|kube_statefulset.*"
+          action = "keep"
+        }
+
+        forward_to = [prometheus.remote_write.metrics.receiver]
+      }
+
       prometheus.remote_write "metrics" {
         endpoint {
           url = "http://grafana-mimir-nginx/api/v1/push"
         }
       }
+
+
+
+  resources:
+    requests:
+      cpu: 1m
+      memory: 5Mi
+    limits:
+      cpu: 1
+      memory: 400Mi

grafana/values.yaml

@@ -6,3 +6,5 @@ grafana:
     annotations:
       "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt"
 
+  persistence:
+    enabled: true    

helmfile.lock

@@ -18,6 +18,9 @@ dependencies:
 - name: k8up
   repository: https://k8up-io.github.io/k8up
   version: 4.8.4
+- name: kube-state-metrics
+  repository: https://prometheus-community.github.io/helm-charts
+  version: 5.30.1
 - name: lgtm-distributed
   repository: https://grafana.github.io/helm-charts
   version: 2.1.0
@@ -48,5 +51,5 @@ dependencies:
 - name: traefik
   repository: https://traefik.github.io/charts
   version: 34.4.1
-digest: sha256:e2c65297653c325106e0a9055ed7a59f9c801035089e1684934ceef50a032ac8
-generated: "2025-03-17T21:13:03.161466198-06:00"
+digest: sha256:5645dcc617a44ae663f1550e08169a323c8d96e491f45178362a5389763f4b2f
+generated: "2025-03-18T20:53:53.212208477-06:00"

helmfile.yaml

@@ -106,7 +106,10 @@ releases:
     chart: grafana/alloy
     values:
       - grafana/alloy_values.yaml
-      
+  - name: kube-state-metrics
+    namespace: grafana
+    createNamespace: true
+    chart: prometheus-community/kube-state-metrics
   # goatchat matrix
   - name: goatchat
     namespace: goatchat