From 34b1e9e0772e9afff8b6a4bf4bf6a768ef8ab2d4 Mon Sep 17 00:00:00 2001 From: Grant <5445379+grantdhunter@users.noreply.github.com> Date: Sat, 15 Mar 2025 15:10:17 -0600 Subject: [PATCH] Add monitoring stack --- .gitignore | 2 + grafana/alloy_values.yaml | 23 +++++++++++ grafana/grafana_values.yaml | 9 +++++ grafana/loki_values.yaml | 70 ++++++++++++++++++++++++++++++++++ grafana/prometheus_values.yaml | 3 ++ helmfile.lock | 20 ++++++++-- helmfile.yaml | 50 ++++++++++++++++++++++++ rook-ceph/values.yaml | 4 ++ traefik/values.yaml | 6 +++ 9 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 grafana/alloy_values.yaml create mode 100644 grafana/grafana_values.yaml create mode 100644 grafana/loki_values.yaml create mode 100644 grafana/prometheus_values.yaml create mode 100644 rook-ceph/values.yaml diff --git a/.gitignore b/.gitignore index 30df71a..d4b137f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ talos/secrets.yaml kgnot/config.production.json 53ll/config.production.json +*.key +*.pub \ No newline at end of file diff --git a/grafana/alloy_values.yaml b/grafana/alloy_values.yaml new file mode 100644 index 0000000..08fe506 --- /dev/null +++ b/grafana/alloy_values.yaml @@ -0,0 +1,23 @@ +alloy: + configMap: + content: |- + logging { + level = "info" + format = "logfmt" + } + + discovery.kubernetes "pods" { + role = "pod" + } + + loki.source.kubernetes "pods" { + targets = discovery.kubernetes.pods.targets + forward_to = [loki.write.loki.receiver] + + } + + loki.write "loki" { + endpoint { + url = "http://loki.grafana.svc.cluster.local:3100/loki/api/v1/push" + } + } diff --git a/grafana/grafana_values.yaml b/grafana/grafana_values.yaml new file mode 100644 index 0000000..78b5c36 --- /dev/null +++ b/grafana/grafana_values.yaml @@ -0,0 +1,9 @@ +ingress: + enabled: true + hosts: + - watcher.incngrnt.ca + annotations: + "traefik.ingress.kubernetes.io/router.tls.certresolver": "letsencrypt" + +persistence: + enabled: true diff --git a/grafana/loki_values.yaml b/grafana/loki_values.yaml new file mode 100644 index 0000000..9716ce4 --- /dev/null +++ b/grafana/loki_values.yaml @@ -0,0 +1,70 @@ +loki: + commonConfig: + replication_factor: 1 + schemaConfig: + configs: + - from: "2024-04-01" + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + ruler: + enable_api: true + auth_enabled: false + +resultsCache: + resources: + request: + cpu: 100ms + memory: 500Mi + limits: + memory: 500Mi +chunksCache: + resources: + request: + cpu: 100ms + memory: 500Mi + limits: + memory: 500Mi +minio: + enabled: true + + +deploymentMode: SingleBinary + +singleBinary: + replicas: 1 + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 diff --git a/grafana/prometheus_values.yaml b/grafana/prometheus_values.yaml new file mode 100644 index 0000000..08fcfe3 --- /dev/null +++ b/grafana/prometheus_values.yaml @@ -0,0 +1,3 @@ +rometheus-node-exporter: + rbac: + pspEnabled: true diff --git a/helmfile.lock b/helmfile.lock index 036c117..35cdc21 100644 --- a/helmfile.lock +++ b/helmfile.lock @@ -1,17 +1,26 @@ version: 0.170.1 dependencies: +- name: alloy + repository: https://grafana.github.io/helm-charts + version: 0.12.5 - name: ghost repository: https://charts.bitnami.com/bitnami - version: 22.1.19 + version: 22.2.0 - name: ghost repository: https://charts.bitnami.com/bitnami - version: 22.1.19 + version: 22.2.0 - name: gitea repository: https://dl.gitea.io/charts version: 11.0.0 +- name: grafana + repository: https://grafana.github.io/helm-charts + version: 8.10.3 - name: k8up repository: https://k8up-io.github.io/k8up version: 4.8.4 +- name: loki + repository: https://grafana.github.io/helm-charts + version: 6.28.0 - name: mariadb repository: https://charts.bitnami.com/bitnami version: 20.4.1 @@ -24,6 +33,9 @@ dependencies: - name: postgrescluster repository: git+https://github.com/grantdhunter/postgres-operator@helm?ref=main version: 5.7.2 +- name: prometheus + repository: https://prometheus-community.github.io/helm-charts + version: 27.5.1 - name: rook-ceph repository: https://charts.rook.io/release version: v1.16.5 @@ -36,5 +48,5 @@ dependencies: - name: traefik repository: https://traefik.github.io/charts version: 34.4.1 -digest: sha256:14c9bc504c5060f8bbce5ec9a8df737a19a7216428a31c1cb34ef5c6648e45c5 -generated: "2025-03-12T21:27:22.529913117-06:00" +digest: sha256:b28767f0ec4d8549e0b1de7446f3468555a2a67bf88d2b554e9a12c2de723d2d +generated: "2025-03-15T14:34:49.001292746-06:00" diff --git a/helmfile.yaml b/helmfile.yaml index 2dcda48..067a52b 100644 --- a/helmfile.yaml +++ b/helmfile.yaml @@ -19,6 +19,10 @@ repositories: url: https://pkgs.tailscale.com/helmcharts - name: gitea url: https://dl.gitea.io/charts + - name: grafana + url: https://grafana.github.io/helm-charts + - name: prometheus-community + url: https://prometheus-community.github.io/helm-charts releases: # networking - name: metallb @@ -52,6 +56,8 @@ releases: namespace: rook-ceph createNamespace: true chart: rook-release/rook-ceph + values: + - ./rook-ceph/values.yaml - name: rook-ceph-cluster namespace: rook-ceph createNamespace: true @@ -83,6 +89,36 @@ releases: setString: - name: auth.rootPassword value: {{ requiredEnv "MARIADB_ROOT_PASSWORD" }} + +# monitoring + - name: grafana + namespace: grafana + createNamespace: true + chart: grafana/grafana + values: + - grafana/grafana_values.yaml + setString: + - name: adminPassword + value: VYHEKk0Q9KfqQ3UpTx8oc4InrXlUQivUuEeGU8LJ + - name: prometheus + namespace: grafana + createNamespace: true + chart: prometheus-community/prometheus + values: + - grafana/prometheus_values.yaml + - name: loki + namespace: grafana + createNamespace: true + chart: grafana/loki + values: + - grafana/loki_values.yaml + - name: alloy + namespace: grafana + createNamespace: true + chart: grafana/alloy + values: + - grafana/alloy_values.yaml + # goatchat matrix - name: goatchat namespace: goatchat @@ -172,3 +208,17 @@ releases: - name: repoPassword value: {{ requiredEnv "k8UP_REPO_PASSWORD" }} + - name: gitea-backup + namespace: gitea + chart: ./k8up-backup + createNamespace: true + values: + - ./k8up-backup/values_override.yaml + setString: + - name: credentials.id + value: {{ requiredEnv "HETZNER_S3_ACCESS_KEY" }} + - name: credentials.key + value: {{ requiredEnv "HETZNER_S3_ACCESS_SECRET" }} + - name: repoPassword + value: {{ requiredEnv "k8UP_REPO_PASSWORD" }} + diff --git a/rook-ceph/values.yaml b/rook-ceph/values.yaml new file mode 100644 index 0000000..27bc9b4 --- /dev/null +++ b/rook-ceph/values.yaml @@ -0,0 +1,4 @@ +resources: + requests: + cpu: 100m + memory: 128Mi diff --git a/traefik/values.yaml b/traefik/values.yaml index 63d17e4..bb247ff 100644 --- a/traefik/values.yaml +++ b/traefik/values.yaml @@ -19,6 +19,12 @@ additionalArguments: persistence: enabled: true + +logs: + format: json + access: + enabled: true + format: json ingressRoute: dashboard: