Refactor structure to conform to Flux bootstrap

.github/workflows/e2e.yaml

@@ -0,0 +1,47 @@
+name: e2e
+
+on:
+ workflow_dispatch:
+ push:
+ branches: [ '*' ]
+ tags-ignore: [ '*' ]
+
+jobs:
+ kubernetes:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Setup Flux
+ uses: fluxcd/flux2/action@main
+ - name: Setup Kubernetes
+ uses: helm/kind-action@v1.8.0
+ with:
+ cluster_name: flux
+ version: v0.20.0
+ # The versions below should target the newest Kubernetes version
+ # Keep this up-to-date with https://endoflife.date/kubernetes
+ node_image: kindest/node:v1.28.0@sha256:9f3ff58f19dcf1a0611d11e8ac989fdb30a28f40f236f59f0bea31fb956ccf5c
+ kubectl_version: v1.28.0
+ - name: Install Flux in Kubernetes Kind
+ run: flux install
+ - name: Setup cluster reconciliation
+ run: |
+ flux create source git flux-system \
+ --url=${{ github.event.repository.html_url }} \
+ --branch=${GITHUB_REF#refs/heads/} \
+ --ignore-paths="clusters/**/flux-system/"
+ flux create kustomization flux-system \
+ --source=flux-system \
+ --path=./clusters/test
+ - name: Verify cluster reconciliation
+ run: |
+ kubectl -n flux-system wait kustomization/monitoring-controllers --for=condition=ready --timeout=10m
+ kubectl -n flux-system wait kustomization/monitoring-configs --for=condition=ready --timeout=1m
+ - name: Debug failure
+ if: failure()
+ run: |
+ kubectl -n flux-system logs deploy/source-controller
+ kubectl -n flux-system logs deploy/kustomize-controller
+ kubectl -n flux-system logs deploy/helm-controller
+ flux get all --all-namespaces

.github/workflows/test.yaml

@@ -0,0 +1,23 @@
+name: test
+
+on:
+ workflow_dispatch:
+ pull_request:
+ push:
+ branches: [ '*' ]
+ tags-ignore: [ '*' ]
+
+jobs:
+ manifests:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Setup yq
+ uses: fluxcd/pkg/actions/yq@main
+ - name: Setup kubeconform
+ uses: fluxcd/pkg/actions/kubeconform@main
+ - name: Setup kustomize
+ uses: fluxcd/pkg/actions/kustomize@main
+ - name: Validate manifests
+ run: ./scripts/validate.sh

README.md

@@ -1,2 +1,97 @@
 # flux2-monitoring-example
-Prometheus monitoring for the Flux control plane
+
+This repository is an example of how to make use of
+[kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack)
+and
+[loki-stack](https://github.com/grafana/helm-charts/tree/main/charts/loki-stack)
+to monitor Flux.
+
+Components:
+
+* **kube-state-metrics** - generates metrics about the state of the Flux objects
+* **Prometheus Operator** - manages Prometheus clusters atop Kubernetes
+* **Prometheus** - collects and stores metrics from the Flux controllers and kube-state-metrics
+* **Promtail** - collects the logs from the Flux controllers
+* **Loki** - stores the logs collected by Promtail
+* **Grafana** dashboards - displays the Flux control plane resource usage, reconciliation stats and logs
+
+## Quickstart
+
+### Create a Kubernetes cluster
+
+For a quick local test, you can use [Kubernetes kind](https://kind.sigs.k8s.io/docs/user/quick-start/).
+Any other Kubernetes setup will work as well though.
+
+Create a cluster called `test` with the kind CLI:
+
+```shell
+kind create cluster --name test
+```
+
+### Fork the GitHub repository
+
+In order to follow this guide you'll need a GitHub account and a
+[personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line)
+that can create repositories (check all permissions under `repo`).
+
+Add the GitHub PAT and username to your shell environment:
+
+```sh
+export GITHUB_TOKEN=<your-token>
+export GITHUB_USER=<your-username>
+```
+
+Fork this repository on your personal account and clone it locally:
+
+```shell
+git clone https://github.com/${GITHUB_USER}/flux2-monitoring-example.git
+cd flux2-monitoring-example
+```
+
+### Bootstrap Flux
+
+Install the Flux controllers on the test cluster:
+
+```shell
+flux bootstrap github \
+ --owner=${GITHUB_USER} \
+ --repository=flux2-monitoring-example \
+ --branch=main \
+ --personal \
+ --path=clusters/test
+```
+
+Wait for Flux to deploy the monitoring stack with:
+
+```shell
+flux get kustomizations --watch
+```
+
+After Flux has finished reconciling, you can list the pods in the monitoring namespace with:
+
+```console
+$ kubectl -n monitoring get po
+NAME READY
+kube-prometheus-stack-grafana-5c976ff4cf-xgmwm 3/3
+kube-prometheus-stack-kube-state-metrics-5dcf4c4697-jvlvh 1/1
+kube-prometheus-stack-operator-75f9fdcbf6-98zmh 1/1
+kube-prometheus-stack-prometheus-node-exporter-j4vhb 1/1
+loki-stack-0 1/1
+loki-stack-promtail-dcg64 1/1
+prometheus-kube-prometheus-stack-prometheus-0 2/2
+```
+
+### Accessing Grafana
+
+To access Grafana, start port forward in a separate shell:
+
+```shell
+kubectl -n monitoring port-forward svc/kube-prometheus-stack-grafana 3000:80
+```
+
+Navigate to `http://localhost:3000` in your browser and login with user `admin` and password `flux`.
+
+Flux dashboards:
+- [Reconciliation stats](http://localhost:3000/d/flux-cluster/flux-cluster-stats)
+- [Control plane stats](http://localhost:3000/d/flux-control-plane/flux-control-plane)
+- [Control plane logs](http://localhost:3000/d/flux-logs/flux-logs)

clusters/test/flux-system/gotk-components.yaml

@@ -0,0 +1 @@
+# This file will be generated automatically by flux boostrap.

clusters/test/flux-system/gotk-sync.yaml

@@ -0,0 +1 @@
+# This file will be generated automatically by flux boostrap.

clusters/test/flux-system/kustomization.yaml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+ - gotk-components.yaml
+ - gotk-sync.yaml
+labels:
+ - pairs:
+ toolkit.fluxcd.io/tenant: sre-team
+patches:
+ - patch: |
+ - op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --concurrent=20
+ - op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --requeue-dependency=5s
+ target:
+ kind: Deployment
+ name: "(kustomize-controller|helm-controller|source-controller)"

clusters/test/monitoring.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+ name: monitoring-controllers
+ namespace: flux-system
+spec:
+ interval: 1h
+ retryInterval: 2m
+ timeout: 10m
+ prune: true
+ wait: true
+ sourceRef:
+ kind: GitRepository
+ name: flux-system
+ path: ./monitoring/controllers
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+ name: monitoring-configs
+ namespace: flux-system
+spec:
+ dependsOn:
+ - name: monitoring-controllers
+ interval: 1h
+ retryInterval: 2m
+ timeout: 5m
+ prune: true
+ wait: true
+ sourceRef:
+ kind: GitRepository
+ name: flux-system
+ path: ./monitoring/configs

kube-prometheus-stack/release.yaml → monitoring/controllers/kube-prometheus-stack/release.yaml

@@ -3,15 +3,15 @@ kind: HelmRelease
 metadata:
  name: kube-prometheus-stack
 spec:
- interval: 5m
+ interval: 1h
  chart:
  spec:
  version: "48.x"
  chart: kube-prometheus-stack
  sourceRef:
  kind: HelmRepository
  name: prometheus-community
- interval: 60m
+ interval: 1h
  install:
  crds: Create
  upgrade:
@@ -33,6 +33,7 @@ spec:
  app.kubernetes.io/component: monitoring
  grafana:
  defaultDashboardsEnabled: false
+ adminPassword: flux
  kube-state-metrics:
  collectors: []
  extraArgs:

kube-prometheus-stack/repository.yaml → monitoring/controllers/kube-prometheus-stack/repository.yaml

@@ -3,6 +3,6 @@ kind: HelmRepository
 metadata:
  name: prometheus-community
 spec:
- interval: 120m
- type: default
- url: https://prometheus-community.github.io/helm-charts
+ interval: 12h
+ type: oci
+ url: oci://ghcr.io/prometheus-community/charts

scripts/validate.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# This script downloads the Flux OpenAPI schemas, then it validates the
+# Flux custom resources and the kustomize overlays using kubeconform.
+# This script is meant to be run locally and in CI before the changes
+# are merged on the main branch that's synced by Flux.
+
+# Copyright 2023 The Flux authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Prerequisites
+# - yq v4.34
+# - kustomize v5.0
+# - kubeconform v0.6
+
+set -o errexit
+set -o pipefail
+
+# mirror kustomize-controller build options
+kustomize_flags=("--load-restrictor=LoadRestrictionsNone")
+kustomize_config="kustomization.yaml"
+
+# skip Kubernetes Secrets due to SOPS fields failing validation
+kubeconform_flags=("-skip=Secret")
+kubeconform_config=("-strict" "-ignore-missing-schemas" "-schema-location" "default" "-schema-location" "/tmp/flux-crd-schemas" "-verbose")
+
+echo "INFO - Downloading Flux OpenAPI schemas"
+mkdir -p /tmp/flux-crd-schemas/master-standalone-strict
+curl -sL https://github.com/fluxcd/flux2/releases/latest/download/crd-schemas.tar.gz | tar zxf - -C /tmp/flux-crd-schemas/master-standalone-strict
+
+find . -type f -name '*.yaml' -print0 | while IFS= read -r -d $'\0' file;
+ do
+ echo "INFO - Validating $file"
+ yq e 'true' "$file" > /dev/null
+done
+
+echo "INFO - Validating clusters"
+find ./clusters -maxdepth 2 -type f -name '*.yaml' -print0 | while IFS= read -r -d $'\0' file;
+ do
+ kubeconform "${kubeconform_flags[@]}" "${kubeconform_config[@]}" "${file}"
+ if [[ ${PIPESTATUS[0]} != 0 ]]; then
+ exit 1
+ fi
+done
+
+echo "INFO - Validating kustomize overlays"
+find . -type f -name $kustomize_config -print0 | while IFS= read -r -d $'\0' file;
+ do
+ echo "INFO - Validating kustomization ${file/%$kustomize_config}"
+ kustomize build "${file/%$kustomize_config}" "${kustomize_flags[@]}" | \
+ kubeconform "${kubeconform_flags[@]}" "${kubeconform_config[@]}"
+ if [[ ${PIPESTATUS[0]} != 0 ]]; then
+ exit 1
+ fi
+done