diff --git a/documentation/web/docs/user/grafana-dashboards.md b/documentation/web/docs/user/grafana-dashboards.md index ac20012e..0357579a 100644 --- a/documentation/web/docs/user/grafana-dashboards.md +++ b/documentation/web/docs/user/grafana-dashboards.md @@ -31,7 +31,7 @@ The **Klio Clusters** dashboard is split into row sections: and the backup success ratio. - **Retained backups** — the retention window of the physical PostgreSQL - backups the Klio server keeps for each cluster: counts by tier, + backups the Klio server keeps for each cluster: counts by tier and server, latest/oldest backup age, start/end LSN and PostgreSQL timeline. - **Backup maintenance** — tier-2 relay, backup verification and maintenance run totals per cluster. diff --git a/documentation/web/docs/user/images/clusters_dashboard.png b/documentation/web/docs/user/images/clusters_dashboard.png index 964950a7..87f86c03 100644 Binary files a/documentation/web/docs/user/images/clusters_dashboard.png and b/documentation/web/docs/user/images/clusters_dashboard.png differ diff --git a/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png b/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png index 144c52c7..ff0452bf 100644 Binary files a/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png and b/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png differ diff --git a/documentation/web/docs/user/images/servers_dashboard.png b/documentation/web/docs/user/images/servers_dashboard.png index 45291f2c..47f645cd 100644 Binary files a/documentation/web/docs/user/images/servers_dashboard.png and b/documentation/web/docs/user/images/servers_dashboard.png differ diff --git a/observability/grafana/build.go b/observability/grafana/build.go index d1765624..d16c20b2 100644 --- a/observability/grafana/build.go +++ b/observability/grafana/build.go @@ -42,12 +42,19 @@ func clusterDashboard() *dashboard.DashboardBuilder { builder := newDashboard("klio-clusters", "Klio Clusters", "Backups, WAL archiving and WAL restore of the PostgreSQL clusters backed up by Klio."). WithVariable(labelVariable("namespace", "Namespace", - `label_values({__name__=~"klio_(plugin|client)_.+"}, k8s_namespace_name)`)). + `label_values({__name__=~"klio_(plugin|client)_.+"}, k8s_namespace_name)`). + AllValue(matchAny)). + // "All" expands to the clusters of the selected namespaces. Matching any + // value would also select clusters from other namespaces on the panels + // that are not filtered by namespace. Refreshed with the time range, so + // "All" picks up clusters that appear after the dashboard loads. WithVariable(labelVariable("cluster", "Cluster", - `label_values({__name__=~"klio_(plugin|client)_.+",k8s_namespace_name=~"$namespace"}, cluster_name)`)). + `label_values({__name__=~"klio_(plugin|client)_.+",k8s_namespace_name=~"$namespace"}, cluster_name)`). + Refresh(dashboard.VariableRefreshOnTimeRangeChanged)). WithVariable(labelVariable("pod", "Instance", `label_values({__name__=~"klio_(plugin|client)_.+",k8s_namespace_name=~"$namespace",`+ - `cluster_name=~"$cluster"}, k8s_pod_name)`)) + `cluster_name=~"$cluster"}, k8s_pod_name)`). + AllValue(matchAny)) layoutDashboard(builder, "*Namespace* is where the PostgreSQL cluster runs. *Cluster* lists every cluster that emits Klio "+ @@ -97,11 +104,19 @@ func serverDashboard() *dashboard.DashboardBuilder { builder := newDashboard("klio-servers", "Klio Servers", "Health, served clusters, Kopia snapshots and WAL processing of the Klio servers."). WithVariable(labelVariable("namespace", "Server namespace", - `label_values(klio_server_uptime_seconds, k8s_namespace_name)`)). + `label_values(klio_server_uptime_seconds, k8s_namespace_name)`). + AllValue(matchAny)). + // "All" expands to the servers of the selected namespaces, for the same + // reason, and refreshed, as the cluster variable of the Klio Clusters + // dashboard. WithVariable(labelVariable("server", "Server", - `label_values(klio_server_uptime_seconds{k8s_namespace_name=~"$namespace"}, service_name)`)). + `label_values(klio_server_uptime_seconds{k8s_namespace_name=~"$namespace"}, service_name)`). + Refresh(dashboard.VariableRefreshOnTimeRangeChanged)). + // Not sourced from klio_server_wal_written_total: a server that receives + // no WAL, such as a read-only one, would list no clusters. WithVariable(labelVariable("cluster", "Served cluster", - `label_values(klio_server_wal_written_total{service_name=~"$server"}, cluster_name)`)) + `label_values({__name__=~"klio_server_.+",service_name=~"$server"}, cluster_name)`). + AllValue(matchAny)) layoutDashboard(builder, "*Server namespace* is where the Klio server runs, not where the clusters it backs run. *Server* "+ @@ -219,8 +234,14 @@ func layoutSection(builder *dashboard.DashboardBuilder, y *int, title string, pa } } +// matchAny is the "All" value of the variables whose "All" must match any +// series, whatever the values listed by the variable. +const matchAny = ".+" + // labelVariable builds a multi-value Prometheus label_values template variable // that defaults to "All" so the dashboard shows every series until filtered. +// Unless the caller sets AllValue, "All" expands to the values the variable +// lists. func labelVariable(name, label, query string) *dashboard.QueryVariableBuilder { return dashboard.NewQueryVariableBuilder(name). Label(label). @@ -230,7 +251,6 @@ func labelVariable(name, label, query string) *dashboard.QueryVariableBuilder { Sort(dashboard.VariableSortAlphabeticalAsc). Multi(true). IncludeAll(true). - AllValue(".+"). Current(dashboard.VariableOption{ Selected: cog.ToPtr(true), Text: dashboard.StringOrArrayOfString{String: cog.ToPtr("All")}, diff --git a/observability/grafana/dashboards/klio-clusters.json b/observability/grafana/dashboards/klio-clusters.json index 4c239c0f..9c679182 100644 --- a/observability/grafana/dashboards/klio-clusters.json +++ b/observability/grafana/dashboards/klio-clusters.json @@ -729,14 +729,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (cluster_name, tier) (klio_server_backup_backups{cluster_name=~\"$cluster\"})", + "expr": "sum by (cluster_name, tier, service_name) (klio_server_backup_backups{cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} {{tier}}", + "legendFormat": "{{cluster_name}} {{tier}} ({{service_name}})", "range": true } ], "title": "Retained backups", - "description": "Number of PostgreSQL backups currently retained per cluster and tier.", + "description": "Number of PostgreSQL backups currently retained per cluster and tier, with the Klio server that retains them.", "transparent": false, "datasource": { "type": "prometheus", @@ -2940,10 +2940,9 @@ }, "multi": true, "allowCustomValue": true, - "refresh": 1, + "refresh": 2, "sort": 1, "includeAll": true, - "allValue": ".+", "auto": false, "auto_min": "10s", "auto_count": 30 diff --git a/observability/grafana/dashboards/klio-servers.json b/observability/grafana/dashboards/klio-servers.json index 1fc8ec06..abc00617 100644 --- a/observability/grafana/dashboards/klio-servers.json +++ b/observability/grafana/dashboards/klio-servers.json @@ -256,14 +256,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (cluster_name, tier) (klio_server_backup_backups{service_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "sum by (cluster_name, tier, service_name) (klio_server_backup_backups{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} {{tier}}", + "legendFormat": "{{cluster_name}} {{tier}} ({{service_name}})", "range": true } ], "title": "Retained backups", - "description": "Number of PostgreSQL backups currently retained per cluster and tier.", + "description": "Number of PostgreSQL backups currently retained per cluster and tier, with the Klio server that retains them.", "transparent": false, "datasource": { "type": "prometheus", @@ -501,14 +501,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (cluster, tier) (label_replace(klio_server_backup_snapshots{service_name=~\"$server\",snapshot_source=~\"[^@]*@$cluster:.*\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", + "expr": "sum by (cluster, tier, service_name) (label_replace(klio_server_backup_snapshots{service_name=~\"$server\",snapshot_source=~\"[^@]*@$cluster:.*\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "{{cluster}} {{tier}}", + "legendFormat": "{{cluster}} {{tier}} ({{service_name}})", "range": true } ], "title": "Retained snapshots", - "description": "Total kopia snapshots currently retained per cluster and tier.", + "description": "Total kopia snapshots currently retained per cluster and tier, with the Klio server that retains them.", "transparent": false, "datasource": { "type": "prometheus", @@ -516,7 +516,7 @@ }, "gridPos": { "h": 8, - "w": 4, + "w": 8, "x": 0, "y": 25 }, @@ -574,8 +574,8 @@ }, "gridPos": { "h": 8, - "w": 4, - "x": 4, + "w": 8, + "x": 8, "y": 25 }, "repeatDirection": "h", @@ -631,8 +631,8 @@ }, "gridPos": { "h": 8, - "w": 4, - "x": 8, + "w": 8, + "x": 16, "y": 25 }, "repeatDirection": "h", @@ -689,9 +689,9 @@ }, "gridPos": { "h": 8, - "w": 4, - "x": 12, - "y": 25 + "w": 8, + "x": 0, + "y": 33 }, "repeatDirection": "h", "options": { @@ -747,9 +747,9 @@ }, "gridPos": { "h": 8, - "w": 4, - "x": 16, - "y": 25 + "w": 8, + "x": 8, + "y": 33 }, "repeatDirection": "h", "options": { @@ -804,9 +804,9 @@ }, "gridPos": { "h": 8, - "w": 4, - "x": 20, - "y": 25 + "w": 8, + "x": 16, + "y": 33 }, "repeatDirection": "h", "options": { @@ -846,7 +846,7 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 41 }, "id": 0, "panels": [] @@ -858,7 +858,7 @@ "h": 2, "w": 24, "x": 0, - "y": 34 + "y": 42 }, "repeatDirection": "h", "options": { @@ -911,7 +911,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 44 }, "repeatDirection": "h", "options": { @@ -982,7 +982,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 44 }, "repeatDirection": "h", "options": { @@ -1053,7 +1053,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 52 }, "repeatDirection": "h", "options": { @@ -1124,7 +1124,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 52 }, "repeatDirection": "h", "options": { @@ -1212,10 +1212,9 @@ }, "multi": true, "allowCustomValue": true, - "refresh": 1, + "refresh": 2, "sort": 1, "includeAll": true, - "allValue": ".+", "auto": false, "auto_min": "10s", "auto_count": 30 @@ -1225,7 +1224,7 @@ "name": "cluster", "label": "Served cluster", "skipUrlSync": false, - "query": "label_values(klio_server_wal_written_total{service_name=~\"$server\"}, cluster_name)", + "query": "label_values({__name__=~\"klio_server_.+\",service_name=~\"$server\"}, cluster_name)", "datasource": { "type": "prometheus", "uid": "${datasource}" diff --git a/observability/grafana/server.go b/observability/grafana/server.go index ad49a9b5..bdc2bb6c 100644 --- a/observability/grafana/server.go +++ b/observability/grafana/server.go @@ -41,14 +41,16 @@ func snapshotCluster(selector string) string { // The server panels show the metrics of the Klio server (`klio_server_*`). // Panels shown on both dashboards take their series matcher. -// retainedBackupsPanel counts the retained backups. +// retainedBackupsPanel counts the retained backups, naming the server that +// retains them. func retainedBackupsPanel(matcher string) *stat.PanelBuilder { return statPanel("Retained backups", units.Number, - query(fmt.Sprintf("sum by (cluster_name, tier) (klio_server_backup_backups{%s})", matcher), - "{{cluster_name}} {{tier}}"), + query(fmt.Sprintf("sum by (cluster_name, tier, service_name) (klio_server_backup_backups{%s})", matcher), + "{{cluster_name}} {{tier}} ({{service_name}})"), ).Decimals(0). Orientation(common.VizOrientationHorizontal). - Description("Number of PostgreSQL backups currently retained per cluster and tier.") + Description("Number of PostgreSQL backups currently retained per cluster and tier, with the Klio server " + + "that retains them.") } // latestBackupAgePanel shows the age of the latest retained backup. @@ -270,22 +272,28 @@ func serverHealthPanels() []sizedPanel { // snapshotPanels returns the Kopia base snapshot panels. func snapshotPanels() []sizedPanel { + // Three tiles per row: the per-server legend of Retained snapshots does not + // fit a narrower tile. return []sizedPanel{ - sized(smallPanelWidth, largePanelHeight, statPanel("Retained snapshots", units.Number, - query(fmt.Sprintf("sum by (cluster, tier) (%s)", - snapshotCluster(fmt.Sprintf("klio_server_backup_snapshots{%s}", snapshotMatcher))), "{{cluster}} {{tier}}"), + sized(largePanelWidth, largePanelHeight, statPanel("Retained snapshots", units.Number, + // Grouped by server too: two servers reading the same repository, such + // as a read-only one, would otherwise count its snapshots twice. + query(fmt.Sprintf("sum by (cluster, tier, service_name) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_snapshots{%s}", snapshotMatcher))), + "{{cluster}} {{tier}} ({{service_name}})"), ).Decimals(0). Orientation(common.VizOrientationHorizontal). - Description("Total kopia snapshots currently retained per cluster and tier.")), + Description("Total kopia snapshots currently retained per cluster and tier, with the Klio server "+ + "that retains them.")), - sized(smallPanelWidth, largePanelHeight, statPanel("Latest snapshot size", units.BytesIEC, + sized(largePanelWidth, largePanelHeight, statPanel("Latest snapshot size", units.BytesIEC, query(fmt.Sprintf("max by (cluster, tier) (%s)", snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_size_bytes{%s}", snapshotMatcher))), "{{cluster}} {{tier}}"), ).Orientation(common.VizOrientationHorizontal). Description("Size of the most recent base backup snapshot on Kopia, not accounting for deduplication.")), - sized(smallPanelWidth, largePanelHeight, statPanel("Latest snapshot files", units.Number, + sized(largePanelWidth, largePanelHeight, statPanel("Latest snapshot files", units.Number, query(fmt.Sprintf("max by (cluster, tier) (%s)", snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_files{%s}", snapshotMatcher))), "{{cluster}} {{tier}}"), @@ -293,7 +301,7 @@ func snapshotPanels() []sizedPanel { Orientation(common.VizOrientationHorizontal). Description("Number of files in the most recent base backup snapshot on Kopia.")), - sized(smallPanelWidth, largePanelHeight, statPanel("Latest snapshot dirs", units.Number, + sized(largePanelWidth, largePanelHeight, statPanel("Latest snapshot dirs", units.Number, query(fmt.Sprintf("max by (cluster, tier) (%s)", snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_dirs{%s}", snapshotMatcher))), "{{cluster}} {{tier}}"), @@ -301,14 +309,14 @@ func snapshotPanels() []sizedPanel { Orientation(common.VizOrientationHorizontal). Description("Number of directories in the most recent base backup snapshot on Kopia.")), - sized(smallPanelWidth, largePanelHeight, statPanel("Latest snapshot age", units.DurationInDaysHoursMinutesSeconds, + sized(largePanelWidth, largePanelHeight, statPanel("Latest snapshot age", units.DurationInDaysHoursMinutesSeconds, query(fmt.Sprintf("time() - max by (cluster, tier) (%s)", snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_timestamp_seconds{%s}", snapshotMatcher))), "{{cluster}} {{tier}}"), ).Orientation(common.VizOrientationHorizontal). Description("Age of the most recent base backup snapshot on Kopia, per cluster and tier.")), - sized(smallPanelWidth, largePanelHeight, statPanel("Oldest snapshot age", units.DurationInDaysHoursMinutesSeconds, + sized(largePanelWidth, largePanelHeight, statPanel("Oldest snapshot age", units.DurationInDaysHoursMinutesSeconds, query(fmt.Sprintf("time() - min by (cluster, tier) (%s)", snapshotCluster(fmt.Sprintf("klio_server_backup_oldest_snapshot_timestamp_seconds{%s}", snapshotMatcher))), "{{cluster}} {{tier}}"), diff --git a/operator/config/samples/opentelemetry/README.md b/operator/config/samples/opentelemetry/README.md index 8370021f..35f6390e 100644 --- a/operator/config/samples/opentelemetry/README.md +++ b/operator/config/samples/opentelemetry/README.md @@ -51,8 +51,10 @@ dashboard attribution bugs. `cluster-d` is backed by a *second*, independent Klio server that happens to also be named "klio-a" (see `multi/team-d`): since a Server's StatefulSet pod name is derived from the Server's own name alone, both servers' pods are named "klio-a-klio-0", giving them an -identical host_name label — a case the dashboard's `$server` variable -cannot disambiguate on its own. +identical host_name label. The dashboard's `$server` variable matches the +OpenTelemetry `service.name` instead, which each server sets through its own +`OTEL_SERVICE_NAME` (`klio-server-a` and `klio-server-a-team-d`), so the two +servers stay distinguishable. Every Klio server has its own certificate authority, signing only that server's own **client** certificates — no CA is ever shared between @@ -163,7 +165,7 @@ This deploys `default` (klio-a/cluster-a, klio-b/cluster-b), `team-c` (cluster-c) and `team-d` (a second, independent klio-a, and cluster-d) in one shot. -## Validating the Grafana dashboard with the "multi" sample +## Validating the Grafana dashboards with the "multi" sample Trigger one base backup per cluster (needed before any backup/snapshot panel has data to show): @@ -173,20 +175,33 @@ kubectl apply -f operator/config/samples/opentelemetry/multi/backups-example.yam ``` Once all backups complete and Prometheus has scraped a metrics-collection -cycle, open the Klio Grafana dashboard and confirm: +cycle, open the Klio Grafana dashboards. + +On the **Klio Clusters** dashboard, confirm: - The `$namespace` variable offers `default`, `team-c` and `team-d`. -- The `$server` variable offers only `klio-a-klio-0` and `klio-b-klio-0` - (the value is each server's pod hostname): Grafana's variable query - dedupes identical label values, so the two independent `klio-a` servers - (`default` and `team-d`) collapse into a single, ambiguous entry — this - is the disambiguation gap described above, not a deployment error. - The `$cluster` variable offers `cluster-a`, `cluster-b`, `cluster-c` and - `cluster-d`, and narrows correctly when `$namespace`/`$server` are - filtered (e.g. selecting `$namespace=team-c` should only ever offer - `cluster-c`). -- Per-cluster and per-server panels correctly attribute data instead of - aggregating everything together, and in particular that the two + `cluster-d`, and narrows when `$namespace` is filtered (e.g. selecting + `$namespace=team-c` should only ever offer `cluster-c`). +- With `$namespace=team-c` and `$cluster=All`, every panel, including the ones + built from the Klio server metrics, only shows `cluster-c`. +- Per-cluster panels correctly attribute data instead of aggregating + everything together. + +On the **Klio Servers** dashboard, confirm: + +- The `$namespace` variable offers the namespaces where the Klio servers run, + `default` and `team-d`, not `team-c`. +- The `$server` variable offers `klio-server-a`, `klio-server-a-team-d` and + `klio-server-b`: one entry per server, matched against the OpenTelemetry + `service.name` that each server's `OTEL_SERVICE_NAME` sets. The two + independent `klio-a` servers (`default` and `team-d`) are separate entries. +- With `$namespace=team-d` and `$server=All`, every panel only shows the + server running in `team-d`. +- The `$cluster` variable (*Served cluster*) offers the clusters backed by the + selected servers, including `cluster-c`, which runs in a different namespace + than `klio-b`. +- Per-server panels correctly attribute data, and in particular the two same-named `klio-a` servers (in `default` and in `team-d`) are not conflated.