fix(observability): scope the "All" of the cluster and server variables

documentation/web/docs/user/grafana-dashboards.md

@@ -31,7 +31,7 @@ The **Klio Clusters** dashboard is split into row sections:
   and the backup success ratio.
 
 - **Retained backups** — the retention window of the physical PostgreSQL
-  backups the Klio server keeps for each cluster: counts by tier,
+  backups the Klio server keeps for each cluster: counts by tier and server,
   latest/oldest backup age, start/end LSN and PostgreSQL timeline.
 - **Backup maintenance** — tier-2 relay, backup verification and
   maintenance run totals per cluster.

observability/grafana/build.go

@@ -42,12 +42,19 @@ func clusterDashboard() *dashboard.DashboardBuilder {
 	builder := newDashboard("klio-clusters", "Klio Clusters",
 		"Backups, WAL archiving and WAL restore of the PostgreSQL clusters backed up by Klio.").
 		WithVariable(labelVariable("namespace", "Namespace",
-			`label_values({__name__=~"klio_(plugin|client)_.+"}, k8s_namespace_name)`)).
+			`label_values({__name__=~"klio_(plugin|client)_.+"}, k8s_namespace_name)`).
+			AllValue(matchAny)).
+		// "All" expands to the clusters of the selected namespaces. Matching any
+		// value would also select clusters from other namespaces on the panels
+		// that are not filtered by namespace. Refreshed with the time range, so
+		// "All" picks up clusters that appear after the dashboard loads.
 		WithVariable(labelVariable("cluster", "Cluster",
-			`label_values({__name__=~"klio_(plugin|client)_.+",k8s_namespace_name=~"$namespace"}, cluster_name)`)).
+			`label_values({__name__=~"klio_(plugin|client)_.+",k8s_namespace_name=~"$namespace"}, cluster_name)`).
+			Refresh(dashboard.VariableRefreshOnTimeRangeChanged)).
 		WithVariable(labelVariable("pod", "Instance",
 			`label_values({__name__=~"klio_(plugin|client)_.+",k8s_namespace_name=~"$namespace",`+
-				`cluster_name=~"$cluster"}, k8s_pod_name)`))
+				`cluster_name=~"$cluster"}, k8s_pod_name)`).
+			AllValue(matchAny))
 
 	layoutDashboard(builder,
 		"*Namespace* is where the PostgreSQL cluster runs. *Cluster* lists every cluster that emits Klio "+
@@ -97,11 +104,19 @@ func serverDashboard() *dashboard.DashboardBuilder {
 	builder := newDashboard("klio-servers", "Klio Servers",
 		"Health, served clusters, Kopia snapshots and WAL processing of the Klio servers.").
 		WithVariable(labelVariable("namespace", "Server namespace",
-			`label_values(klio_server_uptime_seconds, k8s_namespace_name)`)).
+			`label_values(klio_server_uptime_seconds, k8s_namespace_name)`).
+			AllValue(matchAny)).
+		// "All" expands to the servers of the selected namespaces, for the same
+		// reason, and refreshed, as the cluster variable of the Klio Clusters
+		// dashboard.
 		WithVariable(labelVariable("server", "Server",
-			`label_values(klio_server_uptime_seconds{k8s_namespace_name=~"$namespace"}, service_name)`)).
+			`label_values(klio_server_uptime_seconds{k8s_namespace_name=~"$namespace"}, service_name)`).
+			Refresh(dashboard.VariableRefreshOnTimeRangeChanged)).
+		// Not sourced from klio_server_wal_written_total: a server that receives
+		// no WAL, such as a read-only one, would list no clusters.
 		WithVariable(labelVariable("cluster", "Served cluster",
-			`label_values(klio_server_wal_written_total{service_name=~"$server"}, cluster_name)`))
+			`label_values({__name__=~"klio_server_.+",service_name=~"$server"}, cluster_name)`).
+			AllValue(matchAny))
 
 	layoutDashboard(builder,
 		"*Server namespace* is where the Klio server runs, not where the clusters it backs run. *Server* "+
@@ -219,8 +234,14 @@ func layoutSection(builder *dashboard.DashboardBuilder, y *int, title string, pa
 	}
 }
 
+// matchAny is the "All" value of the variables whose "All" must match any
+// series, whatever the values listed by the variable.
+const matchAny = ".+"
+
 // labelVariable builds a multi-value Prometheus label_values template variable
 // that defaults to "All" so the dashboard shows every series until filtered.
+// Unless the caller sets AllValue, "All" expands to the values the variable
+// lists.
 func labelVariable(name, label, query string) *dashboard.QueryVariableBuilder {
 	return dashboard.NewQueryVariableBuilder(name).
 		Label(label).
@@ -230,7 +251,6 @@ func labelVariable(name, label, query string) *dashboard.QueryVariableBuilder {
 		Sort(dashboard.VariableSortAlphabeticalAsc).
 		Multi(true).
 		IncludeAll(true).
-		AllValue(".+").
 		Current(dashboard.VariableOption{
 			Selected: cog.ToPtr(true),
 			Text:     dashboard.StringOrArrayOfString{String: cog.ToPtr("All")},

observability/grafana/dashboards/klio-clusters.json

@@ -729,14 +729,14 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "sum by (cluster_name, tier) (klio_server_backup_backups{cluster_name=~\"$cluster\"})",
+          "expr": "sum by (cluster_name, tier, service_name) (klio_server_backup_backups{cluster_name=~\"$cluster\"})",
           "instant": false,
-          "legendFormat": "{{cluster_name}} {{tier}}",
+          "legendFormat": "{{cluster_name}} {{tier}} ({{service_name}})",
           "range": true
         }
       ],
       "title": "Retained backups",
-      "description": "Number of PostgreSQL backups currently retained per cluster and tier.",
+      "description": "Number of PostgreSQL backups currently retained per cluster and tier, with the Klio server that retains them.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -2940,10 +2940,9 @@
         },
         "multi": true,
         "allowCustomValue": true,
-        "refresh": 1,
+        "refresh": 2,
         "sort": 1,
         "includeAll": true,
-        "allValue": ".+",
         "auto": false,
         "auto_min": "10s",
         "auto_count": 30

observability/grafana/dashboards/klio-servers.json

@@ -256,14 +256,14 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "sum by (cluster_name, tier) (klio_server_backup_backups{service_name=~\"$server\",cluster_name=~\"$cluster\"})",
+          "expr": "sum by (cluster_name, tier, service_name) (klio_server_backup_backups{service_name=~\"$server\",cluster_name=~\"$cluster\"})",
           "instant": false,
-          "legendFormat": "{{cluster_name}} {{tier}}",
+          "legendFormat": "{{cluster_name}} {{tier}} ({{service_name}})",
           "range": true
         }
       ],
       "title": "Retained backups",
-      "description": "Number of PostgreSQL backups currently retained per cluster and tier.",
+      "description": "Number of PostgreSQL backups currently retained per cluster and tier, with the Klio server that retains them.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -501,22 +501,22 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "sum by (cluster, tier) (label_replace(klio_server_backup_snapshots{service_name=~\"$server\",snapshot_source=~\"[^@]*@$cluster:.*\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))",
+          "expr": "sum by (cluster, tier, service_name) (label_replace(klio_server_backup_snapshots{service_name=~\"$server\",snapshot_source=~\"[^@]*@$cluster:.*\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))",
           "instant": false,
-          "legendFormat": "{{cluster}} {{tier}}",
+          "legendFormat": "{{cluster}} {{tier}} ({{service_name}})",
           "range": true
         }
       ],
       "title": "Retained snapshots",
-      "description": "Total kopia snapshots currently retained per cluster and tier.",
+      "description": "Total kopia snapshots currently retained per cluster and tier, with the Klio server that retains them.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
         "uid": "${datasource}"
       },
       "gridPos": {
         "h": 8,
-        "w": 4,
+        "w": 8,
         "x": 0,
         "y": 25
       },
@@ -574,8 +574,8 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 4,
-        "x": 4,
+        "w": 8,
+        "x": 8,
         "y": 25
       },
       "repeatDirection": "h",
@@ -631,8 +631,8 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 4,
-        "x": 8,
+        "w": 8,
+        "x": 16,
         "y": 25
       },
       "repeatDirection": "h",
@@ -689,9 +689,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 4,
-        "x": 12,
-        "y": 25
+        "w": 8,
+        "x": 0,
+        "y": 33
       },
       "repeatDirection": "h",
       "options": {
@@ -747,9 +747,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 4,
-        "x": 16,
-        "y": 25
+        "w": 8,
+        "x": 8,
+        "y": 33
       },
       "repeatDirection": "h",
       "options": {
@@ -804,9 +804,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 4,
-        "x": 20,
-        "y": 25
+        "w": 8,
+        "x": 16,
+        "y": 33
       },
       "repeatDirection": "h",
       "options": {
@@ -846,7 +846,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 33
+        "y": 41
       },
       "id": 0,
       "panels": []
@@ -858,7 +858,7 @@
         "h": 2,
         "w": 24,
         "x": 0,
-        "y": 34
+        "y": 42
       },
       "repeatDirection": "h",
       "options": {
@@ -911,7 +911,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 36
+        "y": 44
       },
       "repeatDirection": "h",
       "options": {
@@ -982,7 +982,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 36
+        "y": 44
       },
       "repeatDirection": "h",
       "options": {
@@ -1053,7 +1053,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 44
+        "y": 52
       },
       "repeatDirection": "h",
       "options": {
@@ -1124,7 +1124,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 44
+        "y": 52
       },
       "repeatDirection": "h",
       "options": {
@@ -1212,10 +1212,9 @@
         },
         "multi": true,
         "allowCustomValue": true,
-        "refresh": 1,
+        "refresh": 2,
         "sort": 1,
         "includeAll": true,
-        "allValue": ".+",
         "auto": false,
         "auto_min": "10s",
         "auto_count": 30
@@ -1225,7 +1224,7 @@
         "name": "cluster",
         "label": "Served cluster",
         "skipUrlSync": false,
-        "query": "label_values(klio_server_wal_written_total{service_name=~\"$server\"}, cluster_name)",
+        "query": "label_values({__name__=~\"klio_server_.+\",service_name=~\"$server\"}, cluster_name)",
         "datasource": {
           "type": "prometheus",
           "uid": "${datasource}"

observability/grafana/server.go

@@ -41,14 +41,16 @@ func snapshotCluster(selector string) string {
 // The server panels show the metrics of the Klio server (`klio_server_*`).
 // Panels shown on both dashboards take their series matcher.
 
-// retainedBackupsPanel counts the retained backups.
+// retainedBackupsPanel counts the retained backups, naming the server that
+// retains them.
 func retainedBackupsPanel(matcher string) *stat.PanelBuilder {
 	return statPanel("Retained backups", units.Number,
-		query(fmt.Sprintf("sum by (cluster_name, tier) (klio_server_backup_backups{%s})", matcher),
-			"{{cluster_name}} {{tier}}"),
+		query(fmt.Sprintf("sum by (cluster_name, tier, service_name) (klio_server_backup_backups{%s})", matcher),
+			"{{cluster_name}} {{tier}} ({{service_name}})"),
 	).Decimals(0).
 		Orientation(common.VizOrientationHorizontal).
-		Description("Number of PostgreSQL backups currently retained per cluster and tier.")
+		Description("Number of PostgreSQL backups currently retained per cluster and tier, with the Klio server " +
+			"that retains them.")
 }
 
 // latestBackupAgePanel shows the age of the latest retained backup.
@@ -270,45 +272,51 @@ func serverHealthPanels() []sizedPanel {
 
 // snapshotPanels returns the Kopia base snapshot panels.
 func snapshotPanels() []sizedPanel {
+	// Three tiles per row: the per-server legend of Retained snapshots does not
+	// fit a narrower tile.
 	return []sizedPanel{
-		sized(smallPanelWidth, largePanelHeight, statPanel("Retained snapshots", units.Number,
-			query(fmt.Sprintf("sum by (cluster, tier) (%s)",
-				snapshotCluster(fmt.Sprintf("klio_server_backup_snapshots{%s}", snapshotMatcher))), "{{cluster}} {{tier}}"),
+		sized(largePanelWidth, largePanelHeight, statPanel("Retained snapshots", units.Number,
+			// Grouped by server too: two servers reading the same repository, such
+			// as a read-only one, would otherwise count its snapshots twice.
+			query(fmt.Sprintf("sum by (cluster, tier, service_name) (%s)",
+				snapshotCluster(fmt.Sprintf("klio_server_backup_snapshots{%s}", snapshotMatcher))),
+				"{{cluster}} {{tier}} ({{service_name}})"),
 		).Decimals(0).
 			Orientation(common.VizOrientationHorizontal).
-			Description("Total kopia snapshots currently retained per cluster and tier.")),
+			Description("Total kopia snapshots currently retained per cluster and tier, with the Klio server "+
+				"that retains them.")),
 
-		sized(smallPanelWidth, largePanelHeight, statPanel("Latest snapshot size", units.BytesIEC,
+		sized(largePanelWidth, largePanelHeight, statPanel("Latest snapshot size", units.BytesIEC,
 			query(fmt.Sprintf("max by (cluster, tier) (%s)",
 				snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_size_bytes{%s}", snapshotMatcher))),
 				"{{cluster}} {{tier}}"),
 		).Orientation(common.VizOrientationHorizontal).
 			Description("Size of the most recent base backup snapshot on Kopia, not accounting for deduplication.")),
 
-		sized(smallPanelWidth, largePanelHeight, statPanel("Latest snapshot files", units.Number,
+		sized(largePanelWidth, largePanelHeight, statPanel("Latest snapshot files", units.Number,
 			query(fmt.Sprintf("max by (cluster, tier) (%s)",
 				snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_files{%s}", snapshotMatcher))),
 				"{{cluster}} {{tier}}"),
 		).Decimals(0).
 			Orientation(common.VizOrientationHorizontal).
 			Description("Number of files in the most recent base backup snapshot on Kopia.")),
 
-		sized(smallPanelWidth, largePanelHeight, statPanel("Latest snapshot dirs", units.Number,
+		sized(largePanelWidth, largePanelHeight, statPanel("Latest snapshot dirs", units.Number,
 			query(fmt.Sprintf("max by (cluster, tier) (%s)",
 				snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_dirs{%s}", snapshotMatcher))),
 				"{{cluster}} {{tier}}"),
 		).Decimals(0).
 			Orientation(common.VizOrientationHorizontal).
 			Description("Number of directories in the most recent base backup snapshot on Kopia.")),
 
-		sized(smallPanelWidth, largePanelHeight, statPanel("Latest snapshot age", units.DurationInDaysHoursMinutesSeconds,
+		sized(largePanelWidth, largePanelHeight, statPanel("Latest snapshot age", units.DurationInDaysHoursMinutesSeconds,
 			query(fmt.Sprintf("time() - max by (cluster, tier) (%s)",
 				snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_timestamp_seconds{%s}", snapshotMatcher))),
 				"{{cluster}} {{tier}}"),
 		).Orientation(common.VizOrientationHorizontal).
 			Description("Age of the most recent base backup snapshot on Kopia, per cluster and tier.")),
 
-		sized(smallPanelWidth, largePanelHeight, statPanel("Oldest snapshot age", units.DurationInDaysHoursMinutesSeconds,
+		sized(largePanelWidth, largePanelHeight, statPanel("Oldest snapshot age", units.DurationInDaysHoursMinutesSeconds,
 			query(fmt.Sprintf("time() - min by (cluster, tier) (%s)",
 				snapshotCluster(fmt.Sprintf("klio_server_backup_oldest_snapshot_timestamp_seconds{%s}", snapshotMatcher))),
 				"{{cluster}} {{tier}}"),