feat: update exporter reliability and OpenShell 0.1 support

projects/tools/openshell-exporter/Dockerfile

@@ -1,5 +1,5 @@
-# syntax=docker/dockerfile:1.19@sha256:b6afd42430b15f2d2a4c5a02b919e98a525b785b1aaff16747d2f623364e39b6
-FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.26.8-bookworm@sha256:9fdc884aacc3bec89b20ffc69f4bb369c78210e3e4f600387b5128b12c199f81 AS builder
+# syntax=docker/dockerfile:1.28.0@sha256:bb22d9815c728170f72750f4e5b0d672e06176142e1d602c7e66c050100b7e5b
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.27.2-bookworm@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS builder
 ARG TARGETOS
 ARG TARGETARCH
 WORKDIR /src

projects/tools/openshell-exporter/README.md

@@ -8,19 +8,21 @@ provided as-is; use at your own risk.
 For other deployment methods and more advanced features, contact
 [delgadof@nvidia.com](mailto:delgadof@nvidia.com).
 
+Gateway API collection requires OpenShell >=0.1.0; the current target is v0.1.3.
+The first run below uses a sample file and needs no gateway.
+
 ## Quickstart: local process
 
-Use [Go 1.26.8](https://go.dev/doc/install) and a Bash-compatible shell. Get the
+Use [Go 1.27.2](https://go.dev/doc/install) and a Bash-compatible shell. Get the
 source and enter the project directory:
 
 ```sh
 git clone https://github.com/NVIDIA/OpenShell-Research.git
 cd OpenShell-Research/projects/tools/openshell-exporter
 ```
 
-The build
-downloads public dependencies. This first run uses a synthetic OCSF record; it
-needs no OpenShell gateway, credentials, or model service.
+The build downloads public dependencies. The synthetic sample needs no
+credentials or model service.
 
 ```sh
 go build -o bin/openshell-event-exporter ./cmd/openshell-event-exporter
@@ -59,15 +61,15 @@ The image health check probes `127.0.0.1:13133`; retain that port inside the
 container. Its optional `healthcheck --writable DIRECTORY ...` command checks
 persistence-directory permissions.
 
-Use the same health and output checks. With existing checkpoints the example is
-not replayed; append a new record to test more delivery. Stop with
+Use the same health and output checks as the local process. Saved checkpoints
+prevent replay of the example; append a new record to test more delivery. Stop with
 `docker stop openshell-exporter`. Build locally; no published image is required.
 Linux AMD64/ARM64 and native macOS ARM64 are build targets; Docker on Apple Silicon
 runs the Linux ARM64 image through Docker Desktop.
 
-## Connect real inputs and destinations
+## Choose real inputs
 
-For actual OCSF logs, set `receivers.file_log/ocsf.include` in [config.yaml](config.yaml)
+For existing OCSF JSONL, set `receivers.file_log/ocsf.include` in [config.yaml](config.yaml)
 to the readable source file glob. In Docker, mount the source directory read-only
 and use its container path in the glob. Mount the directory, so rotated files remain
 visible. Logs inside an OpenShell sandbox must first be made accessible to the
@@ -79,11 +81,62 @@ filtering, and delivery. Select the needed pipelines and mount their credentials
 and persistent storage. The template is a reference, not a ready-to-run deployment.
 `openshell-event-exporter components` lists the full component set.
 
-For an HTTPS receiver that expects **bare OCSF inside CloudEvents `data`**, use
-[http.config.yaml](http.config.yaml). It reads the same input files and sends
-CloudEvents batches with a persistent retry queue. This configuration bypasses
-normalization and redaction; source content is forwarded as supplied. Provision
-`secrets/destination-token` (raw bearer token) and `secrets/destination-ca.pem`
+### Gateway and supervisor logs (v0.1.3)
+
+The default config includes two additional inputs:
+
+| Source | Path on the host |
+|---|---|
+| Gateway JSONL | `input/gateways/*/gateway-ocsf.jsonl*` |
+| Captured supervisor stream | `input/supervisors/*/stderr.log*` |
+
+Docker mounts these below `/work/input`. Keep replicas in separate directories,
+and retain rotation segments and checkpoints.
+
+Enable gateway JSONL in its `gateway.toml`, using a writable persistent path:
+
+```toml
+[openshell.gateway.ocsf_log]
+path = "/path/to/input/gateways/replica-1/gateway-ocsf.jsonl"
+rotation = "daily"
+max_files = 7
+```
+
+Restart the gateway after changing its configuration. Gateway-wide records may
+have no sandbox association. This output does not include supervisor OCSF.
+
+For supervisor records, enable `ocsf_json_enabled` with an authorized OpenShell
+administrator, then capture the raw platform log stream:
+
+```sh
+openshell settings set --global --key ocsf_json_enabled --value true
+mkdir -p input/supervisors/sandbox-1
+docker logs --follow SUPERVISOR_CONTAINER > input/supervisors/sandbox-1/stderr.log 2>&1
+```
+
+Use `podman logs` in place of `docker logs` when appropriate. This command stays
+attached; run the exporter separately. The parser selects `OCSF-JSON` lines,
+decodes the payload, and ignores shorthand copies. It expects the unwrapped
+platform stream, not Docker JSON/CRI storage wrappers or an extra timestamp
+prefix. Collect one structured path per producer; do not also ship that supervisor's JSONL files.
+A replayed platform-log capture can duplicate events; `correlation.ocsf_event_uid`
+retains the native ID for downstream correlation without changing exporter IDs.
+
+Keep `ocsf_schema_version` unset: this exporter validates native 1.8.0 and replay
+1.7.0, not the lossy 1.1/1.3 downgrade. Both upstream sinks are best-effort; source
+queue loss or deleted runtime logs cannot be recovered by exporter checkpoints.
+
+## Send to an HTTPS receiver
+
+[http.config.yaml](http.config.yaml) reads the original JSONL inputs and sends
+**bare OCSF inside CloudEvents `data`**, using a persistent retry queue.
+It bypasses normalization and redaction: source content is forwarded as supplied.
+For normalized, redacted delivery, copy `processors.openshell` from `config.yaml`
+into `http.config.yaml`, then set `service.pipelines.logs.processors` to
+`[memory_limiter, openshell]`. Your receiver must read the OCSF record from
+`data.original`.
+
+Provision `secrets/destination-token` (raw bearer token) and `secrets/destination-ca.pem`
 (the trusted issuing CA), restrict their access, then run:
 
 ```sh
@@ -100,11 +153,8 @@ Docker Desktop can reach a host receiver using `host.docker.internal`; Linux
 Docker Engine can add `--add-host host.docker.internal:host-gateway`. The receiver
 must listen on a reachable interface. Bearer authentication requires HTTPS.
 
-For normalized, redacted HTTP delivery, add `openshell` from `config.yaml` to that
-pipeline after `memory_limiter`. The receiver must then read the redacted OCSF
-record from `data.original`. A 2xx acknowledges a batch; retryable failures retry
-and may produce duplicates. Consumers should deduplicate by `(source,id)`.
-Raw mode hashes the body for identity, so identical source bodies share an ID.
+A 2xx acknowledges a batch; retryable failures may produce duplicates. Consumers should deduplicate by `(source,id)`.
+In raw mode, identical source bodies share an ID.
 Append a new input record when switching configurations; saved checkpoints do not
 replay records already read.
 Health and local recovery output alone do not establish successful remote delivery.
@@ -119,9 +169,8 @@ and exporter logs. The exporter observes evidence; it does not approve policy ch
 `go install ./cmd/openshell-event-exporter` installs from this checkout. The module
 path is `github.com/NVIDIA/OpenShell-Research/projects/tools/openshell-exporter`; a remote
 `go install ...@version` needs a public release tagged
-`projects/tools/openshell-exporter/v<version>`. No such release is published by
-this change.
-Custom Collectors can import the component packages and register their `NewFactory`.
+`projects/tools/openshell-exporter/v<version>`. No such release is published by this change.
+Custom Collectors can import component packages and register their `NewFactory`.
 
 ```sh
 go mod verify
@@ -137,7 +186,7 @@ exporter does not compile those packages: `go list -deps ./...` contains no
 dependencies change.
 
 The source is extracted from exporter snapshot
-`6a440f05249ac05aa9b9ef8acada4c909ec86bc8`. This distribution retains the Go
+`75a41db6be6cc6ad160e3619ad09270d1cdb5b1d`. This distribution retains the Go
 implementation and component tests; deployment tooling lives separately.
 Preserve LICENSE and NOTICE when redistributing. Synthetic local and container
 checks do not qualify live gateway or external application behavior.

projects/tools/openshell-exporter/cmd/healthcheck/main.go

@@ -14,8 +14,19 @@ import (
 
 func main() {
 	if len(os.Args) > 1 {
+		if os.Args[1] == "--status" && len(os.Args) <= 3 {
+			endpoint := "http://127.0.0.1:8888/metrics"
+			if len(os.Args) == 3 {
+				endpoint = os.Args[2]
+			}
+			if err := deliveryStatus(context.Background(), os.Stdout, endpoint); err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(1)
+			}
+			return
+		}
 		if os.Args[1] != "--writable" || len(os.Args) < 3 {
-			fmt.Fprintln(os.Stderr, "usage: healthcheck [--writable DIRECTORY ...]")
+			fmt.Fprintln(os.Stderr, "usage: healthcheck [--writable DIRECTORY ... | --status [METRICS_URL]]")
 			os.Exit(2)
 		}
 		for _, directory := range os.Args[2:] {

projects/tools/openshell-exporter/cmd/healthcheck/status.go

@@ -0,0 +1,173 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type statusMetric struct {
+	name      string
+	label     string
+	timestamp bool
+}
+
+var statusSections = []struct {
+	name    string
+	metrics []statusMetric
+}{
+	{"Sources (observation is not destination delivery)", []statusMetric{
+		{"openshell_exporter_source_configured", "Configured profile", false},
+		{"openshell_exporter_source_enabled", "Enabled receiver", false},
+		{"openshell_exporter_source_observed", "Observed profile", false},
+		{"openshell_exporter_source_last_success_unixtime", "Last observation (UTC)", true},
+	}},
+	{"CloudEvents delivery (HTTP acknowledgement only)", []statusMetric{
+		{"openshell_exporter_delivery_events", "Acknowledged events", false},
+		{"openshell_exporter_destination_last_success_unixtime", "Last acknowledgement (UTC)", true},
+		{"openshell_exporter_delivery_retryable_failures", "Retryable failures", false},
+		{"openshell_exporter_delivery_permanent_rejections", "Permanently rejected responses", false},
+		{"openshell_exporter_events_oversized", "Oversized events", false},
+	}},
+	{"Collector queues (units follow configured queue sizer; excludes in-flight sends)", []statusMetric{
+		{"otelcol_exporter_queue_size", "Backlog", false},
+		{"otelcol_exporter_queue_capacity", "Capacity", false},
+		{"otelcol_exporter_enqueue_failed_log_records", "Enqueue failures", false},
+		{"otelcol_exporter_send_failed_log_records", "Send failures", false},
+	}},
+	{"WatchSandbox (non-resumable)", []statusMetric{
+		{"openshell_exporter_stream_reconnects", "Reconnects", false},
+		{"openshell_exporter_stream_gap_warnings", "Gap risks", false},
+		{"openshell_exporter_stream_backpressure_active", "Waiting to retry downstream", false},
+		{"openshell_exporter_stream_unacknowledged_records", "Unacknowledged source records", false},
+	}},
+	{"Storage (availability is a probe, not proof of durable delivery)", []statusMetric{
+		{"openshell_exporter_storage_configured", "Configured", false},
+		{"openshell_exporter_storage_available", "Available (1=yes, 0=no)", false},
+		{"openshell_exporter_storage_free_bytes", "Free bytes", false},
+		{"openshell_exporter_storage_free_inodes", "Free inodes", false},
+		{"openshell_exporter_storage_scan_truncated", "State scan truncated (1=yes)", false},
+	}},
+}
+
+// deliveryStatus is deliberately observational: neither zero backlog nor an HTTP
+// acknowledgement proves that every source event reached durable destination storage.
+func deliveryStatus(ctx context.Context, output io.Writer, endpoint string) error {
+	u, err := url.Parse(endpoint)
+	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil {
+		return fmt.Errorf("status endpoint must be an HTTP(S) metrics URL without embedded credentials")
+	}
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
+	response, err := client.Do(request)
+	if err != nil {
+		return fmt.Errorf("read metrics: %w", err)
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("metrics endpoint returned HTTP %d", response.StatusCode)
+	}
+	const maxBytes = 4 << 20
+	body, err := io.ReadAll(io.LimitReader(response.Body, maxBytes+1))
+	if err != nil {
+		return fmt.Errorf("read metrics body: %w", err)
+	}
+	if len(body) > maxBytes {
+		return fmt.Errorf("metrics response exceeds %d bytes", maxBytes)
+	}
+	return renderStatus(output, string(body))
+}
+
+func renderStatus(output io.Writer, body string) error {
+	wanted := map[string]statusMetric{}
+	for _, section := range statusSections {
+		for _, m := range section.metrics {
+			wanted[m.name] = m
+		}
+	}
+	values := map[string][]string{}
+	scanner := bufio.NewScanner(strings.NewReader(body))
+	scanner.Buffer(make([]byte, 4096), 1<<20)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		end := strings.IndexAny(line, "{ \t")
+		if end < 0 {
+			continue
+		}
+		name := line[:end]
+		// Accommodate Prometheus exporters that append the conventional counter suffix.
+		name = strings.TrimSuffix(name, "_total")
+		m, ok := wanted[name]
+		if !ok {
+			continue
+		}
+		rest := strings.TrimSpace(line[end:])
+		labels := ""
+		if strings.HasPrefix(rest, "{") {
+			end = strings.LastIndexByte(rest, '}')
+			if end < 0 {
+				return fmt.Errorf("malformed labels for %s", name)
+			}
+			labels, rest = rest[:end+1], rest[end+1:]
+		}
+		fields := strings.Fields(rest)
+		if len(fields) < 1 || len(fields) > 2 {
+			return fmt.Errorf("malformed sample for %s", name)
+		}
+		value, err := strconv.ParseFloat(fields[0], 64)
+		if err != nil {
+			return fmt.Errorf("invalid value for %s", name)
+		}
+		formatted := fields[0]
+		if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 {
+			formatted = "unavailable"
+		} else if m.timestamp {
+			if value == 0 || value >= float64(math.MaxInt64) {
+				formatted = "not recorded"
+			} else {
+				formatted = time.Unix(int64(value), 0).UTC().Format(time.RFC3339)
+			}
+		}
+		values[name] = append(values[name], fmt.Sprintf("  %s %s: %s\n", m.label, labels, formatted))
+	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("parse metrics: %w", err)
+	}
+	var result strings.Builder
+	result.WriteString("Delivery snapshot. Counters are cumulative since process start; compare successive snapshots.\nMissing metrics are unavailable, never assumed zero. This is not a delivery health verdict.\n")
+	for _, section := range statusSections {
+		fmt.Fprintf(&result, "\n%s\n", section.name)
+		for _, m := range section.metrics {
+			rows := values[m.name]
+			if len(rows) == 0 {
+				fmt.Fprintf(&result, "  %s: unavailable\n", m.label)
+				continue
+			}
+			sort.Strings(rows)
+			for _, row := range rows {
+				result.WriteString(row)
+			}
+		}
+	}
+	_, err := io.WriteString(output, result.String())
+	return err
+}