From 4fd39f020123b7f4c3973c91db175a24e67e8a0c Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Sat, 31 Oct 2026 14:23:59 +0000 Subject: [PATCH 01/16] test: add specs for CrowdSec hub outage behavior --- tests/security/crowdsec-hub-offline.spec.ts | 231 ++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 tests/security/crowdsec-hub-offline.spec.ts diff --git a/tests/security/crowdsec-hub-offline.spec.ts b/tests/security/crowdsec-hub-offline.spec.ts new file mode 100644 index 000000000..6283448f0 --- /dev/null +++ b/tests/security/crowdsec-hub-offline.spec.ts @@ -0,0 +1,231 @@ +/** + * CrowdSec hub offline presets E2E Tests + * + * When the CrowdSec hub cannot be reached, the presets list is answered from the + * built-in and downloaded sources and says so: + * - `hub_offline: true` shows a passive, muted note (no banner, no button) + * - hub-backed rows that are not cached and report `available: false` are shown disabled + * - auto-select skips those rows and picks the first selectable one, or nothing + * - `hub_offline: false` shows no note + * + * The E2E container has no cscli or hub, so the preset list is route-mocked. + * Every case is `test.fixme` until the UI lands. + */ + +import type { Page } from "@playwright/test"; +import { test, expect, loginUser } from "../fixtures/auth-fixtures"; +import { waitForLoadingComplete } from "../utils/wait-helpers"; +import { PRESETS_ROUTE, fulfillJson } from "../utils/crowdsec-stubs"; + +const PRESETS_PATH = "/api/v1/admin/crowdsec/presets"; +const HUB_OFFLINE_NOTE = + "Showing built-in and downloaded presets. The hub list was not loaded."; + +interface PresetRowFixture { + slug: string; + title: string; + source: "hub" | "charon-curated"; + requires_hub: boolean; + available: boolean; + cached: boolean; +} + +/** Hub-backed row that is not cached and cannot be fetched while the hub is offline. */ +const OFFLINE_HUB_ROW: PresetRowFixture = { + slug: "crowdsecurity/e2e-offline-hub-preset", + title: "E2E Offline Hub Preset", + source: "hub", + requires_hub: true, + available: false, + cached: false, +}; + +/** Built-in preset that needs no hub, so it stays selectable offline. */ +const SELECTABLE_ROW: PresetRowFixture = { + slug: "crowdsecurity/e2e-selectable-preset", + title: "E2E Selectable Preset", + source: "charon-curated", + requires_hub: false, + available: true, + cached: false, +}; + +/** Second hub-backed row that is not cached and cannot be fetched while the hub is offline. */ +const SECOND_OFFLINE_HUB_ROW: PresetRowFixture = { + slug: "crowdsecurity/e2e-second-offline-hub-preset", + title: "E2E Second Offline Hub Preset", + source: "hub", + requires_hub: true, + available: false, + cached: false, +}; + +/** + * Stubs GET presets with the given rows and the `hub_offline` flag, and records the + * slugs of every pull request so tests can assert that no unavailable row was pulled. + */ +async function stubPresetList( + page: Page, + presets: PresetRowFixture[], + hubOffline: boolean, +): Promise<{ pullSlugs: string[] }> { + const pullSlugs: string[] = []; + await page.route(PRESETS_ROUTE, async (route) => { + if (route.request().method() !== "GET") { + await route.fallback(); + return; + } + await fulfillJson(route, { presets, hub_offline: hubOffline }); + }); + await page.route("**/api/v1/admin/crowdsec/presets/pull", async (route) => { + if (route.request().method() === "POST") { + const body = route.request().postDataJSON() as { slug?: string }; + pullSlugs.push(body.slug ?? ""); + } + await route.fallback(); + }); + return { pullSlugs }; +} + +/** + * Loads the CrowdSec page and waits for the presets response the test stubbed, so + * assertions run against the list the page actually received. + */ +async function loadPresetsPage(page: Page): Promise { + const presetsResponse = page.waitForResponse( + (response) => + new URL(response.url()).pathname === PRESETS_PATH && + response.request().method() === "GET", + ); + await page.goto("/security/crowdsec"); + await presetsResponse; + await waitForLoadingComplete(page); +} + +/** The preset row (role button) whose accessible name contains the title. */ +function presetRow(page: Page, title: string) { + return page.getByRole("button", { name: new RegExp(title, "i") }); +} + +test.describe("CrowdSec hub offline presets @security", () => { + test.beforeEach(async ({ page, adminUser }) => { + await loginUser(page, adminUser); + await waitForLoadingComplete(page); + }); + + test.fixme("should show the muted hub offline note with no banner or button when hub_offline is true", async ({ + page, + }) => { + await stubPresetList(page, [SELECTABLE_ROW, OFFLINE_HUB_ROW], true); + + await test.step("Load the presets list served without the hub", async () => { + await loadPresetsPage(page); + }); + + await test.step("Verify the offline note is shown with its text", async () => { + const note = page.getByTestId("preset-hub-offline"); + await expect(note).toBeVisible(); + await expect(note).toHaveText(HUB_OFFLINE_NOTE); + }); + + await test.step("Verify the note is passive: no button inside it and no hub unavailable banner", async () => { + await expect( + page.getByTestId("preset-hub-offline").getByRole("button"), + ).toHaveCount(0); + await expect(page.getByTestId("preset-hub-unavailable")).toHaveCount(0); + await expect( + page.getByRole("button", { name: "Retry", exact: true }), + ).toHaveCount(0); + }); + }); + + test.fixme("should show unavailable hub rows disabled and auto-select the first selectable row", async ({ + page, + }) => { + const { pullSlugs } = await stubPresetList( + page, + [OFFLINE_HUB_ROW, SELECTABLE_ROW], + true, + ); + + await test.step("Load the presets list with one unavailable hub row listed first", async () => { + await loadPresetsPage(page); + }); + + await test.step("Verify the unavailable row is shown disabled and not selected", async () => { + const offlineRow = presetRow(page, OFFLINE_HUB_ROW.title); + await expect(offlineRow).toBeVisible(); + await expect(offlineRow).toHaveAttribute("aria-disabled", "true"); + await expect(offlineRow).toHaveAttribute("aria-pressed", "false"); + }); + + await test.step("Verify auto-select skipped the unavailable row and picked the first selectable row", async () => { + await expect(presetRow(page, SELECTABLE_ROW.title)).toHaveAttribute( + "aria-pressed", + "true", + ); + }); + + await test.step("Verify clicking the disabled row does not select it", async () => { + // Force the click past Playwright's aria-disabled actionability check so the handler is exercised. + await presetRow(page, OFFLINE_HUB_ROW.title).click({ force: true }); + await expect(presetRow(page, OFFLINE_HUB_ROW.title)).toHaveAttribute( + "aria-pressed", + "false", + ); + await expect(presetRow(page, SELECTABLE_ROW.title)).toHaveAttribute( + "aria-pressed", + "true", + ); + expect(pullSlugs).not.toContain(OFFLINE_HUB_ROW.slug); + }); + }); + + test.fixme("should select nothing and show no error when every row is unavailable offline", async ({ + page, + }) => { + const { pullSlugs } = await stubPresetList( + page, + [OFFLINE_HUB_ROW, SECOND_OFFLINE_HUB_ROW], + true, + ); + + await test.step("Load the presets list where no row can be used", async () => { + await loadPresetsPage(page); + }); + + await test.step("Verify both rows are disabled and nothing is selected", async () => { + for (const row of [OFFLINE_HUB_ROW, SECOND_OFFLINE_HUB_ROW]) { + await expect(presetRow(page, row.title)).toHaveAttribute( + "aria-disabled", + "true", + ); + await expect(presetRow(page, row.title)).toHaveAttribute( + "aria-pressed", + "false", + ); + } + }); + + await test.step("Verify no error is shown and no pull was sent for either row", async () => { + await expect(page.getByRole("alert")).toHaveCount(0); + await expect(page.getByTestId("preset-hub-unavailable")).toHaveCount(0); + expect(pullSlugs).toEqual([]); + }); + }); + + test.fixme("should show no hub offline note when hub_offline is false", async ({ + page, + }) => { + await stubPresetList(page, [SELECTABLE_ROW, OFFLINE_HUB_ROW], false); + + await test.step("Load the presets list served with the hub", async () => { + await loadPresetsPage(page); + }); + + await test.step("Verify the note is absent and the presets are listed", async () => { + await expect(presetRow(page, SELECTABLE_ROW.title)).toBeVisible(); + await expect(page.getByTestId("preset-hub-offline")).toHaveCount(0); + }); + }); +}); From d79eff34d70d0cd64acbe722f17a36fefc6db8e4 Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Sat, 31 Oct 2026 14:54:45 +0000 Subject: [PATCH 02/16] refactor: prepare hub requests for per-attempt time limits --- .../api/handlers/crowdsec_files_test.go | 57 +- backend/internal/crowdsec/hub_budget.go | 34 ++ backend/internal/crowdsec/hub_budget_test.go | 131 +++++ backend/internal/crowdsec/hub_mirror.go | 361 +++++++++++++ .../internal/crowdsec/hub_mirror_pins_test.go | 26 + backend/internal/crowdsec/hub_mirror_test.go | 492 ++++++++++++++++++ backend/internal/crowdsec/hub_sync.go | 3 + backend/internal/crowdsec/secrets.go | 18 +- backend/internal/crowdsec/secrets_test.go | 67 +++ 9 files changed, 1161 insertions(+), 28 deletions(-) create mode 100644 backend/internal/crowdsec/hub_budget.go create mode 100644 backend/internal/crowdsec/hub_budget_test.go create mode 100644 backend/internal/crowdsec/hub_mirror.go create mode 100644 backend/internal/crowdsec/hub_mirror_pins_test.go create mode 100644 backend/internal/crowdsec/hub_mirror_test.go diff --git a/backend/internal/api/handlers/crowdsec_files_test.go b/backend/internal/api/handlers/crowdsec_files_test.go index b197529f3..4f4472e7c 100644 --- a/backend/internal/api/handlers/crowdsec_files_test.go +++ b/backend/internal/api/handlers/crowdsec_files_test.go @@ -97,32 +97,37 @@ func errorOf(t *testing.T, w *httptest.ResponseRecorder) string { func TestCrowdsecFiles_Predicates(t *testing.T) { t.Parallel() readable := map[string]bool{ - "config.yaml": true, - "config/config.yaml": true, - "config/scenarios/x.yaml": true, - "config/acquis.d/a.conf": true, - "notes.md": true, - "config/data/nested.yaml": true, // only the top-level data/ is protected - "bouncer_key": false, - "bouncer_key.tmp-1234": false, - "config/bouncer_key": false, - "crowdsec.db": false, - "config/crowdsec.db-wal": false, - "data/crowdsec.db": false, - "data/GeoLite2-City.mmdb": false, - "hub_cache/a.tgz": false, - "config/other.db": false, - "config/other.db-journal": false, - "config/CASE.DB": false, - "data": false, - "hub_cache": false, - "local_api_credentials.yaml": false, - "config/local_api_credentials.yaml": false, - "config/online_api_credentials.yaml": false, - "config/Local_API_Credentials.YAML": false, - "config/ONLINE_API_CREDENTIALS.yaml": false, - "config/local_api_credentials.yaml.bak": true, - "config/hub_cache/thing.yaml": true, + "config.yaml": true, + "config/config.yaml": true, + "config/scenarios/x.yaml": true, + "config/acquis.d/a.conf": true, + "notes.md": true, + "config/data/nested.yaml": true, // only the top-level data/ is protected + "bouncer_key": false, + "bouncer_key.tmp-1234": false, + "config/bouncer_key": false, + "crowdsec.db": false, + "config/crowdsec.db-wal": false, + "data/crowdsec.db": false, + "data/GeoLite2-City.mmdb": false, + "hub_cache/a.tgz": false, + "config/other.db": false, + "config/other.db-journal": false, + "config/CASE.DB": false, + "data": false, + "hub_cache": false, + "local_api_credentials.yaml": false, + "config/local_api_credentials.yaml": false, + "config/online_api_credentials.yaml": false, + "config/Local_API_Credentials.YAML": false, + "config/ONLINE_API_CREDENTIALS.yaml": false, + "config/local_api_credentials.yaml.bak": true, + "config/hub_cache/thing.yaml": true, + ".charon-hub-mirror-a1b2.yaml": false, + ".charon-hub-mirror-a1b2.yaml.local": false, + "config/.charon-hub-mirror-a1b2.yaml": false, + "config/.charon-hub-mirror-a1b2.yaml.local": false, + ".charon-hub-mirror-a1b2.yaml.bak": true, } for rel, want := range readable { assert.Equal(t, want, isReadable(rel), "isReadable(%q)", rel) diff --git a/backend/internal/crowdsec/hub_budget.go b/backend/internal/crowdsec/hub_budget.go new file mode 100644 index 000000000..e97443b6f --- /dev/null +++ b/backend/internal/crowdsec/hub_budget.go @@ -0,0 +1,34 @@ +package crowdsec + +import ( + "context" + "time" +) + +// minHubAttemptBudget is the least time one hub request attempt receives, so a short remaining budget still +// leaves a real attempt. The parent deadline still bounds every attempt. +const minHubAttemptBudget = 5 * time.Second + +// attemptContext returns the context for the next of attemptsLeft hub attempts made under parent. The attempt +// gets its share of the time left before parent's deadline (remaining/attemptsLeft), never less than +// minHubAttemptBudget and never more than what remains. A non-positive attemptsLeft counts as one. A parent +// without a deadline is returned unchanged with a no-op cancel: the caller that owns the overall deadline sets +// it before calling. +func attemptContext(parent context.Context, attemptsLeft int) (context.Context, context.CancelFunc) { + deadline, ok := parent.Deadline() + if !ok { + return parent, func() {} + } + if attemptsLeft < 1 { + attemptsLeft = 1 + } + remaining := time.Until(deadline) + budget := remaining / time.Duration(attemptsLeft) + if budget < minHubAttemptBudget { + budget = minHubAttemptBudget + } + if budget > remaining { + budget = remaining + } + return context.WithTimeout(parent, budget) +} diff --git a/backend/internal/crowdsec/hub_budget_test.go b/backend/internal/crowdsec/hub_budget_test.go new file mode 100644 index 000000000..ca76acf32 --- /dev/null +++ b/backend/internal/crowdsec/hub_budget_test.go @@ -0,0 +1,131 @@ +package crowdsec + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// remainingWithin reports the time left until ctx's deadline and fails the test when the deadline is absent. +func remainingWithin(t *testing.T, ctx context.Context) time.Duration { + t.Helper() + deadline, ok := ctx.Deadline() + require.True(t, ok, "attempt context must carry a deadline") + return time.Until(deadline) +} + +func TestAttemptContextWithoutDeadlineReturnsParent(t *testing.T) { + t.Parallel() + parent := context.Background() + ctx, cancel := attemptContext(parent, 2) + require.Equal(t, parent, ctx) + _, ok := ctx.Deadline() + require.False(t, ok) + cancel() // no-op: the caller keeps the parent alive + require.NoError(t, ctx.Err()) +} + +func TestAttemptContextSharesRemainingTime(t *testing.T) { + t.Parallel() + cases := []struct { + name string + attemptsLeft int + min, max time.Duration + }{ + {name: "single attempt takes all", attemptsLeft: 1, min: 29 * time.Second, max: 30 * time.Second}, + {name: "two attempts take half", attemptsLeft: 2, min: 14 * time.Second, max: 15 * time.Second}, + {name: "three attempts take a third", attemptsLeft: 3, min: 9 * time.Second, max: 10 * time.Second}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Each case owns its parent, created when the case runs, so the window is not spent by the + // scheduling of the other cases. + parent, cancelParent := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelParent() + ctx, cancel := attemptContext(parent, tc.attemptsLeft) + defer cancel() + left := remainingWithin(t, ctx) + require.GreaterOrEqual(t, left, tc.min) + require.LessOrEqual(t, left, tc.max) + }) + } +} + +func TestAttemptContextFloorApplies(t *testing.T) { + t.Parallel() + parent, cancelParent := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelParent() + + // 10 s over 4 attempts is 2.5 s, below the 5 s floor, so the attempt gets the floor. + ctx, cancel := attemptContext(parent, 4) + defer cancel() + left := remainingWithin(t, ctx) + require.Greater(t, left, 4*time.Second) + require.LessOrEqual(t, left, minHubAttemptBudget) +} + +func TestAttemptContextCapsAtRemainingBelowFloor(t *testing.T) { + t.Parallel() + parent, cancelParent := context.WithTimeout(context.Background(), 2*time.Second) + defer cancelParent() + + // Only 2 s remain, so a 5 s floor cannot be honoured: the attempt gets what is left. + ctx, cancel := attemptContext(parent, 1) + defer cancel() + left := remainingWithin(t, ctx) + require.Greater(t, left, 1500*time.Millisecond) + require.LessOrEqual(t, left, 2*time.Second) +} + +func TestAttemptContextNonPositiveAttemptsCountsAsOne(t *testing.T) { + t.Parallel() + parent, cancelParent := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelParent() + + for _, attemptsLeft := range []int{0, -3} { + ctx, cancel := attemptContext(parent, attemptsLeft) + left := remainingWithin(t, ctx) + require.Greater(t, left, 9*time.Second) + require.LessOrEqual(t, left, 10*time.Second) + cancel() + } +} + +func TestAttemptContextExpiredParent(t *testing.T) { + t.Parallel() + parent, cancelParent := context.WithTimeout(context.Background(), 0) + defer cancelParent() + + ctx, cancel := attemptContext(parent, 2) + defer cancel() + require.ErrorIs(t, ctx.Err(), context.DeadlineExceeded) +} + +func TestAttemptContextCancelledParentStaysCancelled(t *testing.T) { + t.Parallel() + parent, cancelParent := context.WithTimeout(context.Background(), time.Minute) + cancelParent() + + ctx, cancel := attemptContext(parent, 1) + defer cancel() + require.ErrorIs(t, ctx.Err(), context.Canceled) +} + +func TestAttemptContextCancelReleasesChild(t *testing.T) { + t.Parallel() + parent, cancelParent := context.WithTimeout(context.Background(), time.Minute) + defer cancelParent() + + ctx, cancel := attemptContext(parent, 2) + cancel() + require.ErrorIs(t, ctx.Err(), context.Canceled) + require.NoError(t, parent.Err(), "cancelling an attempt must not cancel its parent") +} + +func TestHubIndexOffline(t *testing.T) { + t.Parallel() + require.False(t, HubIndex{}.Offline()) + require.True(t, HubIndex{offline: true}.Offline()) +} diff --git a/backend/internal/crowdsec/hub_mirror.go b/backend/internal/crowdsec/hub_mirror.go new file mode 100644 index 000000000..8c8678507 --- /dev/null +++ b/backend/internal/crowdsec/hub_mirror.go @@ -0,0 +1,361 @@ +package crowdsec + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/Wikid82/charon/backend/internal/logger" + + "gopkg.in/yaml.v3" +) + +const ( + // mirrorConfigPrefix starts the names of the short-lived cscli configuration copies that point cscli at the + // hub mirror (see mirrorCSCLIConfig). IsHubMirrorConfig recognises them. + mirrorConfigPrefix = ".charon-hub-mirror-" + // mirrorTemplateKey is cscli's hidden hub URL template key. It is a test hook of cscli 1.8.2, not a documented + // option, so the contract doc records it. + mirrorTemplateKey = "__hub_url_template__" + mirrorBranchKey = "hub_branch" + mirrorCSCLISec = "cscli" + // mirrorDefaultBranch is the hub branch cscli is pinned to when the configuration names none. + mirrorDefaultBranch = "master" + // builtinMirrorTemplate is the template of the built-in mirror: the GitHub hub mirror, branch and file. + builtinMirrorTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s" +) + +var ( + // errMirrorTemplateSet means the live configuration already names a hub URL template. The retry must not + // replace an operator's own hub, so it is refused. + errMirrorTemplateSet = errors.New("the cscli config already sets a hub URL template") + // errMirrorTemplateInvalid means the template is not an https URL with exactly the two verbs cscli fills in. + errMirrorTemplateInvalid = errors.New("the hub mirror template must be an https URL with exactly two %s verbs") + // createMirrorTemp creates the derived configuration file. It is a variable so tests can make creation fail. + createMirrorTemp = os.CreateTemp +) + +// MirrorMemory remembers for a short window that the hub mirror answered a cscli hub request, so later requests +// in that window go mirror-first. The window starts at the first win and is not extended by later wins; Forget +// closes it when the live configuration answers again. A nil *MirrorMemory is always inactive. +type MirrorMemory struct { + mu sync.Mutex + ttl time.Duration + until time.Time + now func() time.Time +} + +// NewMirrorMemory returns a memory whose window lasts ttl. A nil clock means time.Now. +func NewMirrorMemory(ttl time.Duration, now func() time.Time) *MirrorMemory { + return &MirrorMemory{ttl: ttl, now: now} +} + +func (m *MirrorMemory) clock() time.Time { + if m.now == nil { + return time.Now() + } + return m.now() +} + +// Active reports whether the mirror-first window is open. +func (m *MirrorMemory) Active() bool { + if m == nil { + return false + } + m.mu.Lock() + defer m.mu.Unlock() + return m.clock().Before(m.until) +} + +// Remember opens the window for ttl after a mirror win. A win while the window is already open changes nothing. +func (m *MirrorMemory) Remember() { + if m == nil { + return + } + m.mu.Lock() + defer m.mu.Unlock() + if now := m.clock(); !now.Before(m.until) { + m.until = now.Add(m.ttl) + } +} + +// Forget closes the window. +func (m *MirrorMemory) Forget() { + if m == nil { + return + } + m.mu.Lock() + defer m.mu.Unlock() + m.until = time.Time{} +} + +// isHubNetworkOp reports whether cscli arguments name an operation that downloads from the hub: `hub update`, +// `hub upgrade`, and ` install` or ` upgrade`. Local reads such as `inspect` and `list` are not. +func isHubNetworkOp(args []string) bool { + if len(args) < 2 { + return false + } + if args[0] == "hub" { + return args[1] == "update" || args[1] == "upgrade" + } + return args[1] == "install" || args[1] == "upgrade" +} + +// isRetryableHubFailure reports whether cscli output names a failure that a retry on the mirror can fix: a +// network failure, or a request that ran out of time. Other failures (an unknown item, a tainted item) are not +// retried. +func isRetryableHubFailure(text string) bool { + if hasNetworkFailure(text) { + return true + } + lower := strings.ToLower(text) + for _, marker := range []string{"context deadline exceeded", "client.timeout", "tls handshake timeout"} { + if strings.Contains(lower, marker) { + return true + } + } + return false +} + +// mirrorHubTemplate returns the cscli hub URL template for a mirror base, such as +// https://raw.githubusercontent.com/crowdsecurity/hub/master. The second result reports whether the operator's base +// was used. An operator base is used only when it passes validateHubURL and ends in the master branch that cscli +// is pinned to; otherwise the built-in mirror is used and a warning names the reason. An empty base means the +// built-in mirror without a warning. +func mirrorHubTemplate(mirrorBase string) (string, bool) { + base := strings.TrimRight(strings.TrimSpace(mirrorBase), "/") + if base == "" { + return builtinMirrorTemplate, false + } + template, refusal := operatorMirrorTemplate(base) + if refusal != "" { + logger.Log().WithField("hub_mirror", HubURLForLog(base)).WithField("reason", refusal). + Warn("hub mirror base not usable, using the built-in mirror") + return builtinMirrorTemplate, false + } + return template, true +} + +// operatorMirrorTemplate turns an operator mirror base into a template, or names why it cannot be used. +func operatorMirrorTemplate(base string) (template, refusal string) { + if err := validateHubURL(base); err != nil { + return "", "not an allowlisted https hub address" + } + prefix, branch := path.Split(base) + if branch != mirrorDefaultBranch { + return "", "the address does not end in the master branch" + } + template = strings.TrimSuffix(prefix, "/") + "/%s/%s" + if !validMirrorTemplate(template) { + return "", "the address cannot form a valid template" + } + return template, "" +} + +// validMirrorTemplate reports whether template is an https URL with exactly two %s verbs and no other percent +// sign, and no control characters. Any other template could make cscli format something unintended. +func validMirrorTemplate(template string) bool { + return strings.HasPrefix(template, "https://") && + strings.Count(template, "%s") == 2 && + strings.Count(template, "%") == 2 && + !strings.ContainsAny(template, "\r\n\x00") +} + +// mirrorCSCLIConfig writes a copy of the cscli configuration at configPath whose cscli section points the hub at +// template, and returns the copy's path with a cleanup that removes it. The copy is built with yaml.v3 (never by +// text concatenation), created with mode 0600 in the same directory so relative paths in it resolve as before, and +// a sibling config.yaml.local is copied next to it because CrowdSec merges that file by name. A live configuration +// that already sets a template is refused with errMirrorTemplateSet, and an unreadable local override fails the +// call, so the caller never retries with a configuration it cannot reproduce. On error the cleanup is a no-op. +// The copy holds the live settings, so IsHubMirrorConfig keeps it out of listings, reads, exports and imports. +func mirrorCSCLIConfig(configPath, template string) (derivedPath string, cleanup func(), err error) { + noop := func() {} + if configPath == "" { + return "", noop, errors.New("cscli config path is empty") + } + if !validMirrorTemplate(template) { + return "", noop, errMirrorTemplateInvalid + } + live := filepath.Clean(configPath) + raw, err := os.ReadFile(live) + if err != nil { + return "", noop, fmt.Errorf("read cscli config: %w", err) + } + content, err := withMirrorTemplate(raw, template) + if err != nil { + return "", noop, err + } + f, err := createMirrorTemp(filepath.Dir(live), mirrorConfigPrefix+"*.yaml") + if err != nil { + return "", noop, fmt.Errorf("create hub mirror config: %w", err) + } + derived := f.Name() + cleanup = func() { + _ = os.Remove(derived) + _ = os.Remove(derived + ".local") + } + if err := writeMirrorFile(f, content); err != nil { + cleanup() + return "", noop, err + } + if err := copyLocalOverride(live+".local", derived+".local"); err != nil { + cleanup() + return "", noop, err + } + return derived, cleanup, nil +} + +// withMirrorTemplate returns the YAML of a cscli configuration with the hub URL template set in its cscli section. +// The section is created when missing, and hub_branch is set to master when it is missing or empty. Every other +// key is kept. +func withMirrorTemplate(raw []byte, template string) ([]byte, error) { + var doc yaml.Node + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("parse cscli config: %w", err) + } + if doc.Kind == 0 { + doc.Kind = yaml.DocumentNode + } + if len(doc.Content) == 0 { + doc.Content = []*yaml.Node{{Kind: yaml.MappingNode, Tag: "!!map"}} + } + root := doc.Content[0] + if root.Kind != yaml.MappingNode { + return nil, errors.New("cscli config root is not a mapping") + } + section, err := childMapping(root, mirrorCSCLISec) + if err != nil { + return nil, err + } + if i := findMappingKey(section, mirrorTemplateKey); i >= 0 && !isEmptyScalar(section.Content[i+1]) { + return nil, errMirrorTemplateSet + } + setMappingValue(section, mirrorTemplateKey, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: template, Style: yaml.DoubleQuotedStyle}) + if i := findMappingKey(section, mirrorBranchKey); i < 0 || isEmptyScalar(section.Content[i+1]) { + setMappingValue(section, mirrorBranchKey, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: mirrorDefaultBranch}) + } + + out, err := yaml.Marshal(&doc) + if err != nil { + return nil, fmt.Errorf("encode hub mirror config: %w", err) + } + return out, nil +} + +// childMapping returns the mapping under key in m, creating an empty one when the key is missing or null. Any +// other kind of value is an error. +func childMapping(m *yaml.Node, key string) (*yaml.Node, error) { + i := findMappingKey(m, key) + if i < 0 { + value := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + m.Content = append(m.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, value) + return value, nil + } + value := m.Content[i+1] + if isNullScalar(value) { + value.Kind, value.Tag, value.Value, value.Style = yaml.MappingNode, "!!map", "", 0 + } + if value.Kind != yaml.MappingNode { + return nil, fmt.Errorf("cscli config %s is not a mapping", key) + } + return value, nil +} + +// findMappingKey returns the index of key's key node in the mapping m, or -1. Keys sit at even positions of Content. +func findMappingKey(m *yaml.Node, key string) int { + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return i + } + } + return -1 +} + +// setMappingValue replaces the value under key in the mapping m, or appends the key when it is missing. +func setMappingValue(m *yaml.Node, key string, value *yaml.Node) { + if i := findMappingKey(m, key); i >= 0 { + m.Content[i+1] = value + return + } + m.Content = append(m.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, value) +} + +// isNullScalar reports whether n is a YAML null (an empty value or ~). +func isNullScalar(n *yaml.Node) bool { + return n.Kind == yaml.ScalarNode && n.Tag == "!!null" +} + +// isEmptyScalar reports whether n is a null or an empty string. +func isEmptyScalar(n *yaml.Node) bool { + return isNullScalar(n) || (n.Kind == yaml.ScalarNode && n.Value == "") +} + +// writeMirrorFile writes data into the file f and closes it. The file was created with mode 0600 by +// os.CreateTemp, so the content is never readable by other users. +func writeMirrorFile(f *os.File, data []byte) error { + _, err := f.Write(data) + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("write hub mirror config: %w", err) + } + return nil +} + +// copyLocalOverride copies the CrowdSec local override src to dst, when src exists. dst is created exclusively with +// mode 0600. A missing override is not an error; an unreadable one is, so the caller does not retry without it. +func copyLocalOverride(src, dst string) error { + raw, err := os.ReadFile(src) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read cscli local override: %w", err) + } + f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("create hub mirror local override: %w", err) + } + return writeMirrorFile(f, raw) +} + +// isMirrorConfigName reports whether a lower-case file name is a derived mirror configuration or its local copy. +func isMirrorConfigName(lower string) bool { + return strings.HasPrefix(lower, mirrorConfigPrefix) && + (strings.HasSuffix(lower, ".yaml") || strings.HasSuffix(lower, ".yaml.local")) +} + +// sweepMirrorLeftovers removes derived mirror configurations that a crash left in the data directory or its config +// directory. It returns an error only when a leftover cannot be removed; a missing directory is not an error. +func sweepMirrorLeftovers(dataDir string) error { + if dataDir == "" { + return nil + } + var errs []error + for _, dir := range []string{dataDir, filepath.Join(dataDir, "config")} { + entries, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + errs = append(errs, fmt.Errorf("list hub mirror leftovers: %w", err)) + continue + } + for _, e := range entries { + if !isMirrorConfigName(strings.ToLower(e.Name())) { + continue + } + if err := os.Remove(filepath.Join(dir, e.Name())); err != nil { + errs = append(errs, fmt.Errorf("remove hub mirror leftover: %w", err)) + } + } + } + return errors.Join(errs...) +} diff --git a/backend/internal/crowdsec/hub_mirror_pins_test.go b/backend/internal/crowdsec/hub_mirror_pins_test.go new file mode 100644 index 000000000..1db49bba5 --- /dev/null +++ b/backend/internal/crowdsec/hub_mirror_pins_test.go @@ -0,0 +1,26 @@ +package crowdsec + +import "testing" + +// The tests below pin behaviour that later commits of the hub outage fallback implement. They stay skipped +// here, and each commit removes the skip for its own pins. + +func TestHubPinPullTriesMirrorWhenFirstAddressHangs(t *testing.T) { + t.Skip("enabled in commit 3") +} + +func TestHubPinFetchIndexWithoutDeadlineBoundedWhenBothAddressesHang(t *testing.T) { + t.Skip("enabled in commit 3") +} + +func TestHubPinCSCLIRetriesOnMirrorAfterNetworkFailure(t *testing.T) { + t.Skip("enabled in commit 4") +} + +func TestHubPinCSCLIDoesNotRetryOnNonNetworkFailure(t *testing.T) { + t.Skip("enabled in commit 4") +} + +func TestHubPinPresetListReportsHubOffline(t *testing.T) { + t.Skip("enabled in commit 6") +} diff --git a/backend/internal/crowdsec/hub_mirror_test.go b/backend/internal/crowdsec/hub_mirror_test.go new file mode 100644 index 000000000..4c8ef6066 --- /dev/null +++ b/backend/internal/crowdsec/hub_mirror_test.go @@ -0,0 +1,492 @@ +package crowdsec + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +const testMirrorTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s" + +func writeCSCLIConfig(t *testing.T, dir, content string) string { + t.Helper() + p := filepath.Join(dir, "config.yaml") + require.NoError(t, os.WriteFile(p, []byte(content), 0o600)) + return p +} + +// dirEntryNames lists the file names in dir, sorted by the OS. +func dirEntryNames(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + return names +} + +func TestIsHubNetworkOp(t *testing.T) { + t.Parallel() + cases := map[string]bool{ + "hub update": true, + "hub upgrade": true, + "collections install x/y": true, + "parsers install x/y": true, + "scenarios upgrade x/y": true, + "postoverflows install x/y": true, + "collections inspect x/y": false, + "collections list": false, + "hub list": false, + "version": false, + "hub": false, + "install": false, + "": false, + "collections remove x/y": false, + "hub install x": false, + "collections install-old x/y": false, + } + for line, want := range cases { + args := strings.Fields(line) + require.Equal(t, want, isHubNetworkOp(args), line) + } + require.False(t, isHubNetworkOp(nil)) +} + +func TestIsRetryableHubFailure(t *testing.T) { + t.Parallel() + cases := map[string]bool{ + "failed http request for https://cdn-hub.crowdsec.net: dial tcp 1.2.3.4:443: connection refused": true, + "lookup cdn-hub.crowdsec.net: no such host": true, + "context deadline exceeded": true, + "Get \"https://x\": Client.Timeout exceeded while awaiting headers": true, + "net/http: TLS handshake timeout": true, + "item not found": false, + "item is tainted": false, + "": false, + } + for text, want := range cases { + require.Equal(t, want, isRetryableHubFailure(text), text) + } +} + +func TestMirrorMemoryWindow(t *testing.T) { + clock := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + now := func() time.Time { return clock } + m := NewMirrorMemory(10*time.Minute, now) + + require.False(t, m.Active()) + m.Remember() + require.True(t, m.Active()) + + // A later mirror win inside the window does not extend it. + clock = clock.Add(5 * time.Minute) + m.Remember() + require.True(t, m.Active()) + + clock = clock.Add(5 * time.Minute) + require.False(t, m.Active(), "window ends ttl after the first win") + + // A win after the window closed opens a new one. + m.Remember() + require.True(t, m.Active()) + + // The live config answering again closes the window. + m.Forget() + require.False(t, m.Active()) +} + +func TestMirrorMemoryNilAndZeroValues(t *testing.T) { + t.Parallel() + var nilMemory *MirrorMemory + require.False(t, nilMemory.Active()) + nilMemory.Remember() + nilMemory.Forget() + + var zero MirrorMemory + require.False(t, zero.Active()) + zero.Remember() + zero.Forget() +} + +func TestMirrorHubTemplate(t *testing.T) { + t.Parallel() + cases := []struct { + name string + base string + want string + operator bool + }{ + {name: "unset uses the built-in mirror", base: "", want: testMirrorTemplate, operator: false}, + {name: "built-in base", base: defaultHubMirrorBaseURL, want: testMirrorTemplate, operator: true}, + {name: "operator master with trailing slash", base: defaultHubMirrorBaseURL + "/", want: testMirrorTemplate, operator: true}, + { + name: "operator fork on the allowlisted mirror host", + base: "https://raw.githubusercontent.com/example/hub/master", + want: "https://raw.githubusercontent.com/example/hub/%s/%s", + operator: true, + }, + {name: "operator other branch is refused", base: "https://raw.githubusercontent.com/crowdsecurity/hub/dev", want: testMirrorTemplate}, + {name: "operator plain http is refused", base: "http://raw.githubusercontent.com/crowdsecurity/hub/master", want: testMirrorTemplate}, + {name: "operator host outside the allowlist is refused", base: "https://example.com/hub/master", want: testMirrorTemplate}, + {name: "operator percent escape would add a verb", base: "https://raw.githubusercontent.com/x%41/hub/master", want: testMirrorTemplate}, + {name: "unparseable operator value is refused", base: "https://[::1", want: testMirrorTemplate}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, operator := mirrorHubTemplate(tc.base) + require.Equal(t, tc.want, got) + require.Equal(t, tc.operator, operator) + }) + } +} + +func TestValidMirrorTemplate(t *testing.T) { + t.Parallel() + require.True(t, validMirrorTemplate(testMirrorTemplate)) + for _, bad := range []string{ + "", + "https://x/%s", + "https://x/%s/%s/%s", + "https://x/%s/%s%", + "https://x/%41/%s/%s", + "http://x/%s/%s", + "https://x/%s/%s\n__hub_url_template__: evil", + "https://x/%s/%s\r", + "https://x/%s/%s\x00", + } { + require.False(t, validMirrorTemplate(bad), "%q", bad) + } +} + +func TestMirrorCSCLIConfigInsertsTemplateAndKeepsOtherKeys(t *testing.T) { + t.Parallel() + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "common:\n log_level: info\ncscli:\n output: human\n hub_branch: dev\n") + + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.NoError(t, err) + require.Equal(t, dir, filepath.Dir(derived)) + require.True(t, strings.HasPrefix(filepath.Base(derived), mirrorConfigPrefix)) + require.True(t, strings.HasSuffix(derived, ".yaml")) + + info, err := os.Stat(derived) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + + raw, err := os.ReadFile(derived) + require.NoError(t, err) + var got map[string]map[string]string + require.NoError(t, yaml.Unmarshal(raw, &got)) + require.Equal(t, testMirrorTemplate, got["cscli"]["__hub_url_template__"]) + require.Equal(t, "dev", got["cscli"]["hub_branch"], "an existing branch is kept") + require.Equal(t, "human", got["cscli"]["output"]) + require.Equal(t, "info", got["common"]["log_level"]) + + // The live file is never touched. + liveRaw, err := os.ReadFile(live) + require.NoError(t, err) + require.Contains(t, string(liveRaw), "hub_branch: dev") + require.NotContains(t, string(liveRaw), "__hub_url_template__") + + cleanup() + _, err = os.Stat(derived) + require.True(t, errors.Is(err, os.ErrNotExist), "cleanup removes the derived file") +} + +func TestMirrorCSCLIConfigSetsBranchWhenMissingOrEmpty(t *testing.T) { + t.Parallel() + cases := map[string]string{ + "section missing": "common:\n log_level: info\n", + "section empty": "cscli:\n", + "branch empty": "cscli:\n hub_branch: \"\"\n", + "branch missing": "cscli:\n output: human\n", + "empty file": "", + "comment only": "# nothing configured yet\n", + "section null tilde": "cscli: ~\n", + "branch null with value": "cscli:\n hub_branch:\n", + } + for name, content := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, content) + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.NoError(t, err) + defer cleanup() + + raw, err := os.ReadFile(derived) + require.NoError(t, err) + var got map[string]map[string]string + require.NoError(t, yaml.Unmarshal(raw, &got)) + require.Equal(t, "master", got["cscli"]["hub_branch"]) + require.Equal(t, testMirrorTemplate, got["cscli"]["__hub_url_template__"]) + }) + } +} + +func TestMirrorCSCLIConfigQuotesTemplateValue(t *testing.T) { + t.Parallel() + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "cscli:\n hub_branch: master\n") + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.NoError(t, err) + defer cleanup() + + raw, err := os.ReadFile(derived) + require.NoError(t, err) + require.Contains(t, string(raw), `__hub_url_template__: "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s"`) +} + +func TestMirrorCSCLIConfigRefusesOperatorTemplate(t *testing.T) { + t.Parallel() + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "cscli:\n __hub_url_template__: https://private.example/%s/%s\n") + + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.ErrorIs(t, err, errMirrorTemplateSet) + require.Empty(t, derived) + require.NotNil(t, cleanup) + cleanup() + require.Equal(t, []string{"config.yaml"}, dirEntryNames(t, dir), "no derived file is left behind") +} + +func TestMirrorCSCLIConfigAllowsEmptyOperatorTemplate(t *testing.T) { + t.Parallel() + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "cscli:\n __hub_url_template__:\n") + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.NoError(t, err) + defer cleanup() + + raw, err := os.ReadFile(derived) + require.NoError(t, err) + require.Contains(t, string(raw), testMirrorTemplate) +} + +func TestMirrorCSCLIConfigRefusesInvalidTemplate(t *testing.T) { + t.Parallel() + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "cscli:\n hub_branch: master\n") + for _, bad := range []string{"https://x/%s", "https://x/%s/%s\nevil: true", "http://x/%s/%s"} { + derived, cleanup, err := mirrorCSCLIConfig(live, bad) + require.ErrorIs(t, err, errMirrorTemplateInvalid, "%q", bad) + require.Empty(t, derived) + cleanup() + } + require.Equal(t, []string{"config.yaml"}, dirEntryNames(t, dir)) +} + +func TestMirrorCSCLIConfigRejectsUnusableInput(t *testing.T) { + t.Parallel() + dir := t.TempDir() + cases := map[string]string{ + "malformed yaml": "cscli: [\n", + "root is a sequence": "- a\n- b\n", + "cscli is a scalar": "cscli: 5\n", + "cscli is a sequence": "cscli: [1]\n", + } + for name, content := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + sub := t.TempDir() + live := writeCSCLIConfig(t, sub, content) + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.Error(t, err) + require.Empty(t, derived) + cleanup() + require.Equal(t, []string{"config.yaml"}, dirEntryNames(t, sub)) + }) + } + + t.Run("missing config", func(t *testing.T) { + t.Parallel() + _, cleanup, err := mirrorCSCLIConfig(filepath.Join(dir, "absent.yaml"), testMirrorTemplate) + require.Error(t, err) + cleanup() + }) + t.Run("empty path", func(t *testing.T) { + t.Parallel() + _, cleanup, err := mirrorCSCLIConfig("", testMirrorTemplate) + require.Error(t, err) + cleanup() + }) +} + +func TestMirrorCSCLIConfigCopiesLocalOverride(t *testing.T) { + t.Parallel() + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "cscli:\n hub_branch: master\n") + const local = "cscli:\n hub_dir: /custom/hub\n" + require.NoError(t, os.WriteFile(live+".local", []byte(local), 0o600)) + + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.NoError(t, err) + + copied, err := os.ReadFile(derived + ".local") + require.NoError(t, err) + require.Equal(t, local, string(copied)) + info, err := os.Stat(derived + ".local") + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + + cleanup() + for _, p := range []string{derived, derived + ".local"} { + _, err := os.Stat(p) + require.True(t, errors.Is(err, os.ErrNotExist), p) + } + require.Equal(t, []string{"config.yaml", "config.yaml.local"}, dirEntryNames(t, dir)) +} + +func TestMirrorCSCLIConfigFailsClosedWhenLocalUnreadable(t *testing.T) { + t.Parallel() + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "cscli:\n hub_branch: master\n") + // A directory where the override file should be cannot be read, so no retry may run without it. + require.NoError(t, os.Mkdir(live+".local", 0o750)) + + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.Error(t, err) + require.Empty(t, derived) + cleanup() + require.Equal(t, []string{"config.yaml", "config.yaml.local"}, dirEntryNames(t, dir)) +} + +func TestMirrorCSCLIConfigFailsWhenTempFileCannotBeCreated(t *testing.T) { + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "cscli:\n hub_branch: master\n") + original := createMirrorTemp + createMirrorTemp = func(string, string) (*os.File, error) { return nil, errors.New("disk full") } + t.Cleanup(func() { createMirrorTemp = original }) + + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.ErrorContains(t, err, "disk full") + require.Empty(t, derived) + cleanup() +} + +func TestSweepMirrorLeftovers(t *testing.T) { + t.Parallel() + dataDir := t.TempDir() + configDir := filepath.Join(dataDir, "config") + require.NoError(t, os.MkdirAll(configDir, 0o750)) + leftovers := []string{ + filepath.Join(configDir, mirrorConfigPrefix+"a1.yaml"), + filepath.Join(configDir, mirrorConfigPrefix+"a1.yaml.local"), + filepath.Join(dataDir, mirrorConfigPrefix+"b2.yaml"), + filepath.Join(dataDir, mirrorConfigPrefix+"b2.yaml.local"), + } + keep := []string{ + filepath.Join(configDir, "config.yaml"), + filepath.Join(configDir, "config.yaml.local"), + filepath.Join(dataDir, ".charon-hub-other.yaml"), + } + for _, p := range append(append([]string{}, leftovers...), keep...) { + require.NoError(t, os.WriteFile(p, []byte("x"), 0o600)) + } + + require.NoError(t, sweepMirrorLeftovers(dataDir)) + for _, p := range leftovers { + _, err := os.Stat(p) + require.True(t, errors.Is(err, os.ErrNotExist), "leftover kept: %s", p) + } + for _, p := range keep { + _, err := os.Stat(p) + require.NoError(t, err, "unrelated file removed: %s", p) + } +} + +func TestSweepMirrorLeftoversEdgeCases(t *testing.T) { + t.Parallel() + t.Run("empty data dir is a no-op", func(t *testing.T) { + t.Parallel() + require.NoError(t, sweepMirrorLeftovers("")) + }) + t.Run("missing data dir is a no-op", func(t *testing.T) { + t.Parallel() + require.NoError(t, sweepMirrorLeftovers(filepath.Join(t.TempDir(), "absent"))) + }) + t.Run("a leftover that cannot be removed is reported", func(t *testing.T) { + t.Parallel() + dataDir := t.TempDir() + stuck := filepath.Join(dataDir, mirrorConfigPrefix+"c3.yaml") + require.NoError(t, os.Mkdir(stuck, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(stuck, "inner"), []byte("x"), 0o600)) + require.Error(t, sweepMirrorLeftovers(dataDir)) + }) +} + +func TestMirrorCSCLIConfigFailsWhenTempFileIsUnusable(t *testing.T) { + dir := t.TempDir() + live := writeCSCLIConfig(t, dir, "cscli:\n hub_branch: master\n") + original := createMirrorTemp + createMirrorTemp = func(d, pattern string) (*os.File, error) { + f, err := os.CreateTemp(d, pattern) + if err != nil { + return nil, err + } + closeErr := f.Close() + return f, closeErr + } + t.Cleanup(func() { createMirrorTemp = original }) + + derived, cleanup, err := mirrorCSCLIConfig(live, testMirrorTemplate) + require.ErrorContains(t, err, "write hub mirror config") + require.Empty(t, derived) + cleanup() + require.Equal(t, []string{"config.yaml"}, dirEntryNames(t, dir), "the unusable temp file is removed") +} + +func TestWriteMirrorFileReportsClosedFile(t *testing.T) { + t.Parallel() + f, err := os.CreateTemp(t.TempDir(), "x*.yaml") + require.NoError(t, err) + require.NoError(t, f.Close()) + require.ErrorContains(t, writeMirrorFile(f, []byte("x")), "write hub mirror config") +} + +func TestCopyLocalOverrideRefusesExistingTarget(t *testing.T) { + t.Parallel() + dir := t.TempDir() + src := filepath.Join(dir, "config.yaml.local") + dst := filepath.Join(dir, "derived.yaml.local") + require.NoError(t, os.WriteFile(src, []byte("new"), 0o600)) + require.NoError(t, os.WriteFile(dst, []byte("keep"), 0o600)) + + require.ErrorContains(t, copyLocalOverride(src, dst), "create hub mirror local override") + kept, err := os.ReadFile(dst) + require.NoError(t, err) + require.Equal(t, "keep", string(kept), "an existing target is never overwritten") +} + +func TestCopyLocalOverrideMissingSourceIsNoop(t *testing.T) { + t.Parallel() + dir := t.TempDir() + dst := filepath.Join(dir, "derived.yaml.local") + require.NoError(t, copyLocalOverride(filepath.Join(dir, "absent.local"), dst)) + _, err := os.Stat(dst) + require.True(t, errors.Is(err, os.ErrNotExist)) +} + +func TestIsMirrorConfigName(t *testing.T) { + t.Parallel() + require.True(t, isMirrorConfigName(".charon-hub-mirror-a.yaml")) + require.True(t, isMirrorConfigName(".charon-hub-mirror-a.yaml.local")) + require.False(t, isMirrorConfigName(".charon-hub-mirror-a.yml")) + require.False(t, isMirrorConfigName("config.yaml")) +} + +func TestSweepMirrorLeftoversUnusableDataDir(t *testing.T) { + t.Parallel() + notADir := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(notADir, []byte("x"), 0o600)) + require.Error(t, sweepMirrorLeftovers(notADir)) +} diff --git a/backend/internal/crowdsec/hub_sync.go b/backend/internal/crowdsec/hub_sync.go index cf4a98298..7580c1fc7 100644 --- a/backend/internal/crowdsec/hub_sync.go +++ b/backend/internal/crowdsec/hub_sync.go @@ -77,6 +77,9 @@ type HubIndex struct { offline bool } +// Offline reports whether the index was listed from the local cscli data because the HTTP index was unreachable. +func (idx HubIndex) Offline() bool { return idx.offline } + // PullResult bundles the pull metadata, preview text, and cache entry. type PullResult struct { Meta CachedPreset diff --git a/backend/internal/crowdsec/secrets.go b/backend/internal/crowdsec/secrets.go index 2e720066d..08515dc37 100644 --- a/backend/internal/crowdsec/secrets.go +++ b/backend/internal/crowdsec/secrets.go @@ -2,6 +2,7 @@ package crowdsec import ( "os" + "path" "path/filepath" "strings" ) @@ -49,9 +50,22 @@ func isSecretName(part string) bool { } // IsPreservedPath reports whether rel is server-local state that imports must neither remove, replace -// nor take from an upload: engine-owned state and stored secrets. +// nor take from an upload: engine-owned state, stored secrets and derived hub mirror configurations. func IsPreservedPath(rel string) bool { - return IsEngineOwnedPath(rel) || IsSecretPath(rel) + return IsEngineOwnedPath(rel) || IsSecretPath(rel) || IsHubMirrorConfig(rel) +} + +// IsHubMirrorConfig reports whether rel names a derived cscli configuration made for a hub mirror retry, or its +// local override copy. Such a file repeats the live engine settings, so it is never listed, read, exported or +// imported. Only the two places a live configuration can sit are matched: directly under DataDir or under +// config/. Matching is case-insensitive. +func IsHubMirrorConfig(rel string) bool { + slash := strings.ToLower(path.Clean(filepath.ToSlash(rel))) + dir := path.Dir(slash) + if dir != "." && dir != "config" { + return false + } + return isMirrorConfigName(path.Base(slash)) } // FindConfigFile returns the CrowdSec config file below dataDir: config/config.yaml when it exists, diff --git a/backend/internal/crowdsec/secrets_test.go b/backend/internal/crowdsec/secrets_test.go index 90970abe8..5ba1995c2 100644 --- a/backend/internal/crowdsec/secrets_test.go +++ b/backend/internal/crowdsec/secrets_test.go @@ -1,6 +1,8 @@ package crowdsec import ( + "bytes" + "context" "os" "path/filepath" "testing" @@ -159,3 +161,68 @@ func TestRestoreKeepingLeavesLiveSecretsAlone(t *testing.T) { require.Equal(t, "live-key", readFile(t, filepath.Join(dir, "bouncer_key"))) }) } + +func TestIsHubMirrorConfig(t *testing.T) { + t.Parallel() + cases := map[string]bool{ + ".charon-hub-mirror-abc.yaml": true, + ".charon-hub-mirror-abc.yaml.local": true, + "config/.charon-hub-mirror-abc.yaml": true, + "config/.charon-hub-mirror-abc.yaml.local": true, + "./.charon-hub-mirror-abc.yaml": true, + "CONFIG/.Charon-Hub-Mirror-ABC.YAML": true, + ".CHARON-HUB-MIRROR-ABC.YAML.LOCAL": true, + `config\.charon-hub-mirror-abc.yaml`: filepath.Separator == '\\', + "config.yaml": false, + "config/config.yaml": false, + "config/config.yaml.local": false, + "charon-hub-mirror-abc.yaml": false, + ".charon-hub-mirror-abc.txt": false, + ".charon-hub-mirror-abc.yaml.bak": false, + "sub/.charon-hub-mirror-abc.yaml": false, + "config/sub/.charon-hub-mirror-abc.yaml": false, + "data/.charon-hub-mirror-abc.yaml": false, + "xcharon-hub-mirror-abc.yaml": false, + "": false, + ".": false, + } + for rel, want := range cases { + require.Equal(t, want, IsHubMirrorConfig(rel), rel) + } +} + +func TestHubMirrorConfigIsPreservedAndExcluded(t *testing.T) { + t.Parallel() + dataDir := t.TempDir() + for _, rel := range []string{ + ".charon-hub-mirror-abc.yaml", + ".charon-hub-mirror-abc.yaml.local", + "config/.charon-hub-mirror-abc.yaml", + "config/.charon-hub-mirror-abc.yaml.local", + } { + require.True(t, IsPreservedPath(rel), "preserved %s", rel) + require.True(t, IsExportExcluded(rel), "export excludes %s", rel) + require.True(t, ImportKeeper(dataDir)(rel), "import keeps %s", rel) + } + require.False(t, IsPreservedPath("config/config.yaml")) + require.False(t, IsExportExcluded("config/config.yaml")) + require.False(t, ImportKeeper(dataDir)("config/config.yaml")) +} + +func TestHubMirrorLeftoverIsLeftOutOfExportArchive(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeExportTree(t, dir, map[string]string{ + "config/config.yaml": "common: {}\n", + "config/.charon-hub-mirror-abc.yaml": "cscli:\n password: s3cret\n", + "config/.charon-hub-mirror-abc.yaml.local": "cscli: {}\n", + ".charon-hub-mirror-def.yaml": "cscli:\n password: s3cret\n", + "config/acquis.yaml": "source: file\n", + }) + + var buf bytes.Buffer + _, omitted, err := WriteExportArchive(context.Background(), dir, &buf) + require.NoError(t, err) + require.Equal(t, []string{"config/acquis.yaml", "config/config.yaml"}, exportNames(t, buf.Bytes())) + require.NotContains(t, omitted, "config/.charon-hub-mirror-abc.yaml") +} From ca980337f21cf3c4fc13101707fc92574e237f09 Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Sat, 31 Oct 2026 15:11:49 +0000 Subject: [PATCH 03/16] fix: use the hub mirror during CrowdSec startup when the hub CDN is unreachable --- .docker/cscli-hub-mirror.sh | 320 +++++++++++++++ .docker/docker-entrypoint.sh | 42 +- .dockerignore | 1 + .github/workflows/quality-checks.yml | 32 ++ Dockerfile | 3 + configs/crowdsec/install_hub_items.sh | 16 +- scripts/cscli_hub_mirror_test.sh | 539 ++++++++++++++++++++++++++ 7 files changed, 949 insertions(+), 4 deletions(-) create mode 100644 .docker/cscli-hub-mirror.sh create mode 100755 scripts/cscli_hub_mirror_test.sh diff --git a/.docker/cscli-hub-mirror.sh b/.docker/cscli-hub-mirror.sh new file mode 100644 index 000000000..b0f1f708b --- /dev/null +++ b/.docker/cscli-hub-mirror.sh @@ -0,0 +1,320 @@ +# shellcheck shell=sh +# CrowdSec hub mirror retry for cscli. POSIX sh, sourced by .docker/docker-entrypoint.sh and by +# configs/crowdsec/install_hub_items.sh. The image copies it to /usr/local/lib/charon/cscli-hub-mirror.sh. +# +# cscli reads one hub URL template from its config (the hidden key __hub_url_template__). When the hub CDN +# is unreachable, a retry runs cscli with a derived copy of the config whose template points at the raw +# GitHub mirror. The live config is never written. +# +# The rules follow backend/internal/crowdsec/hub_mirror.go (template checks, refusal of a template the +# operator already set, the retry classifier). Keep the two in step. +# +# Internal state, set by this helper and not meant for operators: +# CHARON_CSCLI_MIRROR_CONFIG path of the derived config while it exists +# CHARON_CSCLI_MIRROR_FIRST "1" once a mirror retry answered: later cscli_hub calls go mirror-first + +_HUB_MIRROR_TEMPLATE='https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s' +_HUB_MIRROR_KEY='__hub_url_template__' +_HUB_MIRROR_PREFIX='.charon-hub-mirror-' + +# Plain foreground fallback for callers without the entrypoint's stop-aware runner (install_hub_items.sh). +if ! command -v run_interruptible >/dev/null 2>&1; then + run_interruptible() { + "$@" + } +fi + +# hub_mirror_template_valid