diff --git a/.github/workflows/check-setup.yml b/.github/workflows/check-setup.yml new file mode 100644 index 0000000..536a88c --- /dev/null +++ b/.github/workflows/check-setup.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Check setup readiness + +on: + pull_request: + paths: + - .github/workflows/check-setup.yml + - config/components.ts + - config/components.test.ts + - setup/cluster-ready.ts + - setup/cluster-ready.test.ts + - setup/opsfile.yml + +permissions: + contents: read + +jobs: + readiness: + name: Task readiness regression tests + runs-on: ubuntu-22.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Install checksum-pinned Bun 1.3.14 + run: | + curl --fail --location --retry 3 \ + https://github.com/oven-sh/bun/releases/download/bun-v1.3.14/bun-linux-x64.zip \ + --output "$RUNNER_TEMP/bun.zip" + echo "951ee2aee855f08595aeec6225226a298d3fea83a3dcd6465c09cbccdf7e848f $RUNNER_TEMP/bun.zip" | sha256sum --check + unzip -q "$RUNNER_TEMP/bun.zip" -d "$RUNNER_TEMP" + echo "$RUNNER_TEMP/bun-linux-x64" >> "$GITHUB_PATH" + - name: Test component dependencies and setup readiness + run: bun test config/components.test.ts setup/cluster-ready.test.ts diff --git a/config/components.test.ts b/config/components.test.ts new file mode 100644 index 0000000..badb72f --- /dev/null +++ b/config/components.test.ts @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {test, expect} from "bun:test"; +import {plan, validate, keys} from "./components"; +test("all is symmetric and includes Alert Manager", () => { + const enabled=plan("enable",["all"],{}),disabled=plan("disable",["all"],{}); + for(const k of Object.keys(enabled)) expect(disabled[k]).toBe("false"); + expect(disabled[keys.alertmanager]).toBe("false"); + expect(Object.keys(disabled).length).toBe(Object.keys(keys).length); + expect(enabled[keys.slack]).toBeUndefined();expect(enabled[keys.mail]).toBeUndefined(); +}); +test("full includes services but not notifications or scheduling",()=>{ + const p=plan("full",[],{});validate(p); + for(const x of ["slack","mail","affinity","tolerations"])expect(p[keys[x]]).toBeUndefined(); + for(const x of ["seaweedfs","static","etcd","milvus","alertmanager","registry"])expect(p[keys[x]]).toBe("true"); +}); +test("enable adds transitive dependencies",()=>{ + expect(plan("enable",["milvus"],{})).toEqual({[keys.milvus]:"true",[keys.etcd]:"true",[keys.seaweedfs]:"true"}); + expect(plan("enable",["mongodb"],{})[keys.postgres]).toBe("true"); + expect(plan("enable",["static"],{})[keys.seaweedfs]).toBe("true"); +}); +test("disable cascades and preserves unrelated prerequisites",()=>{ + expect(plan("disable",["prometheus"],{})[keys.mail]).toBe("false"); + expect(plan("disable",["alertmanager"],{})[keys.prometheus]).toBeUndefined(); + expect(plan("disable",["milvus"],{})[keys.etcd]).toBeUndefined(); + expect(plan("disable",["etcd"],{})[keys.milvus]).toBe("false"); + expect(plan("disable",["seaweedfs"],{})[keys.static]).toBe("false"); + expect(plan("disable",["postgres"],{})[keys.mongodb]).toBe("false"); +}); +test("notifications require configuration before any changes",()=>{ + expect(()=>plan("enable",["all","mail"],{})).toThrow("Configure notifications"); + expect(()=>plan("enable",["slack"],{})).toThrow("Configure notifications"); + const p=plan("enable",["slack"],{OPERATOR_CONFIG_SLACK_APIURL:"https://example.test",OPERATOR_CONFIG_SLACK_CHANNELNAME:"#test"}); + expect(p[keys.prometheus]).toBe("true");expect(p[keys.alertmanager]).toBe("true"); +}); +test("preflight rejects missing frontend, dependencies and unconfigured alerts",()=>{ + expect(()=>validate({})).toThrow("frontend"); + const full=plan("full",[],{}); + expect(()=>validate({...full,[keys.etcd]:"false"})).toThrow("milvus requires etcd"); + expect(()=>validate({...full,[keys.mail]:"true"})).toThrow("Configure notifications"); +}); +test("unknown input is rejected",()=>{expect(()=>plan("enable",["bogus"],{})).toThrow("Unknown component")}); diff --git a/config/components.ts b/config/components.ts new file mode 100644 index 0000000..bcd8c2b --- /dev/null +++ b/config/components.ts @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export const keys: Record = Object.fromEntries( + ["redis","mongodb","cron","static","postgres","prometheus","quota","milvus","registry","seaweedfs","etcd"].map(x => [x, `OPERATOR_COMPONENT_${x.toUpperCase()}`]) +); +Object.assign(keys, {alertmanager:"OPERATOR_COMPONENT_AM", slack:"OPERATOR_CONFIG_ALERTSLACK", mail:"OPERATOR_CONFIG_ALERTGMAIL", affinity:"OPERATOR_CONFIG_AFFINITY", tolerations:"OPERATOR_CONFIG_TOLERATIONS"}); +const dependencies: Record = {mongodb:["postgres"], milvus:["etcd","seaweedfs"], static:["seaweedfs"], alertmanager:["prometheus"], slack:["alertmanager"], mail:["alertmanager"]}; +export function plan(mode: string, flags: string[], env: Record) { + if (!["enable","disable","full"].includes(mode)) throw new Error("Invalid configuration operation"); + for (const flag of flags) if (flag !== "all" && !keys[flag]) throw new Error(`Unknown component: ${flag}`); + const on = mode !== "disable"; + const selected = new Set(flags.filter(x => x !== "all")); + if (flags.includes("all") || mode === "full") for (const name of Object.keys(keys)) { + // Notification destinations require explicit opt-in. Full also excludes scheduling policies. + if (on && ["slack","mail"].includes(name)) continue; + if (mode === "full" && ["affinity","tolerations"].includes(name)) continue; + selected.add(name); + } + if (on) { + const include = (name: string) => {for (const dep of dependencies[name] || []) if (!selected.has(dep)) {selected.add(dep);include(dep)}}; + for (const name of selected) include(name); + const required: Record = { + slack:["OPERATOR_CONFIG_SLACK_APIURL","OPERATOR_CONFIG_SLACK_CHANNELNAME"], + mail:["OPERATOR_CONFIG_GMAIL_USERNAME","OPERATOR_CONFIG_GMAIL_PASSWORD","OPERATOR_CONFIG_EMAIL_FROM","OPERATOR_CONFIG_EMAIL_TO"] + }; + for (const name of ["slack","mail"]) if (selected.has(name)) { + const missing = required[name].filter(k => !env[k]?.trim()); + if (missing.length) throw new Error(`Configure notifications first with ops config ${name}; missing: ${missing.join(", ")}`); + } + } else { + // Disabling a prerequisite disables its dependants, never unrelated prerequisites. + let changed = true; + while (changed) {changed = false;for (const [name,deps] of Object.entries(dependencies)) { + if (!selected.has(name) && deps.some(dep => selected.has(dep))) {selected.add(name);changed=true;} + }} + } + return Object.fromEntries([...selected].map(name => [keys[name], String(on)])); +} +export function validate(env: Record) { + const failures: string[] = []; + for (const name of ["seaweedfs","static"]) if (env[keys[name]] !== "true") failures.push(`frontend requires ${name}`); + for (const [name,deps] of Object.entries(dependencies)) if (env[keys[name]] === "true") { + for (const dep of deps) if (env[keys[dep]] !== "true") failures.push(`${name} requires ${dep}`); + } + for (const name of ["slack","mail"]) if (env[keys[name]] === "true") { + try {plan("enable",[name],env)} catch(e) {failures.push((e as Error).message)} + } + if (failures.length) throw new Error(`Invalid setup configuration: ${failures.join("; ")}. Review ops config enable/disable before retrying.`); +} +if (import.meta.main) { + try { + const mode = process.argv[2]; + if (mode === "validate") validate(process.env); + else { + const flags = process.argv.slice(3).filter(x => x.endsWith("=true")).map(x => x.slice(2,-5)); + for (const [key,value] of Object.entries(plan(mode,flags,process.env))) console.log(`${key}=${value}`); + } + } catch(e) {console.error(`ERROR: ${(e as Error).message}`);process.exitCode=1;} +} diff --git a/config/docopts.md b/config/docopts.md index 194093e..3fcc1ac 100644 --- a/config/docopts.md +++ b/config/docopts.md @@ -24,7 +24,7 @@ Configure OpenServerless ```text Usage: - config (enable|disable) [--all] [--redis] [--mongodb] [--cron] [--static] [--postgres] [--prometheus] [--slack] [--mail] [--affinity] [--tolerations] [--quota] [--milvus] [--registry] [--seaweedfs] + config (enable|disable) [--all] [--redis] [--mongodb] [--cron] [--static] [--postgres] [--prometheus] [--slack] [--mail] [--affinity] [--tolerations] [--quota] [--milvus] [--registry] [--seaweedfs] [--etcd] [--alertmanager] config apihost (|auto) [--tls=] [--protocol=|auto] config sso config runtimes [] @@ -50,6 +50,7 @@ Usage: config seaweedfs [--s3] [--console] config (status|export|reset) config use [] [--delete] [--rename=] + config full config minimal config slim ``` @@ -60,6 +61,7 @@ Usage: config apihost configure the apihost (auto: auto assign) and enable tls config sso configure SSO/OIDC integration for admin-api; run `ops config sso --help` for details config runtime show the current runtime.json or import the if provided + config full enable all services without automatically enabling notification destinations or scheduling policies config enable enable OpenServerless services to install config disable disable OpenServerless services to install config slack configure Alert Manager over a given slack channel @@ -92,7 +94,9 @@ Usage: ## Options ``` - --all select all services + --etcd select etcd (required by Milvus) + --alertmanager select Alert Manager (requires Prometheus) + --all select all services; enable excludes Slack/mail unless explicitly selected --redis select redis --mongodb select mongodb (FerretDB Proxy) --cron select cron @@ -125,3 +129,11 @@ Usage: --class specify the ingress class. It can be auto, traefik, nginx. On microk8s it should be public. --seaweedfs activate the support of SEAWEEDFS as S3 api and buckets provider ``` + +Enable automatically includes dependencies. Disable cascades to dependent services: +PostgreSQL → MongoDB/FerretDB; etcd → Milvus; SeaweedFS → Milvus/static; +Prometheus → Alert Manager → Slack/mail. Disabling a dependent does not disable +its prerequisites. Slack/mail require explicit selection and configured destinations. +`config full` preserves existing notification/scheduling selections; it does not +activate them. `disable --all` disables every selectable flag, including Alert Manager. +These commands change local configuration, not running Kubernetes resources. diff --git a/config/opsfile.yml b/config/opsfile.yml index 0f4dc04..077581c 100644 --- a/config/opsfile.yml +++ b/config/opsfile.yml @@ -44,88 +44,13 @@ tasks: silent: true cmds: - | - if {{.__redis}} || {{.__all}} - then config OPERATOR_COMPONENT_REDIS=true - fi - - | - if {{.__mongodb}} || {{.__all}} - then - config OPERATOR_COMPONENT_MONGODB=true - - if {{.__mongodb}} - then echo 'MongoDB (FerretDB) deployment requires PosgresSQL. Enabling it.' - config OPERATOR_COMPONENT_POSTGRES=true - fi - fi - - | - if {{.__cron}} || {{.__all}} - then config OPERATOR_COMPONENT_CRON=true - fi - - | - if {{.__static}} || {{.__all}} - then config OPERATOR_COMPONENT_STATIC=true - fi - - | - if {{.__postgres}} || {{.__all}} - then config OPERATOR_COMPONENT_POSTGRES=true - fi - - | - if {{.__prometheus}} || {{.__all}} - then config OPERATOR_COMPONENT_PROMETHEUS=true - fi - - | - if {{.__slack}} || {{.__all}} - then - config OPERATOR_CONFIG_ALERTSLACK=true - fi - - | - if {{.__mail}} || {{.__all}} - then - config OPERATOR_CONFIG_ALERTGMAIL=true - fi - - | - if {{.__affinity}} || {{.__all}} - then - config OPERATOR_CONFIG_AFFINITY=true - fi - - | - if {{.__tolerations}} || {{.__all}} - then - config OPERATOR_CONFIG_TOLERATIONS=true - fi - - | - if {{.__quota}} || {{.__all}} - then - config OPERATOR_COMPONENT_QUOTA=true - fi - - | - if {{.__slack}} || {{.__mail}} || {{.__all}} - then echo 'Alert Manager deployment requires Prometheus. Enabling it.' - config OPERATOR_COMPONENT_PROMETHEUS=true - config OPERATOR_COMPONENT_AM=true - fi - - | - if {{.__milvus}} || {{.__all}} - then - config OPERATOR_COMPONENT_MILVUS=true - config OPERATOR_COMPONENT_ETCD=true - - if {{.__milvus}} - then echo 'Milvus deployment requires SEAWEEDFS. Enabling it.' - config OPERATOR_COMPONENT_SEAWEEDFS=true + CHANGES=$(bun "$OPS_ROOT/config/components.ts" enable --all={{.__all}} --redis={{.__redis}} --mongodb={{.__mongodb}} --cron={{.__cron}} --static={{.__static}} --postgres={{.__postgres}} --prometheus={{.__prometheus}} --slack={{.__slack}} --mail={{.__mail}} --affinity={{.__affinity}} --tolerations={{.__tolerations}} --quota={{.__quota}} --milvus={{.__milvus}} --registry={{.__registry}} --seaweedfs={{.__seaweedfs}} --etcd={{.__etcd}} --alertmanager={{.__alertmanager}}) || exit 1 + for SETTING in $CHANGES; do config "$SETTING"; done + if echo "$CHANGES" | rg -q '^OPERATOR_COMPONENT_REGISTRY=true$'; then + if test -z "$REGISTRY_CONFIG_SECRET_PUSH_PULL"; then + config REGISTRY_CONFIG_SECRET_PUSH_PULL="$(random --str 12)" fi fi - - | - if {{.__registry}} || {{.__all}} - then - config OPERATOR_COMPONENT_REGISTRY=true - config REGISTRY_CONFIG_SECRET_PUSH_PULL="$(random --str 12)" - fi - - | - if {{.__seaweedfs}} || {{.__all}} - then - config OPERATOR_COMPONENT_SEAWEEDFS=true - fi - config CONFIGURED=true - task: status @@ -133,80 +58,22 @@ tasks: silent: true cmds: - | - if {{.__redis}} || {{.__all}} - then config OPERATOR_COMPONENT_REDIS=false - fi - - | - if {{.__mongodb}} || {{.__all}} - then config OPERATOR_COMPONENT_MONGODB=false - fi - - | - if {{.__cron}} || {{.__all}} - then config OPERATOR_COMPONENT_CRON=false - fi - - | - if {{.__static}} || {{.__all}} - then config OPERATOR_COMPONENT_STATIC=false - fi - - | - if {{.__postgres}} || {{.__all}} - then config OPERATOR_COMPONENT_POSTGRES=false - if {{.__postgres}} - then echo 'Disabling PosgresSQL forces deactivation of MongoDB (FerretDB)' - config OPERATOR_COMPONENT_MONGODB=false - fi - fi + CHANGES=$(bun "$OPS_ROOT/config/components.ts" disable --all={{.__all}} --redis={{.__redis}} --mongodb={{.__mongodb}} --cron={{.__cron}} --static={{.__static}} --postgres={{.__postgres}} --prometheus={{.__prometheus}} --slack={{.__slack}} --mail={{.__mail}} --affinity={{.__affinity}} --tolerations={{.__tolerations}} --quota={{.__quota}} --milvus={{.__milvus}} --registry={{.__registry}} --seaweedfs={{.__seaweedfs}} --etcd={{.__etcd}} --alertmanager={{.__alertmanager}}) || exit 1 + for SETTING in $CHANGES; do config "$SETTING"; done + - task: status + + full: + silent: true + cmds: - | - if {{.__prometheus}} || {{.__all}} - then config OPERATOR_COMPONENT_PROMETHEUS=false - if {{.__prometheus}} - then echo 'Disabing Prometheus forces deactivation of Alert Manager.' - config OPERATOR_COMPONENT_AM=false - config OPERATOR_CONFIG_ALERTGMAIL=false - config OPERATOR_CONFIG_ALERTSLACK=false + CHANGES=$(bun "$OPS_ROOT/config/components.ts" full ) || exit 1 + for SETTING in $CHANGES; do config "$SETTING"; done + if echo "$CHANGES" | rg -q '^OPERATOR_COMPONENT_REGISTRY=true$'; then + if test -z "$REGISTRY_CONFIG_SECRET_PUSH_PULL"; then + config REGISTRY_CONFIG_SECRET_PUSH_PULL="$(random --str 12)" fi fi - - | - if {{.__slack}} || {{.__all}} - then - config OPERATOR_CONFIG_ALERTSLACK=false - fi - - | - if {{.__mail}} || {{.__all}} - then - config OPERATOR_CONFIG_ALERTGMAIL=false - fi - - | - if {{.__quota}} || {{.__all}} - then - config OPERATOR_COMPONENT_QUOTA=false - fi - - | - if {{.__affinity}} || {{.__all}} - then - config OPERATOR_CONFIG_AFFINITY=false - fi - - | - if {{.__tolerations}} || {{.__all}} - then - config OPERATOR_CONFIG_TOLERATIONS=false - fi - - | - if {{.__milvus}} || {{.__all}} - then - config OPERATOR_COMPONENT_MILVUS=false - config OPERATOR_COMPONENT_ETCD=false - fi - - | - if {{.__registry}} || {{.__all}} - then - config OPERATOR_COMPONENT_REGISTRY=false - fi - - | - if {{.__seaweedfs}} || {{.__all}} - then - config OPERATOR_COMPONENT_SEAWEEDFS=false - fi + - config CONFIGURED=true - task: status status: diff --git a/setup/cluster-ready.test.ts b/setup/cluster-ready.test.ts new file mode 100644 index 0000000..feb7dba --- /dev/null +++ b/setup/cluster-ready.test.ts @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {test, expect} from "bun:test"; +import {defaultUser, ingress, registrySecret, waitFor} from "./cluster-ready"; +test("ingress requires a hostname, not just an object", async () => { + await expect(ingress(async () => ({spec: {rules: []}}), 0)).rejects.toThrow("Timed out"); + await ingress(async () => ({spec: {rules: [{host: "lab.test"}]}}), 0); +}); +test("missing ingress times out", async () => { + await expect(ingress(async () => undefined, 0)).rejects.toThrow("Timed out"); +}); +test("API errors are not swallowed", async () => { + await expect(ingress(async () => {throw new Error("Forbidden")}, 0)).rejects.toThrow("Forbidden"); +}); +test("disabled registry performs no config read or mutation", async () => { + let calls = 0; + await registrySecret(async () => {calls++;return {spec:{components:{registry:false}}}}, 0); + expect(calls).toBe(1); +}); +test("enabled registry without credentials times out without mutating", async () => { + const calls: string[][] = []; + await expect(registrySecret(async args => { + calls.push(args);return args[1] === "whisk/controller" ? {spec:{components:{registry:true}}} : {metadata:{annotations:{}}}; + }, 0)).rejects.toThrow("registry credentials"); + expect(calls.every(args => args[0] === "get")).toBe(true); +}); +test("credentials applied without delete or password in arguments", async () => { + const calls: any[] = []; + const kube = async (args: string[], input?: string) => { + calls.push({args,input}); + if (args[1] === "whisk/controller") return {spec:{components:{registry:true}}}; + if (args[1] === "cm/config") return {metadata:{annotations:{registry_username:"custom",registry_password:"a b$!",registry_internal_host:"registry:5000"}}}; + }; + await registrySecret(kube, 0); + await registrySecret(kube, 0); + const mutations = calls.filter(x => x.input); + expect(mutations.length).toBe(2); + expect(calls.some(x => x.args[0] === "delete")).toBe(false); + expect(JSON.stringify(calls.map(x=>x.args))).not.toContain("a b$!"); + const secret = JSON.parse(mutations[0].input); + const config = JSON.parse(Buffer.from(secret.data[".dockerconfigjson"],"base64").toString()); + expect(config.auths["registry:5000"].password).toBe("a b$!"); + expect(config.auths["registry:5000"].username).toBe("custom"); +}); + +test("delayed resource becomes available", async () => { + let reads = 0; + expect(await waitFor(async () => ++reads === 3 ? "ready" : undefined, "resource", 100, async () => {})).toBe("ready"); + expect(reads).toBe(3); +}); +test("default user is created only when missing", async () => { + let created = 0; + await defaultUser(async () => created ? {status:{conditions:[{type:"Ready",status:"True"}]}} : undefined, async () => {created++}, 0); + expect(created).toBe(1); +}); +test("repeated setup preserves a ready default user's credentials and data", async () => { + let created = 0; + const calls: string[][] = []; + await defaultUser(async args => {calls.push(args);return {status:{conditions:[{type:"Ready",status:"True"}]}}}, async () => {created++}, 0); + expect(created).toBe(0); + expect(calls.every(args => args[0] === "get")).toBe(true); +}); +test("default user read errors do not trigger creation", async () => { + let created = 0; + await expect(defaultUser(async () => {throw new Error("Forbidden")}, async () => {created++}, 0)).rejects.toThrow("Forbidden"); + expect(created).toBe(0); +}); +test("deleting default user is not recreated", async () => { + let created = 0; + await expect(defaultUser(async () => ({metadata:{deletionTimestamp:"2026-10-08"}}), async () => {created++}, 0)).rejects.toThrow("being deleted"); + expect(created).toBe(0); +}); + +test("new default user must become Ready before setup succeeds", async () => { + let created=0; + await expect(defaultUser(async () => created ? {status:{conditions:[{type:"Ready",status:"False"}]}} : undefined, async () => {created++}, 0)).rejects.toThrow("default user devel to become Ready"); + expect(created).toBe(1); +}); diff --git a/setup/cluster-ready.ts b/setup/cluster-ready.ts new file mode 100644 index 0000000..8075995 --- /dev/null +++ b/setup/cluster-ready.ts @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export type Kube = (args: string[], input?: string) => Promise; +const pause = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); +export async function waitFor(read: () => Promise, description: string, timeout = 600_000, sleep = pause) { + const deadline = Date.now() + timeout; + do { + const value = await read(); + if (value) return value; + if (Date.now() >= deadline) break; + await sleep(Math.min(2000, Math.max(0, deadline - Date.now()))); + } while (true); + throw new Error(`Timed out waiting for ${description}`); +} +export async function ingress(kube: Kube, timeout?: number) { + await waitFor(async () => { + const obj = await kube(["get", "ingress/apihost", "--ignore-not-found", "-o", "json"]); + return obj?.spec?.rules?.[0]?.host; + }, "ingress/apihost with a hostname", timeout); +} +export async function defaultUser(kube: Kube, create: () => Promise, timeout?: number) { + const args = ["get", "wsku/devel", "--ignore-not-found", "-o", "json"]; + const existing = await kube(args); + if (!existing) { + await create(); + } + if (existing?.metadata?.deletionTimestamp) throw new Error("Default user devel is being deleted; retry setup after deletion completes"); + await waitFor(async () => { + const user = await kube(args); + return user?.status?.conditions?.some((c: any) => c.type === "Ready" && c.status === "True"); + }, "default user devel to become Ready", timeout); + console.log(existing ? "Default user devel already exists and is Ready; preserving its credentials and data." : "Default user devel is Ready."); +} +export async function registrySecret(kube: Kube, timeout?: number) { + const whisk = await kube(["get", "whisk/controller", "-o", "json"]); + if (whisk.spec.components.registry !== true) { + console.log("Registry disabled; skipping registry build credentials."); + return; + } + const annotations = await waitFor(async () => { + const cm = await kube(["get", "cm/config", "--ignore-not-found", "-o", "json"]); + const a = cm?.metadata?.annotations; + return a?.registry_password ? a : undefined; + }, "registry credentials from the operator", timeout); + const username = annotations.registry_username || "opsuser"; + const password = annotations.registry_password; + const host = annotations.registry_internal_host || "http://openserverless-registry-svc:5000"; + const config = {auths: {[host]: {username, password, auth: Buffer.from(`${username}:${password}`).toString("base64")}}}; + // Server-side apply avoids persisting a second credential copy in last-applied annotations. + await kube(["apply", "--server-side", "--field-manager=ops-registry-credentials", "-f", "-"], JSON.stringify({ + apiVersion: "v1", kind: "Secret", metadata: {name: "registry-pull-secret-int", namespace: "openserverless"}, + type: "kubernetes.io/dockerconfigjson", data: {".dockerconfigjson": Buffer.from(JSON.stringify(config)).toString("base64")} + })); +} +const kube: Kube = async (args, input) => { + const p = Bun.spawn(["kubectl", "-n", "openserverless", ...args], { + stdin: input === undefined ? "ignore" : new Blob([input]), stdout: "pipe", stderr: "pipe" + }); + const [out, err, code] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited]); + // Mutation errors can include the submitted Secret: do not print them. + if (code !== 0) throw new Error(input ? "Unable to apply registry credentials (check Kubernetes permissions/field ownership)" : `kubectl failed: ${err.trim()}`); + return input ? undefined : out.trim() ? JSON.parse(out) : undefined; +}; +if (import.meta.main) { + try { + if (process.argv[2] === "ingress") await ingress(kube); + else if (process.argv[2] === "registry-secret") await registrySecret(kube); + else if (process.argv[2] === "default-user") await defaultUser(kube, async () => { + const child = Bun.spawn([process.env.OPS_CMD || "ops", "setup", "openserverless", "add-user"], { + stdin: "inherit", stdout: "inherit", stderr: "inherit" + }); + if (await child.exited !== 0) throw new Error("Unable to create the default user devel"); + }); + else throw new Error("Expected ingress, registry-secret or default-user"); + } catch (e) { console.error(`ERROR: ${(e as Error).message}`); process.exitCode = 1; } +} diff --git a/setup/kubernetes/opsfile.yml b/setup/kubernetes/opsfile.yml index 1495f18..6f44e09 100644 --- a/setup/kubernetes/opsfile.yml +++ b/setup/kubernetes/opsfile.yml @@ -319,7 +319,8 @@ tasks: - task: wait vars: OBJECT: - sh: echo "pod/$(cat _seaweedfs)" + sh: | + if [ "$OPERATOR_COMPONENT_SEAWEEDFS" = "true" ]; then echo "pod/$(cat _seaweedfs)"; fi COND: sh: echo "$OPERATOR_COMPONENT_SEAWEEDFS" - task: wait-pod @@ -331,7 +332,8 @@ tasks: - task: wait vars: OBJECT: - sh: echo "pod/$(cat _milvus)" + sh: | + if [ "$OPERATOR_COMPONENT_MILVUS" = "true" ]; then echo "pod/$(cat _milvus)"; fi COND: sh: echo "$OPERATOR_COMPONENT_MILVUS" @@ -366,11 +368,19 @@ tasks: - task: secrets - task: runtimes + wait-ingress: + silent: true + status: + - config SKIP_OPERATOR_LAUNCH + cmds: + - bun "$OPS_ROOT/setup/cluster-ready.ts" ingress + post-deploy: desc: wait results cmds: - task: wait-openwhisk - task: wait-services + - task: wait-ingress deploy: silent: true diff --git a/setup/openserverless/system-api/opsfile.yml b/setup/openserverless/system-api/opsfile.yml index 970c15b..6607fde 100644 --- a/setup/openserverless/system-api/opsfile.yml +++ b/setup/openserverless/system-api/opsfile.yml @@ -47,10 +47,6 @@ env: #sh: ops util kubectl -- --kubeconfig "$KUBECONFIG" -n openserverless get ingress/apihost -o jsonpath='{.spec.rules[0].host}' sh: ops util kubeget ingress/apihost .spec.rules[0].host - REGISTRY_PASS: - #sh: ops util kubectl -- --kubeconfig "$KUBECONFIG" -n openserverless get cm/config -o jsonpath='{.metadata.annotations.registry_password}' - sh: ops util kubeget cm/config .metadata.annotations.registry_password - INGRESS_TYPE: sh: ops util ingress-type @@ -75,11 +71,7 @@ tasks: desc: Create secret to push to internal registry silent: true cmds: - - task: remove-secret - - | - if ! kubectl -n openserverless get secret registry-pull-secret-int >/dev/null 2>&1; - then kubectl -n openserverless create secret docker-registry registry-pull-secret-int --docker-server=http://openserverless-registry-svc:5000 --docker-username=opsuser --docker-password=${REGISTRY_PASS} - fi + - bun "$OPS_ROOT/setup/cluster-ready.ts" registry-secret remove-secret: desc: Delete secret to push to internal registry diff --git a/setup/opsfile.yml b/setup/opsfile.yml index 8f67b9a..29a24e1 100644 --- a/setup/opsfile.yml +++ b/setup/opsfile.yml @@ -159,12 +159,13 @@ tasks: elif {{.__uninstall}} then {{.RUN}} ops setup kubernetes delete CONTEXT="{{._context_}}" else + bun "$OPS_ROOT/config/components.ts" validate {{.RUN}} ops setup kubernetes create CONTEXT="{{._context_}}" {{.RUN}} ops setup openserverless streamer deploy {{.RUN}} ops setup openserverless system-api deploy {{.RUN}} ops setup openserverless tika deploy {{.RUN}} ops setup openserverless login - {{.RUN}} ops setup openserverless add-user + bun "$OPS_ROOT/setup/cluster-ready.ts" default-user config STATUS_LAST=cluster if test -n "{{._context_}}" then config STATUS_CONTEXT="{{._context_}}"