diff --git a/cmd/spinloop/cloud.go b/cmd/spinloop/cloud.go index 03cf656c..2cc4515e 100644 --- a/cmd/spinloop/cloud.go +++ b/cmd/spinloop/cloud.go @@ -831,6 +831,22 @@ func modelPresetKey(runner string) string { } } +// modelSourceFlags names the canonical flag keys under which a PARAM (or a +// preset, through the same keys) may name a runner's model — llamacpp's +// hf-repo/model/hf-file, mtplx's model. A runner that takes its model +// positionally has none, so a PARAM cannot stand in for a MODEL there and the +// error that names the ways a model may be given does not name one either. +func modelSourceFlags(runner string) []string { + switch runner { + case "llamacpp": + return []string{"hf-repo", "model", "hf-file"} + case "mtplx": + return []string{"model"} + default: + return nil + } +} + // contextPresetKey names the preset key holding the context window, in that // runner's own vocabulary. func contextPresetKey(runner string) string { @@ -876,9 +892,10 @@ func dropOwned(owned func(string) bool, params []preset.Param) []preset.Param { // deployConfigFor turns a Spinloop (plus its preset, if any) into the config the // deploy Lambda accepts. The Spinloop is the single source of truth: PROVIDER -// picks the runner, MODEL or the preset's hf names the weights, CONTEXT the -// window, ALIAS the served name, and whatever else the preset sets becomes the -// runner's own flags. +// picks the runner, MODEL, the preset's model reference, or a PARAM naming the +// model names the weights, CONTEXT the window, ALIAS the served name, and +// whatever the preset and the PARAMs set becomes the runner's own flags, a +// PARAM standing over the preset where both set a flag. // // A context size is required, because a cloud instance has to be sized before // it is provisioned. deployConfigWithoutContext is the same derivation for a @@ -932,12 +949,13 @@ func deployConfig(sel spinloop.Selection, spinloopPath string, target deployTarg } dc.Runner = runner - // The preset supplies the model and the runner flags when the Spinloop does - // not state them, so a single preset can drive both serve and deploy. The - // [*] globals are a separate layer that the chosen section overrides — - // exactly as preset.Args does for a local serve — so settings written there - // (commonly ngl and jinja) are not lost. - var global, params []preset.Param + // The preset and the PARAMs supply the model and the runner flags when the + // Spinloop does not state them, so a single Spinloop can drive both serve + // and deploy. The [*] globals are a separate layer that the chosen section + // overrides, and the PARAMs stand over the preset — the same layering a + // local serve gives them — so settings written there (commonly ngl and + // jinja) are not lost. + var global, params, paramLayer []preset.Param if sel.Preset != "" { presetPath, err := resolvePresetPath(sel.Preset, spinloopPath) if err != nil { @@ -958,14 +976,35 @@ func deployConfig(sel spinloop.Selection, spinloopPath string, target deployTarg global, params = pre.Global, sec.Params } + // The Spinloop's own PARAMs are the layer between the preset and its typed + // instructions: they supply the model, the context window, and the slot + // count the preset's keys do, and ride into ServeArgs with the preset's + // own layers, with the same owned-flag dropping. + paramLayer = selectionParams(sel) + + // The model falls back in the Spinloop's own order: MODEL, then a PARAM + // naming the model for a runner that takes it as a flag, then the preset's + // model key — a PARAM standing over the preset, exactly as one does for a + // local serve. The PARAM is read in the runner's own dialect, through the + // engine: its spelling of the model key is not the preset's (an MTPLX + // preset's `m` is not a model key, for instance). model := sel.Model + if model == "" { + engine, err := engineFor(dc.Runner) + if err != nil { + return dc, err + } + model, _ = paramModel(engine, paramLayer) + } if model == "" { model = presetValue(modelPresetKey(dc.Runner), global, params) } if model == "" { - return dc, fmt.Errorf( - "nothing to deploy: set MODEL in %s, or %s in its preset", - spinloopPath, modelPresetKey(dc.Runner)) + fix := fmt.Sprintf("set MODEL in %s, or %s in its preset", spinloopPath, modelPresetKey(dc.Runner)) + if len(modelSourceFlags(dc.Runner)) > 0 { + fix += ", or a PARAM naming the model" + } + return dc, fmt.Errorf("nothing to deploy: %s", fix) } // A local model path is refused only where the destination fetches the // weights itself — the cloud, which cannot ship a file. A node has the file @@ -981,10 +1020,10 @@ func deployConfig(sel spinloop.Selection, spinloopPath string, target deployTarg context := sel.Context if context == "" { - context = presetValue(contextPresetKey(dc.Runner), global, params) + context = presetValue(contextPresetKey(dc.Runner), global, params, paramLayer) } if context == "" && target.requireContext { - return dc, fmt.Errorf("no context size: set CONTEXT in %s, or %s in its preset", spinloopPath, contextPresetKey(dc.Runner)) + return dc, fmt.Errorf("no context size: set CONTEXT in %s, or %s in its preset, or a PARAM naming it", spinloopPath, contextPresetKey(dc.Runner)) } if context != "" { n, err := contextsize.Parse(context) @@ -994,16 +1033,16 @@ func deployConfig(sel spinloop.Selection, spinloopPath string, target deployTarg dc.ContextSize = n } - // Parallel falls back to the preset's own parallelism key the same way - // context falls back to ctx-size, since that value is about to be dropped - // from serveArgs below (it is cloud-owned) — capturing it here is what - // keeps it from being silently lost rather than re-emitted as the + // Parallel falls back to the preset's and PARAMs' own parallelism key the + // same way context falls back to ctx-size, since that value is about to be + // dropped from serveArgs below (it is cloud-owned) — capturing it here is + // what keeps it from being silently lost rather than re-emitted as the // deployment's own computed flag. The key is the runner's own spelling: // reading only llama.cpp's would drop a vLLM preset's max-num-seqs on the // floor, since dropOwned removes it either way. parallel := sel.Parallel if key := parallelPresetKey(dc.Runner); parallel == "" && key != "" { - parallel = presetValue(key, global, params) + parallel = presetValue(key, global, params, paramLayer) } if parallel != "" { n, err := parseParallel(parallel) @@ -1020,20 +1059,25 @@ func deployConfig(sel spinloop.Selection, spinloopPath string, target deployTarg dc.ServedModelName = dc.ModelID } - // Companion weights are read from the same preset keys that drive a local - // serve, so one preset still works both ways. Only where the destination - // fetches the weights itself: it pulls them from the model's own repo, so - // only the filename travels and the local path the user downloaded to is - // meaningless there. A node already has its files, and keeps its own path. + // Companion weights are read from the same preset and PARAM keys that + // drive a local serve, so one Spinloop still works both ways. Only where + // the destination fetches the weights itself: it pulls them from the + // model's own repo, so only the filename travels and the local path the + // user downloaded to is meaningless there. A node already has its files, + // and keeps its own path. if target.seedsWeights { - dc.Companions = companionsFrom(global, params) + dc.Companions = companionsFrom(global, params, paramLayer) } // The Spinloop's own bind is the final layer, where it travels at all: it - // wins over the preset's host and port exactly as it does for a local - // serve, so a woken engine listens wherever the Spinloop says rather than - // on the engine's default. - layers := [][]preset.Param{dropOwned(target.owns, global), dropOwned(target.owns, params)} + // wins over the preset's and PARAMs' host and port exactly as it does for + // a local serve, so a woken engine listens wherever the Spinloop says + // rather than on the engine's default. + layers := [][]preset.Param{ + dropOwned(target.owns, global), + dropOwned(target.owns, params), + dropOwned(target.owns, paramLayer), + } if target.carriesBaseURL { bind, err := bindAddressParams(sel) if err != nil { diff --git a/cmd/spinloop/cloud_deploy_test.go b/cmd/spinloop/cloud_deploy_test.go index d7ee0345..702b1b99 100644 --- a/cmd/spinloop/cloud_deploy_test.go +++ b/cmd/spinloop/cloud_deploy_test.go @@ -294,6 +294,213 @@ func TestDeployConfigFor_Rejects(t *testing.T) { } } +// A PARAM is a layer of its own in the deployment: it supplies the model, the +// window, and the slot count when the Spinloop does not state them, and its +// remaining values ride into ServeArgs with the same owned-flag dropping a +// preset gets — so a Spinloop with no preset file at all can still be +// deployed, on the same terms `spinloop serve` runs it. +func TestDeployConfigFor_FromParams(t *testing.T) { + dc := deployConfigFrom(t, + "PROVIDER llamacpp\nALIAS qwen3.8-27b\n"+ + "PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL\n"+ + "PARAM ctx-size = 131072\n"+ + "PARAM np = 2\n"+ + "PARAM ngl = 99\n"+ + "PARAM fa = on\n", "") + + if dc.Runner != "llamacpp" { + t.Errorf("runner = %q, want llamacpp", dc.Runner) + } + if dc.ModelID != "unsloth/Qwen3.8-27B-GGUF" || dc.Quant != "UD-Q5_K_XL" { + t.Errorf("model = %q quant = %q, want the PARAM's repo and quant split", dc.ModelID, dc.Quant) + } + if dc.ContextSize != 131072 { + t.Errorf("contextSize = %d, want the PARAM's ctx-size", dc.ContextSize) + } + if dc.Parallel != 2 { + t.Errorf("parallel = %d, want the PARAM's np", dc.Parallel) + } + if dc.ServedModelName != "qwen3.8-27b" { + t.Errorf("servedModelName = %q, want the ALIAS", dc.ServedModelName) + } + args := strings.Join(dc.ServeArgs, " ") + // The PARAMs the deployment does not compute itself ride through. + for _, want := range []string{"--n-gpu-layers 99", "--flash-attn on"} { + if !strings.Contains(args, want) { + t.Errorf("serveArgs %q is missing the PARAM's %q", args, want) + } + } + // Set by the cloud, so the PARAMs' values must not be echoed back — the + // model travels as ModelID and Quant, not as a flag. + for _, unwanted := range []string{"--hf-repo", "--ctx-size", "--parallel", "unsloth/Qwen3.8-27B-GGUF"} { + if strings.Contains(args, unwanted) { + t.Errorf("serveArgs %q should not carry the cloud-owned %q", args, unwanted) + } + } +} + +// Where a flag is set both in the preset and as a PARAM, the Spinloop's own +// PARAM stands over the preset — the same precedence a local serve gives it — +// and a preset value the PARAM does not set still rides through. +func TestDeployConfigFor_ParamBeatsPreset(t *testing.T) { + preset := "[*]\n" + + "ngl = 10\n" + + "\n" + + "[q]\n" + + "hf = org/model:Q4_K_M\n" + + "ctx-size = 4096\n" + + "np = 4\n" + + "temp = 0.7\n" + dc := deployConfigFrom(t, + "PROVIDER llamacpp\nALIAS q\n"+ + "PARAM hf = org/other:Q8_0\n"+ + "PARAM ctx-size = 8192\n"+ + "PARAM temp = 1.0\n"+ + "PRESET ./preset.ini\n", preset) + + // The model falls back MODEL, then PARAM, then the preset: the PARAM's + // repo deploys. + if dc.ModelID != "org/other" || dc.Quant != "Q8_0" { + t.Errorf("the PARAM's hf should win over the preset's, got %q:%q", dc.ModelID, dc.Quant) + } + // The window: the PARAM stands over the preset's ctx-size. + if dc.ContextSize != 8192 { + t.Errorf("contextSize = %d, want the PARAM's ctx-size, not the preset's", dc.ContextSize) + } + // The slot count: the PARAM does not set it, so the preset's np stands. + if dc.Parallel != 4 { + t.Errorf("parallel = %d, want the preset's np (no PARAM sets it)", dc.Parallel) + } + args := strings.Join(dc.ServeArgs, " ") + if !strings.Contains(args, "--temp 1.0") || strings.Contains(args, "0.7") { + t.Errorf("the PARAM's temp should stand over the preset's, got %q", args) + } + // A preset value no PARAM sets still rides through, from the [*] globals. + if !strings.Contains(args, "--n-gpu-layers 10") { + t.Errorf("the preset's ngl should ride through, got %q", args) + } +} + +// The owned-flag drop is per-destination, and the PARAM layer goes through it +// exactly as the preset's layers do: a PARAM's bind is dropped for a cloud +// deploy (the cloud assigns its own) and stands for a node wake (the machine +// is the operator's), and the node wake derives the model, the window, and the +// slot count from PARAMs the same way a deploy does. +func TestDeployConfigForNode_ParamsLikePreset(t *testing.T) { + spinloopPath := writeDeploySpinloop(t, + "PROVIDER llamacpp\nALIAS qwen\n"+ + "PARAM hf = org/model:Q4_K_M\n"+ + "PARAM ctx-size = 4096\n"+ + "PARAM np = 4\n"+ + "PARAM host = 0.0.0.0\n"+ + "PARAM port = 9090\n", "") + sel, _, err := readSpinloop("test", spinloopPath) + if err != nil { + t.Fatal(err) + } + + node, err := deployConfigForNode(sel, spinloopPath) + if err != nil { + t.Fatalf("deployConfigForNode: %v", err) + } + if node.ModelID != "org/model" || node.Quant != "Q4_K_M" { + t.Errorf("model = %q:%q, want the PARAM's hf", node.ModelID, node.Quant) + } + if node.ContextSize != 4096 { + t.Errorf("contextSize = %d, want the PARAM's ctx-size", node.ContextSize) + } + if node.Parallel != 4 { + t.Errorf("parallel = %d, want the PARAM's np", node.Parallel) + } + nodeArgs := strings.Join(node.ServeArgs, " ") + for _, want := range []string{"--host 0.0.0.0", "--port 9090"} { + if !strings.Contains(nodeArgs, want) { + t.Errorf("a node's serve args should keep the PARAM's bind %q, got: %s", want, nodeArgs) + } + } + for _, unwanted := range []string{"--hf-repo", "--ctx-size", "--parallel"} { + if strings.Contains(nodeArgs, unwanted) { + t.Errorf("a node's serve args should not repeat %q, got: %s", unwanted, nodeArgs) + } + } + + // The cloud still drops the same bind: it assigns its own. + sel.Context = "4096" // the cloud requires one + cloud, err := deployConfigFor(sel, spinloopPath) + if err != nil { + t.Fatal(err) + } + cloudArgs := strings.Join(cloud.ServeArgs, " ") + for _, unwanted := range []string{"--host", "--port", "0.0.0.0", "9090"} { + if strings.Contains(cloudArgs, unwanted) { + t.Errorf("the cloud sets its own bind, so %q should be dropped, got: %s", unwanted, cloudArgs) + } + } +} + +// A PARAM naming the model may be a local path, the way a MODEL may be: the +// cloud refuses it (it cannot ship a file), and a node wake carries it, the +// machine having the file the Spinloop points at. +func TestDeployConfigFor_ParamNamesLocalModelPath(t *testing.T) { + spinloopPath := writeDeploySpinloop(t, + "PROVIDER llamacpp\nCONTEXT 32k\nPARAM hf = ./models/qwen.gguf\n", "") + sel, _, err := readSpinloop("test", spinloopPath) + if err != nil { + t.Fatal(err) + } + + node, err := deployConfigForNode(sel, spinloopPath) + if err != nil { + t.Fatalf("a node wake with a PARAM's local model path should succeed: %v", err) + } + if node.ModelID != "./models/qwen.gguf" { + t.Errorf("modelId = %q, want the local path", node.ModelID) + } + + // The cloud still refuses the same file: it downloads from Hugging Face. + if _, err := deployConfigFor(sel, spinloopPath); err == nil || + !strings.Contains(err.Error(), "cannot deploy the local model file") { + t.Errorf("the cloud should refuse a PARAM's local model path, got %v", err) + } +} + +// vLLM takes its model positionally, so a PARAM naming one is not a model to +// it: the error stays MODEL-and-preset, unlike the flag-taking engines whose +// error names a PARAM as a third way. +func TestDeployConfigFor_VllmParamsAreNotAModel(t *testing.T) { + spinloopPath := writeDeploySpinloop(t, + "PROVIDER vllm\nCONTEXT 32k\nPARAM hf = org/model\n", "") + sel, _, err := readSpinloop("test", spinloopPath) + if err != nil { + t.Fatal(err) + } + _, err = deployConfigFor(sel, spinloopPath) + if err == nil || !strings.Contains(err.Error(), "nothing to deploy") { + t.Fatalf("want a nothing-to-deploy error, got %v", err) + } + if strings.Contains(err.Error(), "PARAM") { + t.Errorf("vLLM takes its model positionally, so the error should not name a PARAM: %v", err) + } +} + +// The flag-taking engines' error does name the PARAM, as the third way a model +// may be given. +func TestDeployConfigFor_NoModelErrorNamesAParam(t *testing.T) { + spinloopPath := writeDeploySpinloop(t, + "PROVIDER llamacpp\nCONTEXT 32k\n", "") + sel, _, err := readSpinloop("test", spinloopPath) + if err != nil { + t.Fatal(err) + } + _, err = deployConfigFor(sel, spinloopPath) + if err == nil || !strings.Contains(err.Error(), "nothing to deploy") { + t.Fatalf("want a nothing-to-deploy error, got %v", err) + } + if !strings.Contains(err.Error(), "a PARAM naming the model") { + t.Errorf("the error should name a PARAM as an alternative: %v", err) + } +} + func TestSplitModelQuant(t *testing.T) { for _, tc := range []struct{ in, repo, quant string }{ {"org/model:Q4_K_M", "org/model", "Q4_K_M"}, diff --git a/cmd/spinloop/serve.go b/cmd/spinloop/serve.go index b3ca26fd..e9d30467 100644 --- a/cmd/spinloop/serve.go +++ b/cmd/spinloop/serve.go @@ -82,6 +82,12 @@ type serveEngine struct { // load. oMLX serves a whole model directory and picks per request, so it // starts happily with neither a PRESET nor a MODEL. needsModel bool + // modelFlags are the canonical flag names that name the engine's model, + // from modelSourceFlags. A PARAM set to one of them names the model in + // place of a MODEL, the way a preset's model reference does. An engine + // that takes its model positionally (vllm) has none: a PARAM cannot stand + // in for a MODEL there. + modelFlags []string // installHint completes " not found — ...". installHint string // metricsArgs switches the engine's own /metrics endpoint on. Appended @@ -119,6 +125,7 @@ var engines = map[string]serveEngine{ dialect: preset.LlamaCpp, params: llamacppServeParams, needsModel: true, + modelFlags: modelSourceFlags("llamacpp"), installHint: "install llama.cpp (e.g. brew install llama.cpp) or check the path", metricsArgs: []string{"--metrics"}, metricsEngine: "llamacpp", @@ -139,6 +146,7 @@ var engines = map[string]serveEngine{ dialect: preset.MTPLX, params: mtplxServeParams, needsModel: true, + modelFlags: modelSourceFlags("mtplx"), installHint: "install MTPLX (https://mtplx.com) or check the path", // MTPLX has no Prometheus endpoint to scrape, so there is no // metrics switch to append and no dialect for the scraper. @@ -215,9 +223,10 @@ func serveCmd() *cobra.Command { Short: "run the Spinloop's inference server", Long: fmt.Sprintf(`runs the inference server the Spinloop's PROVIDER names — %s. With a PRESET it turns the matching section into the command, reading it in -that engine's flag vocabulary; otherwise it derives one from the Spinloop's -own instructions. Prints the command before running it; --dry-run/-n prints -without launching the server. +that engine's flag vocabulary; PARAM lines pass the same options inline, +standing over the preset where both set a flag. Otherwise it derives the +command from the Spinloop's own instructions. Prints the command before +running it; --dry-run/-n prints without launching the server. On a terminal the server runs under a full-screen view: its metrics — each resource series as a gauge of the current reading with its retained history @@ -316,14 +325,18 @@ func runServe(args []string, dryRun, apiOn bool, apiAddr, logLevel string) error // buildServeArgv turns a Spinloop into the engine command, from its PRESET // section when it names one and from the Spinloop's own instructions otherwise, -// narrating which source it used to w. It is `spinloop serve`'s alone: the -// daemon reads no Spinloop, so nothing else builds a command this way. +// narrating which source it used to w. The Spinloop's PARAMs ride along either +// way, standing over the preset and under its typed instructions. It is +// `spinloop serve`'s alone: the daemon reads no Spinloop, so nothing else +// builds a command this way. func buildServeArgv(w io.Writer, engine serveEngine, sel spinloop.Selection, spinloopPath string) ([]string, error) { - // Anything the Spinloop states overrides the preset's own values. + // Anything the Spinloop states overrides its PARAMs, which override the + // preset's own values. params, err := engine.params(sel) if err != nil { return nil, err } + selParams := selectionParams(sel) if sel.Preset != "" { presetPath, err := resolvePresetPath(sel.Preset, spinloopPath) @@ -344,17 +357,26 @@ func buildServeArgv(w io.Writer, engine serveEngine, sel spinloop.Selection, spi if err != nil { return nil, fmt.Errorf("%s: %w", presetPath, err) } - argv := pre.CommandIn(engine.dialect, engine.binary(), subcommandFor(engine, sel), sec, params) + argv := pre.CommandIn(engine.dialect, engine.binary(), subcommandFor(engine, sel), sec, selParams, params) fmt.Fprintf(w, "Using preset %s (model %s)\n\n", presetPath, sec.Name) return argv, nil } - if engine.needsModel && sel.Model == "" { - return nil, fmt.Errorf("serve needs a PRESET or a MODEL (an HF repo like org/model:quant, or a path to a .gguf)") + model := sel.Model + if model == "" { + // For an engine that takes its model as a flag, a PARAM naming the + // model serves in place of a MODEL. + model, _ = paramModel(engine, selParams) } - argv := assembleEngineArgv(engine, subcommandFor(engine, sel), params, nil) - if sel.Model != "" { - fmt.Fprintf(w, "Serving %s from %s\n\n", sel.Model, spinloopPath) + if engine.needsModel && model == "" { + if len(engine.modelFlags) == 0 { + return nil, fmt.Errorf("serve needs a PRESET or a MODEL: %s takes its model positionally, where a PARAM cannot stand in", sel.Provider) + } + return nil, fmt.Errorf("serve needs a PRESET, a MODEL (an HF repo like org/model:quant, or a path to a .gguf), or a PARAM naming the model") + } + argv := assembleEngineArgv(engine, subcommandFor(engine, sel), engine.dialect.Merge(selParams, params), nil) + if model != "" { + fmt.Fprintf(w, "Serving %s from %s\n\n", model, spinloopPath) } else { // An engine that needs no model to start (oMLX serves a whole // directory) has nothing to name but itself. @@ -363,6 +385,35 @@ func buildServeArgv(w io.Writer, engine serveEngine, sel spinloop.Selection, spi return argv, nil } +// selectionParams converts the Spinloop's PARAM instructions into the preset +// layer they render as, in file order. +func selectionParams(sel spinloop.Selection) []preset.Param { + var params []preset.Param + for _, p := range sel.Params { + params = append(params, preset.Param{Key: p.Key, Value: p.Value}) + } + return params +} + +// paramModel returns the model a PARAM layer names for this engine, if any: +// the value of the last PARAM whose key is one of the engine's model flags, +// which is how the dialect merge would resolve repeated keys. Engines whose +// model is positional have no model flags and so never find one. The deploy +// path shares it, reading the same PARAMs in the same runner's dialect. +func paramModel(engine serveEngine, params []preset.Param) (string, bool) { + var model string + found := false + for _, p := range params { + key := engine.dialect.CanonicalKey(p.Key) + for _, flag := range engine.modelFlags { + if key == flag && p.Value != "" { + model, found = p.Value, true + } + } + } + return model, found +} + // subcommandFor returns the engine's subcommand with the positional model riding // along right after it (before any flags). It copies the engine's own subcommand // before appending, so a positional never aliases that slice. diff --git a/cmd/spinloop/serve_test.go b/cmd/spinloop/serve_test.go index b4a6eebb..14b604ac 100644 --- a/cmd/spinloop/serve_test.go +++ b/cmd/spinloop/serve_test.go @@ -892,3 +892,206 @@ func TestCmdServe_UnsupportedProvider(t *testing.T) { } } } + +// TestCmdServe_ParamOverridesPreset is the rule the keyword exists for: where a +// param is set both in the preset and as a PARAM, the Spinloop file wins. +func TestCmdServe_ParamOverridesPreset(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "preset.ini"), "[qwen]\nhf = org/model\ntemp = 0.7\n") + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, "PROVIDER llamacpp\nALIAS qwen\nPARAM temp = 1.0\nPRESET preset.ini\n") + + out := captureStdout(t, func() { + if err := cmdServe([]string{"--dry-run", spinloopPath}); err != nil { + t.Fatalf("cmdServe: %v", err) + } + }) + if !strings.Contains(out, "--temp 1.0") { + t.Errorf("PARAM should override the preset's temp, missing --temp 1.0:\n%s", out) + } + if strings.Contains(out, "--temp 0.7") { + t.Errorf("the preset's temp should have lost to the PARAM:\n%s", out) + } +} + +// TestCmdServe_InstructionOverridesParam checks the other side of the same +// rule: a typed instruction stands over a PARAM, so CONTEXT wins over a +// PARAM's own ctx-size — and unscaled, since it is the Spinloop's own. +func TestCmdServe_InstructionOverridesParam(t *testing.T) { + dir := t.TempDir() + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, "PROVIDER llamacpp\nMODEL org/model\nCONTEXT 32768\nPARAM ctx-size = 5000\n") + + out := captureStdout(t, func() { + if err := cmdServe([]string{"--dry-run", spinloopPath}); err != nil { + t.Fatalf("cmdServe: %v", err) + } + }) + if !strings.Contains(out, "--ctx-size 32768") { + t.Errorf("CONTEXT should stand over the PARAM's ctx-size, missing --ctx-size 32768:\n%s", out) + } + if strings.Contains(out, "--ctx-size 5000") { + t.Errorf("the PARAM's ctx-size should have lost to CONTEXT:\n%s", out) + } +} + +// TestCmdServe_ParamNamesModel is the issue's own configuration: no MODEL and +// no PRESET, the model named entirely by a PARAM, with the PARAMs rendering in +// llama.cpp's dialect (alias-normalised, booleans bare). +func TestCmdServe_ParamNamesModel(t *testing.T) { + dir := t.TempDir() + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, + "PROVIDER llamacpp\n"+ + "ALIAS qwen3.8-27b\n"+ + "PARAM jinja = 1\n"+ + "PARAM temp = 1.0\n"+ + "PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL\n"+ + "PARAM no-mmproj = 1\n") + + out := captureStdout(t, func() { + if err := cmdServe([]string{"--dry-run", spinloopPath}); err != nil { + t.Fatalf("cmdServe: %v", err) + } + }) + for _, want := range []string{ + "--hf-repo unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL", + "--alias qwen3.8-27b", + "--jinja", + "--temp 1.0", + "--no-mmproj", + } { + if !strings.Contains(out, want) { + t.Errorf("command missing %q:\n%s", want, out) + } + } + if !strings.Contains(out, "Serving unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL") { + t.Errorf("the narration should name the model the PARAM gives:\n%s", out) + } +} + +// TestCmdServe_MTPPLXParamNamesModel is the same model-sourcing for MTPLX, +// whose dialect passes keys through unchanged. +func TestCmdServe_MTPPLXParamNamesModel(t *testing.T) { + dir := t.TempDir() + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, "PROVIDER mtplx\nPARAM model = org/mtplx-pack\nPARAM scheduler-mode = ar_batch\n") + + out := captureStdout(t, func() { + if err := cmdServe([]string{"--dry-run", spinloopPath}); err != nil { + t.Fatalf("cmdServe: %v", err) + } + }) + for _, want := range []string{"--model org/mtplx-pack", "--scheduler-mode ar_batch"} { + if !strings.Contains(out, want) { + t.Errorf("command missing %q:\n%s", want, out) + } + } +} + +// TestCmdServe_VllmNeedsModelDespiteParams pins the one engine a PARAM cannot +// name the model for: vLLM takes it positionally, so the MODEL requirement +// stands even with PARAMs present, and the error says why. +func TestCmdServe_VllmNeedsModelDespiteParams(t *testing.T) { + dir := t.TempDir() + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, "PROVIDER vllm\nPARAM gpu-memory-utilization = 0.9\n") + + var err error + captureStdout(t, func() { err = cmdServe([]string{"--dry-run", spinloopPath}) }) + if err == nil { + t.Fatal("expected an error: a PARAM cannot stand in for vLLM's positional model") + } + if !strings.Contains(err.Error(), "positionally") { + t.Errorf("error should name the positional argument, got: %v", err) + } +} + +// TestCmdServe_NoModelErrorNamesTheWays checks the no-model failure explains +// all three ways a model can be named, not just MODEL. +func TestCmdServe_NoModelErrorNamesTheWays(t *testing.T) { + dir := t.TempDir() + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, "PROVIDER llamacpp\n") + + var err error + captureStdout(t, func() { err = cmdServe([]string{"--dry-run", spinloopPath}) }) + if err == nil { + t.Fatal("expected an error when there is neither a PRESET nor a MODEL") + } + for _, want := range []string{"PRESET", "MODEL", "PARAM"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should name %s as a way to supply the model, got: %v", want, err) + } + } +} + +// TestCmdServe_VllmParamsRender checks PARAMs ride into a vLLM command as its +// own flags: the value-taking ones with their values, and an empty value as a +// bare flag, which is how a dialect with no boolean table spells one. +func TestCmdServe_VllmParamsRender(t *testing.T) { + dir := t.TempDir() + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, + "PROVIDER vllm\nMODEL org/model\nALIAS friendly\n"+ + "PARAM gpu-memory-utilization = 0.9\n"+ + "PARAM enable-chunked-prefill =\n") + + out := captureStdout(t, func() { + if err := cmdServe([]string{"--dry-run", spinloopPath}); err != nil { + t.Fatalf("cmdServe: %v", err) + } + }) + for _, want := range []string{"vllm serve org/model", "--gpu-memory-utilization 0.9", "--enable-chunked-prefill", "--served-model-name friendly"} { + if !strings.Contains(out, want) { + t.Errorf("command missing %q:\n%s", want, out) + } + } +} + +// TestCmdServe_ParamContextUnscaledByParallel checks a PARAM's own ctx-size +// holds the same place a preset's does: PARALLEL emits its flag, but the +// PARAM's context passes through unscaled. +func TestCmdServe_ParamContextUnscaledByParallel(t *testing.T) { + dir := t.TempDir() + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, + "PROVIDER llamacpp\nPARAM model = org/model\nPARALLEL 2\nPARAM ctx-size = 4096\n") + + out := captureStdout(t, func() { + if err := cmdServe([]string{"--dry-run", spinloopPath}); err != nil { + t.Fatalf("cmdServe: %v", err) + } + }) + if !strings.Contains(out, "--parallel 2") { + t.Errorf("command missing --parallel 2:\n%s", out) + } + if !strings.Contains(out, "--ctx-size 4096") { + t.Errorf("the PARAM's ctx-size should pass through unscaled, missing --ctx-size 4096:\n%s", out) + } + if strings.Contains(out, "--ctx-size 8192") { + t.Errorf("PARALLEL must not rescale a PARAM's ctx-size:\n%s", out) + } +} + +// TestCmdServe_ParallelOverridesParamValue checks a Spinloop's PARALLEL wins +// over a PARAM's own np, by the same override-by-canonical-name rule it uses +// against a preset. +func TestCmdServe_ParallelOverridesParamValue(t *testing.T) { + dir := t.TempDir() + spinloopPath := filepath.Join(dir, "Spinloop") + mustWrite(t, spinloopPath, + "PROVIDER llamacpp\nPARAM model = org/model\nPARALLEL 2\nPARAM np = 4\n") + + out := captureStdout(t, func() { + if err := cmdServe([]string{"--dry-run", spinloopPath}); err != nil { + t.Fatalf("cmdServe: %v", err) + } + }) + if !strings.Contains(out, "--parallel 2") { + t.Errorf("the Spinloop's PARALLEL should win, missing --parallel 2:\n%s", out) + } + if strings.Contains(out, "--parallel 4") { + t.Errorf("the PARAM's np=4 should have been overridden:\n%s", out) + } +} diff --git a/docs/commands/cloud.md b/docs/commands/cloud.md index ba0f922f..5090c042 100644 --- a/docs/commands/cloud.md +++ b/docs/commands/cloud.md @@ -423,8 +423,9 @@ recovered — only what's shipped from then on. ## Creating an endpoint: `deploy` `spinloop cloud deploy` creates an **environment** on the bootstrapped control -plane and tells it what to serve. It reads the Spinloop and its preset — -`PROVIDER` picks the engine, so the file that runs a model locally under +plane and tells it what to serve. It reads the Spinloop, its preset and its +PARAM lines — `PROVIDER` picks the engine, so the file that runs a model +locally under [`spinloop serve`](serve.md) deploys the same model remotely — and `--env ` names the environment it creates and registers. The flag is required: a deploy without it fails, naming the flag and a name to use. The Spinloop says what the @@ -435,7 +436,7 @@ file: PROVIDER llamacpp # the engine to run: llamacpp or vllm ALIAS qwen3.6-27b # the name your agent asks for — and the name served CONTEXT 131072 -PRESET ./preset.ini # the model and its flags +PRESET ./preset.ini # the model and its flags — or PARAM lines instead ``` ```sh @@ -446,8 +447,8 @@ Deploy discovers the control plane from the bootstrap stack's outputs, then provisions the environment's own Elastic IP, API key, ingress rule and state, registers it under `~/.config/spinloop/clouds//`, and stores what to serve. Everything the endpoint sets itself — host, port, where the weights live, the -API key, the context size, the alias — is dropped from the preset, so one -preset works both locally and remotely without edits. +API key, the context size, the alias — is dropped from the preset and the +PARAMs alike, so one file works both locally and remotely without edits. Who may reach the instance is **per environment**: `--allowed-cidr` sets it, defaulting to your public IP as a `/32` on first deploy; later deploys leave diff --git a/docs/commands/serve.md b/docs/commands/serve.md index bd1aa66f..b63de770 100644 --- a/docs/commands/serve.md +++ b/docs/commands/serve.md @@ -81,9 +81,50 @@ GPU: Any other provider is an error: `serve` launches a self-hosted engine, and the rest of the catalogue names endpoints somebody else runs. -Each engine reads its `PRESET` in its **own** flag vocabulary. That matters: -llama.cpp's short aliases would rewrite another engine's keys (`m` to `--model`, -`c` to `--ctx-size`), so a preset is never portable between engines. +Each engine reads its `PRESET` — and its `PARAM` lines — in its **own** flag +vocabulary. That matters: llama.cpp's short aliases would rewrite another +engine's keys (`m` to `--model`, `c` to `--ctx-size`), so a preset is never +portable between engines, and neither are PARAM lines. + +## PARAM: engine flags in the Spinloop + +A `PARAM` passes one engine flag the Spinloop's typed instructions don't cover, +straight in the Spinloop. It is repeatable — one `key = value` per line — and is +read in the same flag vocabulary a `PRESET` is read in, so for a llama.cpp +Spinloop `ngl` becomes `--n-gpu-layers` and `fa` becomes `--flash-attn`, while +vLLM's and MTPLX's long-form keys (`gpu-memory-utilization`, +`context-window`, …) pass through unchanged. The key is a single token; the +value is the rest of the line, so it may hold spaces and `=` signs, and a flag +that takes none may be written bare instead of given an empty value: + +```dockerfile +PROVIDER llamacpp +ALIAS qwen3.8-27b +CONTEXT 131072 +PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL +PARAM ngl = 99 +PARAM fa = on +PARAM jinja +``` + +Each PARAM renders the way a preset's flag does: a flag that takes no value +renders bare, whether written bare (`PARAM jinja`) or given a truthy value +(`PARAM jinja = 1`), and a falsy value (`0`, `off`, `false`, `no`) omits the +flag altogether; an empty value on any other flag renders it bare too +(`PARAM mmap =` becomes `--mmap`). Where the same flag is set in more than one +place, the Spinloop's own instructions win, then a PARAM, then the preset — +`PARAM ctx-size = 65536` loses to `CONTEXT 131072`. + +A PARAM may also name the model: for llama.cpp (`hf`/`model`/`hf-file`) and +MTPLX (`model`), a PARAM with such a key stands in for `MODEL`, read the way a +preset's model reference is read — so a Spinloop with no preset file at all +still serves (and deploys) its model. vLLM takes its model positionally, so a +PARAM cannot stand in for `MODEL` there, and `serve` says exactly that. + +The same PARAMs ride into [`spinloop cloud deploy`](cloud.md), which reads the +model and the runner's flags from them the way it reads a preset — dropping the +settings the cloud owns. See [the `Spinloop` file](../spinloop-file.md) for the +full rules. ## Parallelism @@ -114,11 +155,11 @@ at all, so there is nothing to compensate there either. If `PARALLEL` is left out entirely, it changes nothing — no `--parallel`-family flag is added, and `CONTEXT` maps to the engine's context flag exactly as it -always has. It only applies to a Spinloop-stated `CONTEXT`: a `PRESET`'s own -`ctx-size`, left unstated by the Spinloop, is not retroactively scaled — the -preset is trusted to already account for its own slots. Like `CONTEXT`, -`PARALLEL` overrides a preset's own `np`/`parallel`, `max-num-seqs`, or -`max-concurrent-requests` value by the usual override rule. +always has. It only applies to a Spinloop-stated `CONTEXT`: a `PRESET`'s (or a +PARAM's) own `ctx-size`, left unstated by the Spinloop, is not retroactively +scaled — the preset is trusted to already account for its own slots. Like +`CONTEXT`, `PARALLEL` overrides a preset's or a PARAM's own `np`/`parallel`, +`max-num-seqs`, or `max-concurrent-requests` value by the usual override rule. `PARALLEL` is Spinloop-file-only — like `PRESET`, it has no meaning for a hosted provider selection, so there is no `spinloop harness add --parallel`. @@ -127,7 +168,8 @@ provider selection, so there is no `spinloop harness add --parallel`. ### Simple case — straight from the Spinloop -With no `PRESET`, `serve` builds the command from the Spinloop itself: +With no `PRESET`, `serve` builds the command from the Spinloop itself — its +typed instructions, and any `PARAM` lines: ```dockerfile PROVIDER llamacpp @@ -144,7 +186,8 @@ path or ends in `.gguf`); `ALIAS`, `CONTEXT`, and `BASEURL` fill in the rest. ### Full control — a llama.cpp preset For flags a Spinloop doesn't model — `-ngl`, `--jinja`, KV-cache types, draft -models — point at a llama.cpp +models — write them as [PARAM](#param-engine-flags-in-the-spinloop) lines in the +Spinloop, or point at a llama.cpp [preset `.ini`](https://github.com/ggml-org/llama.cpp/blob/master/docs/preset.md): a set of `llama-server` flags grouped under named `[model]` sections, with a `[*]` section for shared defaults. Presets are built for the server's router @@ -197,8 +240,8 @@ BASEURL http://127.0.0.1:8000/v1 # omlx-cli serve --host/--port [Parallelism](#parallelism) above. Everything else — the model directory, the memory guard, the SSD cache — comes -from a `PRESET` written in oMLX's own flags, or from oMLX's settings -(`~/.omlx/settings.json`, and its admin panel): +from `PARAM` lines, a `PRESET` written in oMLX's own flags, or from oMLX's +settings (`~/.omlx/settings.json`, and its admin panel): ```ini [*] @@ -249,7 +292,7 @@ it, unlike llama.cpp's. `--tensor-parallel-size`/`--pipeline-parallel-size` (sharding a model across GPUs) are a different concept from `PARALLEL` and are not derived from it — -set them by hand in a `PRESET`. +set them by hand as `PARAM` lines or in a `PRESET`. ## MTPLX @@ -277,7 +320,8 @@ BASEURL http://127.0.0.1:8000/v1 # mtplx serve --host/--p MTPLX's own defaults stand. The scheduling mode (`--scheduler-mode`) is per-deployment tuning, not a -Spinloop field — set it in a `PRESET`, written in MTPLX's own long-form flags. +Spinloop field — set it as a `PARAM` line or in a `PRESET`, written in +MTPLX's own long-form flags. `serve` passes the value through without checking it, so it must be valid for the installed `mtplx` (`mtplx serve --help` lists the current modes). Every preset key is passed through unchanged, so a preset is portable only to MTPLX, @@ -436,8 +480,8 @@ output. - `serve` needs the engine installed: `llama-server` on your `PATH` (e.g. `brew install llama.cpp`), or [oMLX](https://omlx.ai). -- For llama.cpp, a Spinloop with no `PRESET` must name a `MODEL`. oMLX needs - neither. +- For llama.cpp, a Spinloop with no `PRESET` must name a `MODEL` — or have a + PARAM that names one (a `hf`/`model`/`hf-file` key). oMLX needs neither. ## See also diff --git a/docs/spinloop-file.md b/docs/spinloop-file.md index d7ade33e..f362311f 100644 --- a/docs/spinloop-file.md +++ b/docs/spinloop-file.md @@ -81,7 +81,7 @@ Spinloop says what the endpoint serves: PROVIDER llamacpp # the engine to run there, as it would run here ALIAS qwen3.6-27b CONTEXT 131072 -PRESET ./preset.ini +PRESET ./preset.ini # or PARAM lines in this file instead ``` The Spinloop says *what*; it no longer says *where*. Where is an @@ -198,12 +198,13 @@ One instruction per line: a keyword followed by a single value. | `PARALLEL` | no | `spinloop serve`, `spinloop cloud deploy` | `PARALLEL 2` | | `BASEURL` | no | `--base-url` | `BASEURL https://gateway/v1` | | `PRESET` | no | `spinloop serve` | `PRESET ./preset.ini` | +| `PARAM` | no (repeatable) | `spinloop serve`, `spinloop cloud deploy` | `PARAM ngl = 99` | | `ENV` | no (repeatable) | `spinloop cloud`, `spinloop harness` | `ENV AWS_PROFILE=prod` | Rules: - A Spinloop describes **exactly one provider**. `PROVIDER` is required and may - appear only once; so may every other keyword, except `ENV`. + appear only once; so may every other keyword, except `ENV` and `PARAM`. - You need **at least one** of `MODEL` or `ALIAS`. Give a `MODEL` to add a specific model; give an `ALIAS` to name it. - `MODEL` is the reference the **provider itself** understands: an @@ -244,6 +245,25 @@ Rules: launch command — never merely because the Spinloop was read. The file is read in the flag vocabulary of the engine `PROVIDER` names, so a preset written for llama.cpp is not portable to oMLX and vice versa. +- `PARAM` sets engine flags the Spinloop's typed instructions don't cover. It + is repeatable — one `key = value` per line — in the same flag vocabulary a + `PRESET` is read in, so `ngl` becomes `--n-gpu-layers` and `fa` becomes + `--flash-attn` for a llama.cpp Spinloop. The key is a single token; the value + is the rest of the line, so it may hold spaces and `=` signs, and a flag that + takes none may be written bare — `PARAM jinja` — as well as with an empty + value (`PARAM jinja =`). Where a PARAM and a preset set the same flag, the + PARAM stands over the preset; where the Spinloop states an instruction of its + own, the instruction stands over the PARAM — `PARAM ctx-size = 65536` loses + to `CONTEXT 131072`. Used only by + [`spinloop serve`](commands/serve.md) and + [`spinloop cloud deploy`](commands/cloud.md); `apply` ignores it, as it does + `PRESET`. +- A PARAM may also name the model. For an engine whose model is a flag — + llama.cpp's `hf`/`model`/`hf-file`, MTPLX's `model` — a PARAM with such a key + stands in for `MODEL`, and `serve` and `spinloop cloud deploy` read it the way + they read a preset's model reference, so a Spinloop with no preset file + still deploys what it serves. vLLM takes its model positionally, so a PARAM + cannot stand in for `MODEL` there. - `ENV` sets an environment variable on the machine running `spinloop` and is the one keyword that **may repeat**. Its value is a single `KEY=VALUE` token (no spaces). The `spinloop cloud` commands read it — along with a `.env` beside the @@ -282,6 +302,17 @@ PROVIDER llamacpp ALIAS qwen3.6-35b-a3b ``` +The same server with the engine's flags written in the Spinloop itself, in +place of a preset file — a PARAM may name the model: + +```dockerfile +PROVIDER llamacpp +ALIAS qwen3.8-27b +PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL +PARAM ngl = 99 +PARAM jinja +``` + A single model from OpenRouter (its key comes from your `.env` or environment, exactly as with `spinloop harness add`): diff --git a/examples/llamacpp/qwen3.8-27b-mtp/README.md b/examples/llamacpp/qwen3.8-27b-mtp/README.md new file mode 100644 index 00000000..261ac3c9 --- /dev/null +++ b/examples/llamacpp/qwen3.8-27b-mtp/README.md @@ -0,0 +1,153 @@ +# Qwen3.8-27B with MTP speculative decoding on llama.cpp + +Run Unsloth's GGUF build of Qwen3.8-27B with llama.cpp's multi-token +prediction (MTP) speculative decoding — the draft head sits inside the GGUF, so +no separate draft file is downloaded — then point opencode at it with the +[`Spinloop`](Spinloop) in this directory. The same file also deploys it to a +GPU in AWS with [`spinloop cloud`](#running-it-on-aws). + +What makes this example different from its +[`qwen3.8-27b`](../qwen3.8-27b/README.md) sibling: there is no +`preset.ini`. The engine options live in the Spinloop itself, one `PARAM` per +flag: + +```dockerfile +PROVIDER llamacpp +ALIAS qwen3.8-27b +CONTEXT 131072 +PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL +PARAM ngl = 99 +PARAM fa = on +PARAM jinja +PARAM spec-type = draft-mtp +PARAM spec-draft-n-max = 2 +``` + +Each `PARAM` names one engine flag — a `key = value` pair, or a bare key where +the flag takes no value — rendered into the `llama-server` command in +llama.cpp's own dialect — `fa` becomes `--flash-attn`, `ngl` becomes +`--n-gpu-layers`, and so on. `PARAM` +may repeat, and a PARAM stands over a preset when both set the same flag — +a Spinloop can mix the two. Where the Spinloop states an instruction of its +own (`MODEL`, `CONTEXT`, `PARALLEL`, …), the instruction wins over a PARAM. + +`PARAM hf` names the model in place of a `MODEL`: `serve` (and `cloud +deploy`) read it the way they read a preset's `hf`, so a Spinloop with no +preset file at all still deploys what it serves. + +## Prerequisites + +- A recent build of [llama.cpp](https://github.com/ggml-org/llama.cpp) that + includes `llama-server` **and MTP speculative decoding** + (`--spec-type draft-mtp`) (e.g. `brew install llama.cpp`, or build from + source) — new enough to know the `qwen3_5` architecture this model uses. +- A GPU is strongly recommended. The `UD-Q5_K_XL` quant is roughly 20 GB on + disk; for comfortable headroom plan for ~24 GB of VRAM (less if you + offload fewer layers to the GPU). + +## 1. Start llama-server + +```sh +spinloop serve # from this directory; reads ./Spinloop +spinloop serve --dry-run # print the llama-server command without running it +``` + +`spinloop serve` builds the command from the Spinloop's PARAMs, so it runs +equivalently to: + +```sh +llama-server \ + -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \ + --jinja -ngl 99 --flash-attn on \ + --ctx-size 131072 \ + --spec-type draft-mtp --spec-draft-n-max 2 \ + --host 127.0.0.1 --port 8080 +``` + +On first run `llama-server` downloads the `UD-Q5_K_XL` weights into the +llama.cpp cache (`~/.cache/llama.cpp`) and then starts serving. + +What the PARAMs do: + +- `hf` — model repository and quant tag, fetched from Hugging Face. +- `ngl = 99` — offload all layers to the GPU. Lower it (or drop it) for + CPU-only or limited VRAM. +- `fa = on` — flash attention, which also keeps the quantised KV cache an + option (see the sibling example's [KV-cache section](../qwen3.8-27b/README.md#optional-quantise-the-kv-cache) + if 128k does not fit). +- `jinja` — use the model's built-in chat template, written bare because the + flag takes no value. Required for Qwen3 tool calling to work correctly. +- `spec-type = draft-mtp` — speculative decoding with the model's own + multi-token-prediction head. Worth roughly 1.5–2× on decode; + `llama-server` logs the acceptance rate. +- `spec-draft-n-max = 2` — how many draft tokens the head proposes per step. +- `CONTEXT 131072` — the context window in tokens, rendered as `--ctx-size`. + The model natively supports 262144, so raise it on a box with the memory. + +MTP does not support parallel sequences, so there is deliberately no +`PARALLEL` here: one request at a time is the engine's contract, not an +oversight. + +### Check it's up + +```sh +curl http://127.0.0.1:8080/v1/models +``` + +## 2. Point opencode at it + +`llama-server` speaks the OpenAI-compatible API, which is exactly what the +`llamacpp` provider targets (default base URL `http://localhost:8080/v1`). +Apply the [`Spinloop`](Spinloop) in this directory: + +```sh +spinloop harness apply examples/llamacpp/qwen3.8-27b-mtp/Spinloop +# or, from this directory: +spinloop harness apply +``` + +`ALIAS` is the name opencode shows for the model. For a single-model server +it's just a label — `llama-server` serves whichever model it loaded +regardless of what's requested — so call it whatever you find readable. +`CONTEXT` matches opencode's context window to the `--ctx-size` the server +was launched with, so it doesn't overshoot what `llama-server` will accept. + +Running on a non-default host or port? Add a `BASEURL` line to the Spinloop +(the file ships one commented out): + +```dockerfile +BASEURL http://127.0.0.1:9090/v1 +``` + +Now start `opencode` and select `llamacpp/qwen3.8-27b`. + +## Running it on AWS + +The same Spinloop runs this model on a GPU in the cloud — provisioned by +[`spinloop cloud`](../../../docs/commands/cloud.md) — rather than the machine +in front of you, and terminates itself once you stop using it. This is real, +billed AWS infrastructure (an EC2 GPU instance, an Elastic IP, +image-builder pipelines), so each step below shows you a plan and asks for +confirmation before it creates anything. + +```sh +spinloop cloud bootstrap # once per AWS account: the control plane +spinloop cloud deploy --env qwen3.8-27b-mtp # from this directory +eval "$(spinloop cloud start --env qwen3.8-27b-mtp)" # boots (~10 min cold) +spinloop harness open --env qwen3.8-27b-mtp # work +spinloop cloud stop --env qwen3.8-27b-mtp # done — shut it down now +``` + +The deploy reads the PARAMs the same way `serve` does — `hf` names the +weights, the rest become the server's flags — and drops the settings the +cloud owns (the bind, `ctx-size`, …) from them, exactly as it would from a +preset. See [`spinloop cloud`](../../../docs/commands/cloud.md) for the +rest: `logs`, `metrics`, and how to name and switch between multiple +deployed environments. + +## See also + +- The same model without MTP, from a `preset.ini`: + [`examples/llamacpp/qwen3.8-27b`](../qwen3.8-27b/README.md) +- The bigger mixture-of-experts sibling on the same engine: + [`examples/llamacpp/qwen3.6-35b-a3b`](../qwen3.6-35b-a3b/README.md) diff --git a/examples/llamacpp/qwen3.8-27b-mtp/Spinloop b/examples/llamacpp/qwen3.8-27b-mtp/Spinloop new file mode 100644 index 00000000..90736a7d --- /dev/null +++ b/examples/llamacpp/qwen3.8-27b-mtp/Spinloop @@ -0,0 +1,14 @@ +# Qwen3.8-27B with MTP speculative decoding, served locally by llama-server, +# or on a cloud GPU via spinloop cloud — everything in this one file, no preset. +# Engine options live here as PARAMs, each rendered into the llama-server +# command the way the preset's flags would be. +PROVIDER llamacpp +ALIAS qwen3.8-27b +CONTEXT 131072 # keep in step with the PARAMs below; raise it on a bigger box +PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL # the model: repo:QUANT, as a PARAM +PARAM ngl = 99 # offload every layer to the GPU; lower for limited VRAM +PARAM fa = on # flash attention +PARAM jinja # required for Qwen3 tool calling +PARAM spec-type = draft-mtp # MTP: the draft is inside this GGUF, no separate file +PARAM spec-draft-n-max = 2 # draft tokens proposed per step +# BASEURL http://127.0.0.1:9090/v1 # uncomment for a non-default host/port diff --git a/examples/mtplx/qwen3.8-27b-param/README.md b/examples/mtplx/qwen3.8-27b-param/README.md new file mode 100644 index 00000000..e8d2e8ce --- /dev/null +++ b/examples/mtplx/qwen3.8-27b-param/README.md @@ -0,0 +1,132 @@ +# Qwen3.8-27B on MTPLX — PARAMs, no preset + +Run an MTPLX-optimised build of Qwen3.8-27B on your Mac with +[MTPLX](https://mtplx.com), then point opencode at it with the +[`Spinloop`](Spinloop) in this directory. + +What makes this example different from its +[`qwen3.8-27b`](../qwen3.8-27b/README.md) sibling: there is no +`preset.ini`. The engine options live in the Spinloop itself, one `PARAM` per +flag — and `PARAM model` names the model in place of a `MODEL`: + +```dockerfile +PROVIDER mtplx +ALIAS qwen3.8-27b +CONTEXT 32768 +PARALLEL 4 +PARAM model = Youssofal/Qwen3.8-27B-MTPLX-Optimized-Speed +PARAM scheduler-mode = ar_batch +``` + +MTPLX takes its model as a flag, so a PARAM with the key `model` stands in +for `MODEL`: `serve` reads it the way it reads a preset's `model`, so a +Spinloop with no preset file at all still serves its model. MTPLX's +long-form keys pass through unchanged — there is no llama.cpp-style alias to +learn. + +MTPLX is a single-model server: you launch it with one model and it serves +that model over an OpenAI-compatible API. That makes it the closest of the +local engines to +[llama.cpp](../../llamacpp/qwen3.8-27b/README.md), but tuned for Apple +Silicon unified memory. + +## Prerequisites + +- An **Apple Silicon** Mac (M1 or later). MTPLX is Apple Silicon only — there + is no Intel or Linux build, and **no machine image**, so + [`spinloop cloud`](../../../docs/commands/cloud.md) cannot deploy it. It + serves locally, or on a + [fleet node](../../../docs/commands/fleet.md) you run yourself. +- [MTPLX](https://mtplx.com), installed so that `mtplx` is on your `PATH`. +- Enough unified memory for the weights. + +## 1. Start MTPLX + +```sh +spinloop serve # from this directory; reads ./Spinloop +spinloop serve --dry-run # print the `mtplx serve` command without running it +``` + +`spinloop serve` builds the command from the Spinloop, so it runs +equivalently to: + +```sh +mtplx serve \ + --model Youssofal/Qwen3.8-27B-MTPLX-Optimized-Speed \ + --scheduler-mode ar_batch \ + --download \ + --model-id qwen3.8-27b \ + --context-window 32768 \ + --max-active-requests 4 +``` + +The command omits `--host`/`--port` because MTPLX's defaults (127.0.0.1:8000) +stand; the file ships a commented-out `BASEURL` line to change both what +opencode calls and what `serve` binds to. + +What the PARAMs do: + +- `model` — the Hugging Face repo to serve, or a local path, taken verbatim. + A PARAM may name the model for an engine whose model is a flag, and MTPLX's + is. +- `scheduler-mode = ar_batch` — how admitted requests execute. `spinloop` + passes the value through without checking it, so it must be valid for your + `mtplx` build (`mtplx serve --help` lists the current modes). This is + per-deployment tuning, not a Spinloop field. +- `CONTEXT 32768` — the context a single request gets, rendered as + `--context-window`. Never scaled by `PARALLEL`. +- `PARALLEL 4` — an admission cap on how many requests run at once, rendered + as `--max-active-requests`. + +`spinloop serve` always passes `--download` — fetch the repo if it is not +already local, instead of failing the launch — and `--model-id`, the name the +model is served under (the `ALIAS`). + +### Check it's up + +```sh +curl http://127.0.0.1:8000/v1/models +curl http://127.0.0.1:8000/health +``` + +`/health` answers once the OpenAI server is up, which may be before the weights +finish loading. + +## 2. Point opencode at it + +MTPLX speaks the OpenAI-compatible API, which is what the `mtplx` provider +targets (default base URL `http://localhost:8000/v1`, overridable with +`MTPLX_BASE_URL`). Apply the [`Spinloop`](Spinloop) in this directory: + +```sh +spinloop harness apply examples/mtplx/qwen3.8-27b-param/Spinloop +# or, from this directory: +spinloop harness apply +``` + +`ALIAS` is the name opencode shows for the model, and `CONTEXT` matches +opencode's context window to the `--context-window` the server was launched +with. Now start `opencode` and select `mtplx/qwen3.8-27b`. + +## A note on API keys + +By default `spinloop` writes no key for a localhost MTPLX: the `mtplx` provider +is marked `apiKeyOptional`, so a local endpoint with no `OPENAI_API_KEY` set +gets no `apiKey` field at all. If you put a key on the engine, set +`OPENAI_API_KEY` **before** you apply the Spinloop, and `spinloop` writes an +`{env:OPENAI_API_KEY}` reference (never the secret). + +`spinloop serve` never passes `--api-key`: it prints the command it runs, so a +key there would land on your screen and in the process table. A supervised +engine (`serve --api` or `spinloop daemon`) is gated with a key file the daemon +writes instead. + +## See also + +- The same model, its engine options in a `preset.ini`: + [`examples/mtplx/qwen3.8-27b`](../qwen3.8-27b/README.md) +- The same model on vLLM: + [`examples/vllm/qwen3.8-27b`](../vllm/qwen3.8-27b/README.md) +- [`spinloop serve`](../../../docs/commands/serve.md) — the full command + reference +- [The `Spinloop` file](../../../docs/spinloop-file.md) — full syntax diff --git a/examples/mtplx/qwen3.8-27b-param/Spinloop b/examples/mtplx/qwen3.8-27b-param/Spinloop new file mode 100644 index 00000000..cb3067d9 --- /dev/null +++ b/examples/mtplx/qwen3.8-27b-param/Spinloop @@ -0,0 +1,11 @@ +# Qwen3.8-27B served locally by MTPLX on Apple Silicon — the engine options +# as PARAMs in this file, no preset. +PROVIDER mtplx +ALIAS qwen3.8-27b +CONTEXT 32768 # --context-window; raise it on a bigger box +PARALLEL 4 # --max-active-requests +PARAM model = Youssofal/Qwen3.8-27B-MTPLX-Optimized-Speed # a PARAM may name the model +PARAM scheduler-mode = ar_batch # per-deployment; `mtplx serve --help` lists the modes +# BASEURL http://127.0.0.1:8000/v1 # uncomment for a non-default host/port +# `spinloop cloud` cannot deploy this: mtplx is Apple-Silicon-only and has no +# machine image to run it on a cloud GPU diff --git a/examples/omlx/qwen3.6-param/README.md b/examples/omlx/qwen3.6-param/README.md new file mode 100644 index 00000000..84aec3d6 --- /dev/null +++ b/examples/omlx/qwen3.6-param/README.md @@ -0,0 +1,178 @@ +# Qwen3.6-35B-A3B on oMLX — PARAMs, no preset + +Run an MLX build of Qwen3.6-35B-A3B on your Mac with [oMLX](https://omlx.ai), +then point opencode at it with the [`Spinloop`](Spinloop) in this directory. + +What makes this example different from its +[`qwen3.6`](../qwen3.6/README.md) sibling: there is no `preset.ini`. The +engine options live in the Spinloop itself, one `PARAM` per flag: + +```dockerfile +PROVIDER omlx +MODEL Qwen3.6-35B-A3B-4bit +CONTEXT 32768 +PARAM model-dir = /Users/you/models +PARAM memory-guard = safe +PARAM paged-ssd-cache-dir = /Users/you/.omlx/cache +PARAM max-concurrent-requests = 8 +``` + +Paths are passed to the server verbatim, so `~` is not expanded — spell them +out in full, the way the sibling's preset does. + +oMLX is built on [MLX](https://github.com/ml-explore/mlx), Apple's array +framework, so it runs on the GPU and Neural Engine rather than through a CUDA +path. Its headline feature is **paged SSD caching**: KV-cache blocks are +persisted to disk, so returning to a long prompt restores instead of recomputing +it. For a coding agent, which re-sends a large and mostly-unchanged context on +every turn, that is the difference between a usable and an unusable setup — and +it is one of the PARAMs in this Spinloop. + +## Prerequisites + +- An **Apple Silicon** Mac (M1 or later). oMLX is Apple Silicon only — there is + no Intel or Linux build, and no cloud equivalent, so + [`spinloop cloud`](../../../docs/commands/cloud.md) cannot deploy it. +- [oMLX](https://omlx.ai), installed from its DMG or from source. +- Enough unified memory for the weights. The 4-bit build is roughly 20 GB, so + plan for a 32 GB machine or better. + +## 1. Pull the model + +oMLX's admin panel (`http://localhost:8000/admin`) has a Hugging Face browser +that downloads a model in one click, which is the easiest route. + +To do it from the shell instead, put the model in the directory oMLX serves +from — the Spinloop's `model-dir` PARAM, here `/Users/you/models`: + +```sh +hf download mlx-community/Qwen3.6-35B-A3B-4bit \ + --local-dir /Users/you/models/Qwen3.6-35B-A3B-4bit +``` + +The **directory name** is what the model is called over the API — +`Qwen3.6-35B-A3B-4bit` here — unless you set an alias for it in the admin panel. +That is the name the `Spinloop` uses as its `MODEL`. + +## 2. Start oMLX + +```sh +spinloop serve # from this directory; reads ./Spinloop +spinloop serve --dry-run # print the omlx-cli command without running it +``` + +`spinloop serve` builds the command from the Spinloop, so it runs +equivalently to: + +```sh +/Applications/oMLX.app/Contents/MacOS/omlx-cli serve \ + --model-dir /Users/you/models \ + --memory-guard safe \ + --paged-ssd-cache-dir /Users/you/.omlx/cache \ + --max-concurrent-requests 8 +``` + +`serve` finds `omlx-cli` on your `PATH`, falling back to the app bundle — so +this works even if you've only ever launched oMLX from the menu bar. + +What the PARAMs do: + +- `model-dir` — the directory of models to serve. oMLX loads *all* of them and + picks per request, which is why there's no "the model" flag here — `MODEL` + names what the *agent* asks for, and no PARAM can stand in for it. +- `memory-guard = safe` — keep memory headroom for the rest of the machine. + `balanced` gives the model more; `--memory-guard-gb` sets a ceiling directly. +- `paged-ssd-cache-dir` — persist KV-cache blocks to SSD. This is the setting + that makes long agent contexts fast on repeat turns. +- `max-concurrent-requests = 8` — continuous batching width (the default). +- `CONTEXT 32768` — opencode's context window. Unlike llama.cpp there is no + `--ctx-size` to keep it in step with — oMLX sizes its cache dynamically. + +Note these are `omlx-cli` flags, **not** llama.cpp's. Each engine reads its +PARAMs in its own vocabulary, so these lines and the ones under +[`examples/llamacpp/`](../../llamacpp/) are not interchangeable. + +### Check it's up + +```sh +curl http://127.0.0.1:8000/v1/models +``` + +This is also how to confirm what your model is actually called — the response +lists the names requests must use. + +## 3. Point opencode at it + +oMLX speaks the OpenAI-compatible API, which is what the `omlx` provider +targets (default base URL `http://localhost:8000/v1`). Apply the +[`Spinloop`](Spinloop) in this directory: + +```sh +spinloop harness apply examples/omlx/qwen3.6-param/Spinloop +# or, from this directory: +spinloop harness apply +``` + +`MODEL` matters more here than it does for a single-model llama.cpp server: +oMLX serves everything in its model directory and dispatches on the requested +name, so it must match a model oMLX actually has — its directory name, or the +alias you set in the admin panel. This Spinloop names the model downloaded +above; **if you downloaded something else, change `MODEL` to match.** List +what yours exposes with: + +```sh +curl -H "Authorization: Bearer $OPENAI_API_KEY" http://localhost:8000/v1/models +``` + +A name oMLX does not have comes back as `model not found`. And unlike the +llama.cpp examples — where `spinloop serve` can fetch the weights from Hugging +Face — oMLX only serves what you have already downloaded. + +Running on a different host or port? Add a `BASEURL` line (the file ships one +commented out): + +```dockerfile +BASEURL http://127.0.0.1:9100/v1 +``` + +That sets both what opencode calls and what `serve` binds to. + +## A note on API keys + +By default `spinloop` writes no key for a localhost oMLX: the `omlx` provider +is marked `apiKeyOptional`, so a local endpoint with no `OPENAI_API_KEY` set +gets no `apiKey` field at all. + +**But oMLX can require a key even on localhost** — it is a per-install setting +in the admin panel, and some builds turn it on by default. If yours does, +requests without a key come back as `API key required`. To use it, set +`OPENAI_API_KEY` **before** you apply the Spinloop: + +```sh +OPENAI_API_KEY=your-omlx-key spinloop harness apply # writes an {env:OPENAI_API_KEY} reference +OPENAI_API_KEY=your-omlx-key opencode # or `spinloop harness open`, which forwards it +``` + +The *before* matters: with the key set at apply time, `spinloop` writes an +`{env:OPENAI_API_KEY}` reference into the config (never the secret). With it +unset, `spinloop` omits the `apiKey` field entirely — and exporting the key +later does nothing, because there is no reference in the config to resolve. If +you hit `API key required` after the fact, set the variable and re-run +`apply`. + +Reaching an oMLX on another machine works the same way: start it with +`--api-key`, export `OPENAI_API_KEY`, and point `OMLX_BASE_URL` at the Mac. + +`spinloop serve` never passes `--api-key`: it prints the command it runs, so a +key there would land on your screen and in the process table. Configure auth in +oMLX itself. + +## See also + +- The same model, its engine options in a `preset.ini`: + [`examples/omlx/qwen3.6`](../qwen3.6/README.md) +- The same model on llama.cpp: + [`examples/llamacpp/qwen3.6-35b-a3b`](../../llamacpp/qwen3.6-35b-a3b/README.md) +- [`spinloop serve`](../../../docs/commands/serve.md) — the full command + reference +- [The `Spinloop` file](../../../docs/spinloop-file.md) — full syntax diff --git a/examples/omlx/qwen3.6-param/Spinloop b/examples/omlx/qwen3.6-param/Spinloop new file mode 100644 index 00000000..8400330a --- /dev/null +++ b/examples/omlx/qwen3.6-param/Spinloop @@ -0,0 +1,14 @@ +# Qwen3.6-35B-A3B served locally by oMLX on Apple Silicon — the engine options +# as PARAMs in this file, no preset. +# oMLX loads a whole model directory and picks per request, so MODEL names what +# opencode asks for — it is not a launch flag, and no PARAM can stand in for it. +PROVIDER omlx +MODEL Qwen3.6-35B-A3B-4bit +CONTEXT 32768 # opencode's context window +# Paths are passed to the server verbatim, so `~` is not expanded — spell them +# out in full. +PARAM model-dir = /Users/you/models # the directory of models to serve +PARAM memory-guard = safe # leave headroom for the rest of the machine +PARAM paged-ssd-cache-dir = /Users/you/.omlx/cache # persist KV blocks across requests +PARAM max-concurrent-requests = 8 +# BASEURL http://127.0.0.1:8000/v1 # uncomment for a non-default host/port diff --git a/examples/vllm/qwen3.8-27b/README.md b/examples/vllm/qwen3.8-27b/README.md new file mode 100644 index 00000000..43cc2412 --- /dev/null +++ b/examples/vllm/qwen3.8-27b/README.md @@ -0,0 +1,113 @@ +# Qwen3.8-27B on vLLM + +Run Qwen3.8-27B with [vLLM](https://docs.vllm.ai) — the engine Qwen's own +Quickstart recommends for production workloads and high-throughput scenarios +— then point opencode at it with the [`Spinloop`](Spinloop) in this directory. +The same file also deploys it to a GPU in AWS with +[`spinloop cloud`](#running-it-on-aws). + +## The Spinloop + +```dockerfile +PROVIDER vllm +MODEL Qwen/Qwen3.8-27B-FP8 +ALIAS qwen3.8-27b +CONTEXT 32768 +PARAM gpu-memory-utilization = 0.9 +``` + +- `MODEL` is `vllm serve`'s positional argument, not a flag — and the one + thing a `PARAM` cannot supply. Unlike llama.cpp and MTPLX, vLLM has no + model flag, so `PARAM hf = …` is not a model to it: `MODEL` is required. + `MODEL` is passed verbatim, so it may be an HF repo id (fetched by vLLM) or + a local path. +- `ALIAS` becomes `--served-model-name` — the name opencode shows and asks + for. +- `CONTEXT` becomes `--max-model-len`. vLLM's context is never scaled by + `PARALLEL`, so the value is the per-request ceiling as written. +- `PARAM gpu-memory-utilization` is a plain pass-through of an engine flag + the Spinloop's typed instructions don't cover: the fraction of GPU memory + vLLM may use for model weights and the KV cache. Add as many `PARAM`s as + you like — `max-num-seqs`, `enable-chunked-prefill`, a + `chat-template-kwargs` JSON value, … — each rendered into the command the + way a preset's flags would be. + +## Prerequisites + +- [vLLM](https://docs.vllm.ai) on your `PATH` (`pip install vllm`, or the + engine build your driver needs) — new enough to know the `qwen3_5` + architecture this model uses. +- A GPU. The FP8 build is roughly 27 GB on disk; 32k of context fits + comfortably in 24 GB of VRAM, and more headroom buys a larger `CONTEXT`. + +## 1. Start vLLM + +```sh +spinloop serve # from this directory; reads ./Spinloop +spinloop serve --dry-run # print the vllm serve command without running it +``` + +`spinloop serve` builds the command from the Spinloop, so it runs +equivalently to: + +```sh +vllm serve Qwen/Qwen3.8-27B-FP8 \ + --served-model-name qwen3.8-27b \ + --max-model-len 32768 \ + --gpu-memory-utilization 0.9 \ + --port 8000 +``` + +On first run vLLM downloads the weights into its own cache and then starts +serving. The OpenAI-compatible API is at `http://127.0.0.1:8000/v1`. + +### Check it's up + +```sh +curl http://127.0.0.1:8000/v1/models +``` + +## 2. Point opencode at it + +The `vllm` provider targets that OpenAI-compatible endpoint (default base +URL `http://localhost:8000/v1`). Apply the [`Spinloop`](Spinloop) in this +directory: + +```sh +spinloop harness apply examples/vllm/qwen3.8-27b/Spinloop +# or, from this directory: +spinloop harness apply +``` + +Now start `opencode` and select `vllm/qwen3.8-27b`. + +## Running it on AWS + +The same Spinloop runs this model on a GPU in the cloud — provisioned by +[`spinloop cloud`](../../../docs/commands/cloud.md) — rather than the machine +in front of you, and terminates itself once you stop using it. This is real, +billed AWS infrastructure (an EC2 GPU instance, an Elastic IP, +image-builder pipelines), so each step below shows you a plan and asks for +confirmation before it creates anything. + +```sh +spinloop cloud bootstrap # once per AWS account: the control plane +spinloop cloud deploy --env qwen3.8-27b-vllm # from this directory +eval "$(spinloop cloud start --env qwen3.8-27b-vllm)" # boots (~10 min cold) +spinloop harness open --env qwen3.8-27b-vllm # work +spinloop cloud stop --env qwen3.8-27b-vllm # done — shut it down now +``` + +The deploy reads the same instructions — `MODEL` names the weights, `CONTEXT` +the window — and the PARAMs ride into the server's flags the way they do +locally, the settings the cloud owns dropped. See +[`spinloop cloud`](../../../docs/commands/cloud.md) for the rest: `logs`, +`metrics`, and how to name and switch between multiple deployed +environments. + +## See also + +- The same model on llama.cpp, with MTP speculative decoding: + [`examples/llamacpp/qwen3.8-27b-mtp`](../llamacpp/qwen3.8-27b-mtp/README.md) +- The same model on llama.cpp, from a `preset.ini`: + [`examples/llamacpp/qwen3.8-27b`](../llamacpp/qwen3.8-27b/README.md) diff --git a/examples/vllm/qwen3.8-27b/Spinloop b/examples/vllm/qwen3.8-27b/Spinloop new file mode 100644 index 00000000..5a3e9885 --- /dev/null +++ b/examples/vllm/qwen3.8-27b/Spinloop @@ -0,0 +1,9 @@ +# Qwen3.8-27B served locally by vLLM, or on a cloud GPU via spinloop cloud. +# vLLM takes its model positionally, so a MODEL is required here — unlike +# llama.cpp or MTPLX, a PARAM cannot stand in for it. +PROVIDER vllm +MODEL Qwen/Qwen3.8-27B-FP8 # adjust to the FP8 build you run +ALIAS qwen3.8-27b +CONTEXT 32768 # vllm serve --max-model-len +PARAM gpu-memory-utilization = 0.9 +# BASEURL http://0.0.0.0:8000/v1 # uncomment for a non-default host/port diff --git a/internal/preset/preset.go b/internal/preset/preset.go index ea8f0c19..56958750 100644 --- a/internal/preset/preset.go +++ b/internal/preset/preset.go @@ -196,11 +196,10 @@ var OMLX = Dialect{} // applied and keys pass through as written. var MTPLX = Dialect{} -// Flags merges ordered layers of params into flag tokens. Later layers override +// Merge folds ordered layers of params into one layer. Later layers override // earlier ones by canonical flag name, in place, so the first layer fixes the -// order: pass globals, then the section, then any overrides. It does not -// include the binary name; see Command. -func (d Dialect) Flags(layers ...[]Param) []string { +// order: pass globals, then the section, then any overrides. +func (d Dialect) Merge(layers ...[]Param) []Param { var ordered []Param at := map[string]int{} // canonical flag name -> index in ordered for _, layer := range layers { @@ -214,9 +213,16 @@ func (d Dialect) Flags(layers ...[]Param) []string { ordered = append(ordered, kv) } } + return ordered +} +// Flags merges ordered layers of params into flag tokens. Later layers override +// earlier ones by canonical flag name, in place, so the first layer fixes the +// order: pass globals, then the section, then any overrides. It does not +// include the binary name; see Command. +func (d Dialect) Flags(layers ...[]Param) []string { var args []string - for _, kv := range ordered { + for _, kv := range d.Merge(layers...) { args = append(args, d.flagFor(kv.Key, kv.Value)...) } return args diff --git a/internal/spinloop/spinloop.go b/internal/spinloop/spinloop.go index a0fcc2d5..5d59612b 100644 --- a/internal/spinloop/spinloop.go +++ b/internal/spinloop/spinloop.go @@ -14,6 +14,7 @@ // PARALLEL 2 # optional; concurrent request slots for `serve` // BASEURL https://gateway/v1 # optional; API base URL override // PRESET ./preset.ini # optional; llama.cpp preset for `serve` +// PARAM jinja = 1 # optional, repeatable; engine flag for `serve` // ENV AWS_PROFILE=dev # optional, repeatable; local env var // // MODEL is the reference the provider itself understands: an OpenRouter/Bedrock @@ -21,10 +22,16 @@ // (org/model:quant) or a path to a .gguf. ALIAS overrides the friendly name the // harness shows for it (and the name llama-server reports under `serve`). // -// ENV sets an environment variable for the local `spinloop` process — the one -// keyword that may appear more than once. It carries a single KEY=VALUE token -// and is used by the cloud commands (which read it before signing AWS calls); -// it is local-only and never reaches a deployed instance. +// ENV sets an environment variable for the local `spinloop` process — one of +// two keywords that may appear more than once. It carries a single KEY=VALUE +// token and is used by the cloud commands (which read it before signing AWS +// calls); it is local-only and never reaches a deployed instance. +// +// PARAM names one engine flag, in the engine's own vocabulary — `PARAM +// jinja = 1`, `PARAM hf = org/model:QUANT` — the inline alternative to a +// preset file for the same options. Like ENV it may repeat; what a PARAM does +// to the served engine's command is the concern of the command that consumes +// it (`serve`, `cloud deploy`), never of this parser. // // Keywords are matched case-insensitively, but UPPERCASE is canonical (it is // what `spinloop export` emits). Blank lines, full-line `#` comments, and @@ -65,6 +72,10 @@ type Selection struct { // Env holds the Spinloop's ENV instructions in file order. Unlike the other // fields it may carry several entries, since ENV may repeat. Env []EnvVar + // Params holds the Spinloop's PARAM instructions in file order. Like Env + // it may carry several entries, since PARAM may repeat: where the same key + // is set twice, the consumer's layer merge lets the later line stand. + Params []Param // DiscoveredModels is set at apply time, like DisplayName, never parsed // from a Spinloop file: the model IDs a gateway reported when the // selection named neither a model nor an alias to route by. A harness @@ -80,6 +91,14 @@ type EnvVar struct { Value string } +// Param is one PARAM instruction: a single engine flag, in the engine's own +// vocabulary (e.g. `jinja`, `ctx-size`, `hf`), with its value. The value may +// be empty — the PARAM written `key =` or bare `key` — naming a bare flag. +type Param struct { + Key string + Value string +} + // Spinloop keywords, in their canonical (lower-cased) form for matching. const ( kwProvider = "provider" @@ -90,6 +109,7 @@ const ( kwParallel = "parallel" kwBaseURL = "baseurl" kwPreset = "preset" + kwParam = "param" kwEnv = "env" ) @@ -98,7 +118,7 @@ const ( // "" for an unrecognised keyword. func canonicalKeyword(kw string) string { switch kw { - case kwProvider, kwModel, kwAlias, kwContext, kwOutput, kwParallel, kwPreset, kwEnv: + case kwProvider, kwModel, kwAlias, kwContext, kwOutput, kwParallel, kwPreset, kwParam, kwEnv: return kw case kwBaseURL, "base-url", "base_url", "url": return kwBaseURL @@ -132,8 +152,24 @@ func Parse(data []byte) (Selection, error) { "line %d: the REMOTE instruction was removed: name the environment with `spinloop cloud deploy --env ` at deploy time, and pass --env to the commands that act on it (cloud subcommands, apply, unapply, harness)", line) } - return Selection{}, fmt.Errorf("line %d: unknown keyword %q (expected PROVIDER, MODEL, ALIAS, CONTEXT, OUTPUT, PARALLEL, BASEURL, PRESET, or ENV)", line, fields[0]) + return Selection{}, fmt.Errorf("line %d: unknown keyword %q (expected PROVIDER, MODEL, ALIAS, CONTEXT, OUTPUT, PARALLEL, BASEURL, PRESET, PARAM, or ENV)", line, fields[0]) } + // PARAM, like ENV, may repeat, and its value — `key = value` — may + // itself hold spaces, so it is parsed from the rest of the line and + // sidesteps the single-value check. The key is one non-empty token; + // the ` = value` may be left off, as may the value after it — a bare + // key names a bare flag, the way `key =` does. + if canon == kwParam { + rest := strings.TrimSpace(text[len(fields[0]):]) + key, value, _ := strings.Cut(rest, "=") + key = strings.TrimSpace(key) + if key == "" || strings.ContainsAny(key, " \t") { + return Selection{}, fmt.Errorf("line %d: PARAM takes a non-empty key, optionally followed by ` = value`, got %q", line, text) + } + sel.Params = append(sel.Params, Param{Key: key, Value: strings.TrimSpace(value)}) + continue + } + switch { case len(fields) < 2: return Selection{}, fmt.Errorf("line %d: %s needs a value", line, strings.ToUpper(canon)) @@ -221,6 +257,13 @@ func Format(sel Selection) string { line("PARALLEL", sel.Parallel) line("BASEURL", sel.BaseURL) line("PRESET", sel.Preset) + for _, p := range sel.Params { + if p.Value == "" { // a bare flag, written without its ` = value` + line("PARAM", p.Key) + } else { + line("PARAM", p.Key+" = "+p.Value) + } + } for _, e := range sel.Env { line("ENV", e.Key+"="+e.Value) } diff --git a/internal/spinloop/spinloop_test.go b/internal/spinloop/spinloop_test.go index b2a12ae7..5629732c 100644 --- a/internal/spinloop/spinloop_test.go +++ b/internal/spinloop/spinloop_test.go @@ -72,6 +72,55 @@ func TestParse(t *testing.T) { in: "PROVIDER llamacpp\nMODEL m\nENV TOKEN=a=b=c\n", want: Selection{Provider: "llamacpp", Model: "m", Env: []EnvVar{{Key: "TOKEN", Value: "a=b=c"}}}, }, + { + name: "a param", + in: "PROVIDER llamacpp\nMODEL m\nPARAM jinja = 1\n", + want: Selection{Provider: "llamacpp", Model: "m", Params: []Param{{Key: "jinja", Value: "1"}}}, + }, + { + name: "repeated params with a trailing comment", + in: "PROVIDER llamacpp\nPARAM jinja = 1\nPARAM temp = 1.0\n" + + "PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL # repo and quant\n" + + "PARAM no-mmproj = 1\n", + want: Selection{ + Provider: "llamacpp", + Params: []Param{ + {Key: "jinja", Value: "1"}, + {Key: "temp", Value: "1.0"}, + {Key: "hf", Value: "unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL"}, + {Key: "no-mmproj", Value: "1"}, + }, + }, + }, + { + name: "repeated param keys keep file order", + in: "PROVIDER llamacpp\nMODEL m\nPARAM temp = 0.5\nPARAM temp = 1.0\n", + want: Selection{Provider: "llamacpp", Model: "m", Params: []Param{{Key: "temp", Value: "0.5"}, {Key: "temp", Value: "1.0"}}}, + }, + { + name: "param with an empty value", + in: "PROVIDER llamacpp\nMODEL m\nPARAM no-mmap =\n", + want: Selection{Provider: "llamacpp", Model: "m", Params: []Param{{Key: "no-mmap", Value: ""}}}, + }, + { + name: "bare param names a flag with no value", + in: "PROVIDER llamacpp\nMODEL m\nPARAM jinja\n", + want: Selection{Provider: "llamacpp", Model: "m", Params: []Param{{Key: "jinja", Value: ""}}}, + }, + { + name: "param value keeps spaces and later equals signs", + in: "PROVIDER llamacpp\nMODEL m\nPARAM stop = a b = c\n", + want: Selection{Provider: "llamacpp", Model: "m", Params: []Param{{Key: "stop", Value: "a b = c"}}}, + }, + { + name: "param alongside preset and env", + in: "PROVIDER llamacpp\nMODEL m\nPRESET ./preset.ini\nPARAM ngl = 99\nENV AWS_PROFILE=dev\n", + want: Selection{ + Provider: "llamacpp", Model: "m", Preset: "./preset.ini", + Params: []Param{{Key: "ngl", Value: "99"}}, + Env: []EnvVar{{Key: "AWS_PROFILE", Value: "dev"}}, + }, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -88,17 +137,21 @@ func TestParse(t *testing.T) { func TestParse_Errors(t *testing.T) { cases := map[string]string{ - "missing provider": "MODEL llama3.2\n", - "unknown keyword": "PROVIDER ollama\nFLAVOUR vanilla\n", - "family retired": "PROVIDER openrouter\nFAMILY deepseek-v4\n", - "keyword no value": "PROVIDER\n", - "too many values": "PROVIDER a b\n", - "duplicate keyword": "PROVIDER a\nPROVIDER b\n", - "duplicate parallel": "PROVIDER a\nMODEL m\nPARALLEL 1\nPARALLEL 2\n", - "duplicate alias": "PROVIDER a\nMODEL m\nBASEURL u1\nURL u2\n", - "env without equals": "PROVIDER a\nMODEL m\nENV JUST_A_NAME\n", - "env empty key": "PROVIDER a\nMODEL m\nENV =value\n", - "env value with spaces": "PROVIDER a\nMODEL m\nENV KEY=hello world\n", + "missing provider": "MODEL llama3.2\n", + "unknown keyword": "PROVIDER ollama\nFLAVOUR vanilla\n", + "family retired": "PROVIDER openrouter\nFAMILY deepseek-v4\n", + "keyword no value": "PROVIDER\n", + "too many values": "PROVIDER a b\n", + "duplicate keyword": "PROVIDER a\nPROVIDER b\n", + "duplicate parallel": "PROVIDER a\nMODEL m\nPARALLEL 1\nPARALLEL 2\n", + "duplicate alias": "PROVIDER a\nMODEL m\nBASEURL u1\nURL u2\n", + "env without equals": "PROVIDER a\nMODEL m\nENV JUST_A_NAME\n", + "env empty key": "PROVIDER a\nMODEL m\nENV =value\n", + "env value with spaces": "PROVIDER a\nMODEL m\nENV KEY=hello world\n", + "param empty key": "PROVIDER a\nMODEL m\nPARAM =1\n", + "param key with spaces": "PROVIDER a\nMODEL m\nPARAM hf repo = m\n", + "bare param key with spaces": "PROVIDER a\nMODEL m\nPARAM hf repo\n", + "bare param with no key": "PROVIDER a\nMODEL m\nPARAM \n", } for name, in := range cases { t.Run(name, func(t *testing.T) { @@ -119,12 +172,24 @@ func TestFormatRoundTrip(t *testing.T) { Parallel: "2", BaseURL: "https://gateway.example/v1", Preset: "./preset.ini", + Params: []Param{{Key: "jinja", Value: "1"}, {Key: "temp", Value: "1.0"}, {Key: "no-mmap", Value: ""}}, Env: []EnvVar{{Key: "AWS_PROFILE", Value: "dev"}, {Key: "AWS_REGION", Value: "eu-west-2"}}, } out := Format(sel) if !strings.HasPrefix(out, "PROVIDER openrouter\n") { t.Errorf("export not canonical:\n%s", out) } + if !strings.Contains(out, "PARAM jinja = 1") { + t.Errorf("Format should emit PARAM lines, got:\n%s", out) + } + if !strings.Contains(out, "PARAM no-mmap\n") { + t.Errorf("Format should render an empty-valued PARAM bare, got:\n%s", out) + } + // PARAM sits after PRESET and before ENV in the canonical order. + presetAt, paramAt, envAt := strings.Index(out, "PRESET"), strings.Index(out, "PARAM"), strings.Index(out, "ENV") + if !(presetAt >= 0 && paramAt > presetAt && envAt > paramAt) { + t.Errorf("Format should order PRESET before PARAM before ENV, got:\n%s", out) + } if !strings.Contains(out, "ENV AWS_PROFILE=dev") { t.Errorf("Format should emit ENV lines, got:\n%s", out) } @@ -207,3 +272,27 @@ func TestParse_FleetIsAnUnknownKeyword(t *testing.T) { } } } + +// The unknown-keyword error lists the accepted keywords, PARAM among them. +func TestParse_UnknownKeywordNamesParam(t *testing.T) { + _, err := Parse([]byte("PROVIDER x\nWIDGET w\n")) + if err == nil { + t.Fatal("an unknown keyword should error") + } + if !strings.Contains(err.Error(), "PRESET, PARAM, or ENV") { + t.Errorf("error should name PARAM among the accepted keywords, got %q", err) + } +} + +// PARAM's value may hold spaces, so it is carved from the rest of the line +// rather than fielded like every other instruction. +func TestParse_ParamValueFromRestOfLine(t *testing.T) { + sel, err := Parse([]byte("PROVIDER llamacpp\nPARAM chat-template = {\"a\": 1} = {\"b\": 2}\n")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + want := []Param{{Key: "chat-template", Value: `{"a": 1} = {"b": 2}`}} + if !reflect.DeepEqual(sel.Params, want) { + t.Errorf("Params = %+v, want %+v", sel.Params, want) + } +} diff --git a/openspec/changes/add-param-keyword/.openspec.yaml b/openspec/changes/add-param-keyword/.openspec.yaml new file mode 100644 index 00000000..b9ebfc72 --- /dev/null +++ b/openspec/changes/add-param-keyword/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-12 diff --git a/openspec/changes/add-param-keyword/design.md b/openspec/changes/add-param-keyword/design.md new file mode 100644 index 00000000..4ef30b13 --- /dev/null +++ b/openspec/changes/add-param-keyword/design.md @@ -0,0 +1,146 @@ +# Design: PARAM keyword + +## Context + +The Spinloop grammar (`internal/spinloop`, a pure-grammar leaf — stdlib only, no +I/O, never imports `internal/preset`) parses `KEYWORD value` lines where every +instruction except `ENV` must appear once and take exactly one value. Engine +options beyond the typed instructions (`MODEL`, `ALIAS`, `CONTEXT`, `PARALLEL`, +`BASEURL`) today reach the engine only through an external preset `.ini` +(`PRESET`), read in the engine's dialect: `preset.Dialect.Flags(layers...)` +merges ordered layers with later-wins by canonical flag name. + +Two consumers build commands/configs from a Spinloop, and both already treat +the preset as a layer below the Spinloop's own instructions: + +- `cmd/spinloop/serve.go` — `buildServeArgv`: preset path via + `Preset.CommandIn(dialect, binary, sub, sec, overrides...)` (variadic + override layers, already in place); preset-less path via + `assembleEngineArgv` rendering `engine.dialect.Flags(params)` for one layer + (the engine's `params(sel)` output). +- `cmd/spinloop/cloud.go` — `deployConfig(sel, path, target)`: reads the + preset's model/context/parallel keys as fallbacks behind the Spinloop's own + values (`presetValue(key, global, section)`), then layers + `[dropOwned(owns, global), dropOwned(owns, section)]` (plus the bind for + node wakes) into `ServeArgs`, and reads companion weights from the same + layers. `deployConfigForNode` reuses it with a different `owns` set. + +The issue fixes one precedence rule: where a param is set both in the preset +and as a `PARAM`, the Spinloop file wins. + +## Goals / Non-Goals + +Goals (behaviour in the spec deltas): `PARAM` parses like `ENV` (repeatable), +renders in the engine's dialect, sits between the preset and the Spinloop's +typed instructions, may supply the model for engines that take it as a flag, +and flows to cloud deploy and fleet node wakes with the same owned-flag +dropping a preset gets. + +Non-goals: the `hf` command (it emits MODEL-based Spinloops, no PARAM); +`harness export` (a harness config cannot hold engine flags, so there is +nothing to emit); shell completion (no Spinloop-keyword completion exists); +the daemon's push path (it reads a `DeployConfig`, where PARAM is already +baked into `ServeArgs` by `deployConfig`). + +## Decisions + +### 1. PARAM is stored as its own type in `internal/spinloop` + +`Selection` gains `Params []Param` with a small `Param{Key, Value string}` +type defined in the grammar leaf. Alternative — reusing `preset.Param` — +rejected: it would make the pure-grammar leaf import `internal/preset`, +breaking the stated invariant that the leaf never imports outside itself. +`cmd/spinloop` converts `Selection.Params` to `[]preset.Param` at the two +consumers (serve, cloud), which already import both. + +### 2. Precedence: typed instructions > PARAM > preset + +Layer order is `[preset global, preset section, PARAM, Spinloop instructions]` +for serve, and `[dropOwned(global), dropOwned(section), dropOwned(PARAM)]` +(plus bind) for deploy. The issue mandates PARAM-over-preset; +PARAM-under-instructions is the design call, because the typed instructions +are the canonical interface and own side effects — the llamacpp +`ctx-size * parallel` scaling runs off `sel.Context`/`sel.Parallel`, not off +flags, so a `PARAM ctx-size` cannot sneak a second context into the command. +Alternative — PARAM beats everything — rejected: it would let a raw +pass-through silently defeat `CONTEXT`/`PARALLEL`, breaking the scaling +contract the `local-serving` spec pins down. + +### 3. Reuse the dialect's merge rather than invent one + +`Dialect.Flags`'s layer merge (later-wins by canonical key, first-appearance +order) is extracted to `Dialect.Merge(layers ...[]Param) []Param`; `Flags` +becomes render-of-`Merge`. This gives: + +- preset path: `CommandIn` already takes variadic override layers, so PARAM is + passed as one more: `CommandIn(d, bin, sub, sec, selParams, params)` — no + signature change; +- preset-less path: `assembleEngineArgv` receives + `engine.dialect.Merge(selParams, params)` as its single params argument — + no signature change; +- collision semantics for a PARAM against a preset (`ngl` vs `n-gpu-layers`) + are identical to preset-vs-preset, for free. + +Duplicate PARAM keys are kept in file order in the selection; the dialect +merge resolves them last-wins, the same way a preset section's repeated keys +behave. + +### 4. Model sourcing is per-engine, defined once + +`cmd/spinloop` gains `modelSourceFlags(runner) []string` — the canonical flag +names that name the engine's model: llamacpp `hf-repo`, `model`, `hf-file`; +mtplx `model`; vllm none (the model is positional, so `PARAM` cannot stand in +for `MODEL`). One definition serves both sides: serve's `needsModel` check +(`MODEL` or a PARAM naming the model, for engines with model flags) and +deploy's model fallback (`sel.Model` → PARAM's model keys → preset's model +key) — a PARAM standing over the preset, as decision 2 mandates and as a +local serve layers the PARAM over the preset's own model flag, so the same +file deploys what it serves. The PARAM is read in the engine's dialect +through `paramModel(engine, params)`, the core the serve path calls with the +same selection's PARAMs. A PARAM whose model value is a local path deploys +exactly like a +`MODEL` local path does today: the cloud refuses it naming a repo. + +### 5. Cloud parity is layer addition, not a new path + +`deployConfig` converts `sel.Params` once to `[]preset.Param` and then: +extends each of the model, context and parallel fallbacks with +`presetValue(key, paramLayer)` (the existing variadic helper scans the new +layer last), appends `dropOwned(owns, paramLayer)` as the final ServeArgs +layer before the bind, and passes the PARAM layer to `companionsFrom`. +Because `owns` is the seam, `PARAM host = 0.0.0.0` drops for a cloud deploy +(`isCloudOwned`) and survives for a node wake (`isNodeOwned`) — the same +asymmetry a preset's bind has today. The two "nothing to deploy"/"no context +size" errors gain a naming of PARAM as an alternative. + +## Risks / Trade-offs + +- [PARAM sets `ctx-size`/`parallel` for llamacpp] → the layering keeps the + Spinloop's typed values on top, and the scaling contract runs off + `sel.Context`/`sel.Parallel`; a PARAM's own `ctx-size` is treated as a + preset's is — passed through unscaled. +- [The parser's "exactly one value" rule] → `PARAM` is special-cased before + the field-count check, as `ENV` is after it; its value is the rest of the + line split at the first `=`, so values may contain spaces and `=` signs + (e.g. a JSON chat template), while the key must stay one token. +- [vllm/omlx/mtplx dialects have no boolean table] → a boolean flag for those + engines takes an empty value to render bare (`PARAM enable-chunked-prefill =`); + a non-empty value renders as `--flag value`. This is exactly the constraint + a preset for those engines has today, so PARAM is parity, not a regression; + the vllm example uses value-taking flags only. +- [A PARAM key the engine does not know] → passed through verbatim in the + engine's dialect, the same way an unknown preset key is: the engine's own + flag validation is the backstop, unchanged. + +## Migration Plan + +Additive: existing Spinloops parse and behave identically (no PARAM → the new +layers are empty and `Merge` of one layer equals today's `Flags` of it). +Rollback is reverting the change; nothing on disk or in a deployed config +gains a new shape — a deployed `DeployConfig` with PARAM-derived `ServeArgs` +is already just engine flags. + +## Open Questions + +None — the precedence, model sourcing, and cloud parity decisions are settled +here and in the spec deltas. diff --git a/openspec/changes/add-param-keyword/proposal.md b/openspec/changes/add-param-keyword/proposal.md new file mode 100644 index 00000000..0b6d10b3 --- /dev/null +++ b/openspec/changes/add-param-keyword/proposal.md @@ -0,0 +1,64 @@ +# Proposal + +## Why + +Arbitrary engine options today only reach the engine through an external +`preset.ini` named by `PRESET`. That forces a second file (and its own +resolution and fetch rules) to mix and match a handful of options that the +author would keep in the Spinloop anyway. A repeatable `PARAM` instruction +carries engine options inline — the issue's Qwen3.8-27B Spinloop, from +sampling parameters to flash attention to speculative decoding, in one file. + +## What Changes + +- New repeatable `PARAM` instruction in the Spinloop file: `PARAM key = value`, + in the engine's own flag vocabulary, read the way a preset key is read (the + engine's dialect does the aliasing and boolean handling). +- `spinloop serve` takes the Spinloop's `PARAM`s as an override layer: they + beat a preset's values and lose to the Spinloop's own instructions + (`MODEL`, `ALIAS`, `CONTEXT`, `PARALLEL`, `BASEURL`). A PARAM that names the + engine's model source (llama.cpp's `hf`/`model`, MTPLX's `model`) satisfies + serve's model requirement in place of `MODEL` — so a Spinloop with no + `MODEL` but a `PARAM hf = …` serves, exactly as a preset-driven one does. + vLLM takes its model positionally, so it still needs `MODEL`. +- `spinloop cloud deploy` (and the fleet-node wake, which derives its config + the same way) reads `PARAM`s exactly as it reads a preset: the model, + context-size and parallelism fallbacks consult the PARAM layer, and the + remaining PARAMs become the engine's serveArgs minus the settings the + destination owns. +- `apply`/`unapply` ignore `PARAM` (as they already ignore `PRESET`); + `export` does not emit it. +- Two new examples using `PARAM` instead of a preset file: + `examples/llamacpp/qwen3.8-27b-mtp/` (the issue's Q5_K_XL + MTP + speculative-decoding configuration) and `examples/vllm/qwen3.8-27b/` (the + first vLLM example). +- Docs: the Spinloop file page gains `PARAM` in its keyword table and rules, + and `serve`'s help and docs name it. + +## Capabilities + +### New Capabilities + +(None — the behaviour lands in three existing capabilities.) + +### Modified Capabilities + +- `spinloop-files`: a new repeatable `PARAM` instruction in the file grammar + (syntax, repetition, parse errors); `apply`/`unapply` ignore it; the unknown + keyword list and `export`'s canonical output change accordingly. +- `local-serving`: `PARAM` is rendered into the engine command in the engine's + dialect as a layer between the preset and the Spinloop's own instructions; + it may supply the model for engines that take one as a flag. +- `remote-endpoint`: the deploy derivation reads the PARAM layer for the + model, context-size and parallelism fallbacks and folds the remaining PARAMs + into the serveArgs, with the same destination-owned dropping as a preset. + +## Impact + +- `internal/spinloop` — parser, `Selection`, `Format`; tests. +- `cmd/spinloop/serve.go` — engine param layers, model-requirement check; tests. +- `cmd/spinloop/cloud.go` — `deployConfig` fallbacks and serveArgs; tests. +- `docs/spinloop-file.md`, `docs/commands/serve.md`, serve help text. +- `examples/llamacpp/qwen3.8-27b-mtp/`, `examples/vllm/qwen3.8-27b/`. +- No API, dependency, or harness-config change; existing Spinloops parse and + behave exactly as before. diff --git a/openspec/changes/add-param-keyword/specs/local-serving/spec.md b/openspec/changes/add-param-keyword/specs/local-serving/spec.md new file mode 100644 index 00000000..ef0c6305 --- /dev/null +++ b/openspec/changes/add-param-keyword/specs/local-serving/spec.md @@ -0,0 +1,227 @@ +## MODIFIED Requirements + +### Requirement: Preset-less serving + +Without a `PRESET`, the Spinloop SHALL supply the command directly, and the +model SHALL be named by a `MODEL` or, for an engine that takes its model as a +flag, by a `PARAM` naming it (see PARAM instructions): a `MODEL` value that +looks like a local file (ending `.gguf`, or starting with `/`, `./`, `../`, or +`~`) becomes the model path, anything else is treated as a Hugging Face repo +reference. `ALIAS` sets the served model's reported name, `CONTEXT` (parsed +with the standard lenient size format) sets the context size, and `BASEURL`'s +host and port set the server's bind address. + +#### Scenario: Hugging Face repo + +- **WHEN** the Spinloop has `MODEL unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL`, + `ALIAS qwen3.6`, and `CONTEXT 32768` +- **THEN** the command carries the repo reference, the alias, and a context + size of 32768 + +#### Scenario: Local file + +- **WHEN** the Spinloop has `MODEL ./models/qwen.gguf` +- **THEN** the command loads that file as a model path rather than a repo + +#### Scenario: No model to serve + +- **WHEN** the Spinloop has neither `PRESET` nor `MODEL`, and no `PARAM` + naming the model for an engine that takes its model as a flag +- **THEN** the command fails explaining serve needs a preset, a model, or a + PARAM naming the model + +### Requirement: Flag rendering + +Preset keys and PARAM keys SHALL render as the engine's flags. Where an engine +defines short aliases, they SHALL be canonicalised to their long form so the +same flag written different ways collapses to one when layers merge, later +layers overriding earlier ones in place; an engine that defines none SHALL pass +its keys through unchanged. Flags the engine accepts bare SHALL render bare +when truthy and be dropped when falsy (`0`, `false`, `off`, `no`). Printed +commands SHALL quote only the tokens that need it, and SHALL pass values to the +server verbatim. + +#### Scenario: Layer override by canonical name + +- **WHEN** the global section sets `c = 4096` and the model section sets + `ctx-size = 8192` for llama.cpp +- **THEN** the command carries a single context-size flag with value 8192 + +#### Scenario: Boolean flags + +- **WHEN** a llama.cpp preset sets `jinja = 1` and `mmap = 0` +- **THEN** the command includes a bare `--jinja` and no mmap flag + +#### Scenario: An engine with no short aliases + +- **WHEN** an oMLX preset sets `model-dir` and `memory-guard` +- **THEN** both render as their long-form flags with their values unchanged + +### Requirement: Parallelism + +`CONTEXT` SHALL always mean the context window a single request gets, whatever +engine serves it. A Spinloop's `PARALLEL` SHALL set the number of concurrent +request slots, translated per engine so that `CONTEXT`'s meaning holds: + +- For `llamacpp`, `PARALLEL n` SHALL be rendered as `--parallel n`. Because + llama.cpp treats `--ctx-size` as a total budget divided across its parallel + slots, when the Spinloop **also** states `CONTEXT`, the rendered `--ctx-size` + SHALL be `context_tokens * n` rather than `context_tokens`, so each slot + still gets the context the Spinloop asked for. `CONTEXT` set with no + `PARALLEL` SHALL render `--ctx-size` as `context_tokens`, unscaled, exactly + as before this capability existed. +- For `vllm`, `PARALLEL n` SHALL be rendered as `--max-num-seqs n`. + `--max-model-len` (from `CONTEXT`) SHALL NOT be scaled by `PARALLEL`: vLLM's + concurrency is bounded independently of a single request's context length. +- For `omlx`, `PARALLEL n` SHALL be rendered as `--max-concurrent-requests n`. + oMLX has no context flag to scale either way. +- For `mtplx`, `PARALLEL n` SHALL be rendered as `--max-active-requests n`, + MTPLX's cap on admitted concurrent requests. `--context-window` (from + `CONTEXT`) SHALL NOT be scaled by `PARALLEL`. MTPLX's scheduling mode — how + admitted requests execute, serially or in parallel — SHALL remain a preset + concern: a Spinloop's `PARALLEL` SHALL NOT select it. + +A Spinloop stating no `PARALLEL` SHALL produce a command identical to one from +before this capability existed, for all four engines. A `PARALLEL` value +SHALL be validated as a positive integer at the point it is used (`serve`, a +daemon-pushed config, or `cloud deploy`); a value that is not SHALL fail +naming the invalid value rather than being passed to the engine. + +When `CONTEXT` is supplied by a `PRESET` section's own `ctx-size` or a `PARAM`'s +own `ctx-size` rather than by the Spinloop, `PARALLEL` SHALL NOT rescale it — +only a Spinloop-stated `CONTEXT` participates in the `llamacpp` multiply. A +`PARALLEL` value SHALL still override a preset's own or a `PARAM`'s own +`np`/`parallel` (llama.cpp), `max-num-seqs` (vLLM), +`max-concurrent-requests` (oMLX), or `max-active-requests` (MTPLX) value by the +same override-by-canonical-name rule `CONTEXT` already uses against a preset's +`ctx-size`. + +#### Scenario: llama.cpp context is scaled by parallel slots + +- **WHEN** a Spinloop states `PROVIDER llamacpp`, `CONTEXT 128k`, and + `PARALLEL 2` +- **THEN** the printed command includes `--ctx-size 256000` and `--parallel 2` + +#### Scenario: llama.cpp parallel with no context set + +- **WHEN** a Spinloop states `PROVIDER llamacpp` and `PARALLEL 4` with no + `CONTEXT` +- **THEN** the printed command includes `--parallel 4` and no `--ctx-size` + flag is added + +#### Scenario: llama.cpp context with no parallel set is unscaled + +- **WHEN** a Spinloop states `PROVIDER llamacpp` and `CONTEXT 128k` with no + `PARALLEL` +- **THEN** the printed command includes `--ctx-size 128000`, exactly as it + would have before `PARALLEL` existed + +#### Scenario: A preset's own context is not rescaled + +- **WHEN** a Spinloop states `PROVIDER llamacpp`, `PARALLEL 2`, and a `PRESET` + whose section sets `ctx-size` but the Spinloop itself states no `CONTEXT` +- **THEN** the printed command includes `--parallel 2` and the preset's + `ctx-size` value passes through unscaled + +#### Scenario: vLLM concurrency does not touch context + +- **WHEN** a Spinloop states `PROVIDER vllm`, `CONTEXT 128k`, and `PARALLEL 4` +- **THEN** the printed command includes `--max-model-len 128000` and + `--max-num-seqs 4`, with neither value derived from the other + +#### Scenario: oMLX concurrency + +- **WHEN** a Spinloop states `PROVIDER omlx` and `PARALLEL 8` +- **THEN** the printed command includes `--max-concurrent-requests 8` and no + context flag, exactly as with no `PARALLEL` stated + +#### Scenario: MTPLX concurrency does not touch context + +- **WHEN** a Spinloop states `PROVIDER mtplx`, `CONTEXT 128k`, and + `PARALLEL 4` +- **THEN** the printed command includes `--max-active-requests 4` and + `--context-window 128000`, with neither value derived from the other + +#### Scenario: No PARALLEL means no change + +- **WHEN** a Spinloop states no `PARALLEL`, for any of the four engines +- **THEN** the printed command is identical to what it would have been before + this capability existed + +#### Scenario: Invalid parallel count + +- **WHEN** a Spinloop states `PARALLEL 0`, a negative number, or a non-numeric + value, for any of the four engines +- **THEN** the command fails naming the invalid value rather than passing it + to the engine + +#### Scenario: Spinloop PARALLEL overrides a preset's own value + +- **WHEN** a Spinloop states `PROVIDER llamacpp` and `PARALLEL 2`, and its + `PRESET` section separately sets `np = 4` +- **THEN** the printed command includes `--parallel 2`, not `--parallel 4` + +## ADDED Requirements + +### Requirement: PARAM instructions + +A Spinloop's `PARAM` instructions SHALL be one layer of the engine's flags, +rendered in the engine's own dialect exactly as preset keys are rendered. The +layers merge so that a PARAM stands over a preset's values and under the +Spinloop's own instructions (`MODEL`, `ALIAS`, `CONTEXT`, `PARALLEL`, +`BASEURL`): a typed instruction beats a raw pass-through, which beats the +preset. Where the same key is set by several PARAM lines, the later line's +value SHALL stand. + +#### Scenario: A PARAM overrides the preset + +- **WHEN** a preset section sets `temp = 0.7` and the Spinloop says + `PARAM temp = 1.0` +- **THEN** the command carries a single temperature flag with value 1.0 + +#### Scenario: An instruction overrides a PARAM + +- **WHEN** the Spinloop says `CONTEXT 32768` and `PARAM ctx-size = 5000` +- **THEN** the command carries a context size of 32768, unscaled + +#### Scenario: PARAM booleans render as the engine's dialect allows + +- **WHEN** the Spinloop says `PARAM jinja` and `PARAM no-mmproj = 1` for + llamacpp +- **THEN** the command includes a bare `--jinja` and a bare `--no-mmproj`, the + bare key and the `key = 1` form rendering alike + +#### Scenario: A PARAM's context is not rescaled + +- **WHEN** a Spinloop states `PROVIDER llamacpp`, `PARALLEL 2`, and + `PARAM ctx-size = 4096`, and states no `CONTEXT` +- **THEN** the printed command includes `--parallel 2` and the PARAM's + `ctx-size` value passes through unscaled + +#### Scenario: A PARAM's parallelism loses to the Spinloop's + +- **WHEN** a Spinloop states `PROVIDER llamacpp`, `PARALLEL 2`, and + `PARAM np = 4` +- **THEN** the printed command includes `--parallel 2`, not `--parallel 4` + +### Requirement: A PARAM can name the model + +For an engine that takes its model as a flag — `llamacpp` and `mtplx` — a PARAM +naming the model SHALL satisfy serve's model requirement in place of a `MODEL`, +the way a preset's model reference does, and the model SHALL be passed to the +engine as that flag, unaltered. `vllm` takes its model positionally and SHALL +still require a `MODEL`; a PARAM cannot stand in for it. + +#### Scenario: A PARAM names the model + +- **WHEN** the Spinloop says `PROVIDER llamacpp`, `ALIAS qwen3.8-27b`, and + `PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL`, with no `MODEL` and no + `PRESET` +- **THEN** the command carries the repository reference as the model flag, and + serve does not fail for want of a model + +#### Scenario: vLLM still needs a MODEL + +- **WHEN** the Spinloop says `PROVIDER vllm` with PARAMs but no `MODEL` +- **THEN** the command fails explaining serve needs a MODEL, which vLLM takes + positionally diff --git a/openspec/changes/add-param-keyword/specs/remote-endpoint/spec.md b/openspec/changes/add-param-keyword/specs/remote-endpoint/spec.md new file mode 100644 index 00000000..2fcd1317 --- /dev/null +++ b/openspec/changes/add-param-keyword/specs/remote-endpoint/spec.md @@ -0,0 +1,89 @@ +## MODIFIED Requirements + +### Requirement: Deploying what the endpoint serves + +`spinloop cloud deploy` SHALL derive the deployment from the Spinloop, its +preset, and its PARAMs: `PROVIDER` SHALL select the inference engine, `MODEL` or +the preset's or a PARAM's Hugging Face reference SHALL name the weights as a +repository and optional quantisation, `CONTEXT` or the preset's or a PARAM's +context size SHALL set the window, `ALIAS` SHALL set the name the endpoint +serves under (defaulting to the repository), and the preset's and the PARAMs' +remaining settings SHALL become the engine's arguments, a PARAM standing over +the preset's values where both set the same flag. Settings the endpoint owns — +host, port, model location, API key, context size, alias, and metrics — SHALL be +dropped from the preset and the PARAMs alike, so one preset can both serve +locally and deploy unchanged. The request SHALL describe only what to serve, +never where the weights are stored. A `--dry-run` SHALL print the derived +deployment without sending it. + +`deploy` SHALL additionally accept a `--api-key-env ` flag naming the +environment variable that holds the API key to store for the environment. The +flag names a variable, never a literal, and the CLI resolves it from the +process environment before sending it to the control plane to be stored as the +environment's key (see the Environment Deployment specification for how it is +stored and rotated). When a key is supplied, the deploy report SHALL say so +without printing the value. + +Deploy SHALL target a named environment: in addition to deriving what to serve, +it SHALL create and register that environment on the control plane (its Elastic +IP, instance configuration, per-environment API key and ingress, and SSM state), +as defined by the Environment Deployment specification. Deploying SHALL NOT start +the instance. + +#### Scenario: A preset drives both serving and deploying + +- **WHEN** a Spinloop with a preset is deployed +- **THEN** the engine's arguments are the preset's, minus the settings the + endpoint sets itself + +#### Scenario: PARAMs deploy the way a preset does + +- **WHEN** a Spinloop with no preset states `PARAM hf = org/model:quant`, + `PARAM ctx-size = 32768`, and other PARAMs, and is deployed +- **THEN** the deployment names the repository and quantisation from the PARAM, + sets the window from the PARAM's context size, and carries the remaining + PARAMs as the engine's arguments minus the settings the endpoint sets itself + +#### Scenario: A PARAM overrides the preset in a deployment + +- **WHEN** a Spinloop's preset section sets `temp = 0.7` and the Spinloop says + `PARAM temp = 1.0` +- **THEN** the deployed engine's arguments carry the PARAM's value + +#### Scenario: The Spinloop overrides its preset + +- **WHEN** the Spinloop states a `MODEL` and `CONTEXT` that differ from the + preset's +- **THEN** the Spinloop's values are deployed + +#### Scenario: A provider that is not a self-hosted engine + +- **WHEN** a Spinloop naming a hosted provider is deployed +- **THEN** the command fails saying that only a self-hosted engine can be + deployed + +#### Scenario: A local model file + +- **WHEN** a Spinloop naming a local model file is deployed +- **THEN** the command fails saying to name a repository instead, because the + endpoint fetches its own weights + +#### Scenario: Deploying creates and registers the environment + +- **WHEN** a deployment succeeds against a bootstrapped account +- **THEN** the named environment is created and registered in the registry, and + the report says whether the weights still have to be fetched before it can + serve + +#### Scenario: Deploying is not starting + +- **WHEN** a deployment succeeds +- **THEN** the environment is configured but not started, and the report says + whether the weights still have to be fetched before it can serve + +#### Scenario: Deploying with a supplied key + +- **WHEN** `spinloop cloud deploy` is given `--api-key-env SHARED_KEY` and that + variable is set +- **THEN** the key is sent to the control plane to be stored for the + environment, and the report says a key was applied without printing the value diff --git a/openspec/changes/add-param-keyword/specs/spinloop-files/spec.md b/openspec/changes/add-param-keyword/specs/spinloop-files/spec.md new file mode 100644 index 00000000..bce29148 --- /dev/null +++ b/openspec/changes/add-param-keyword/specs/spinloop-files/spec.md @@ -0,0 +1,143 @@ +## MODIFIED Requirements + +### Requirement: Spinloop file format + +A Spinloop SHALL be a flat, line-oriented text file of `KEYWORD value` +instructions. The keywords are `PROVIDER`, `MODEL`, `ALIAS`, `CONTEXT`, +`OUTPUT`, `PARALLEL`, `BASEURL` (also accepted as `BASE-URL`, `BASE_URL`, or +`URL`), `PRESET`, `PARAM`, and `ENV`. Keywords SHALL match +case-insensitively, with UPPERCASE as the canonical form. Blank lines, full-line +`#` comments, and trailing comments introduced by whitespace-then-`#` SHALL be +ignored. Each instruction SHALL take exactly one value; every instruction SHALL +appear at most once, except `ENV` and `PARAM`, which MAY be repeated. An `ENV` +instruction's value SHALL be a single `KEY=VALUE` token with a non-empty key and +no whitespace. A `PARAM` instruction SHALL be a key — a single token with no +whitespace — optionally followed by ` = value`, split at the first `=`: the +value MAY be empty or contain whitespace, and a key with no value names a bare +flag. A PARAM names one engine flag in the +engine's own vocabulary — what it does to the served engine's command is +defined by the `local-serving` capability — and where the same key is set by +several PARAM lines the later line's value SHALL stand. `PARALLEL`'s value SHALL +name a count of concurrent request slots; like `CONTEXT`, its numeric validity +(a positive integer) is enforced by the commands that consume it rather than by +parsing itself, and what it does to the served engine's command is defined by +the `local-serving` capability. `PROVIDER` is required. Parse errors SHALL name +the offending line. + +#### Scenario: A minimal Spinloop + +- **WHEN** a file containing only `PROVIDER openrouter` and + `MODEL deepseek/deepseek-v4-pro` is parsed +- **THEN** it yields a selection of that provider and model + +#### Scenario: Duplicate instruction + +- **WHEN** a Spinloop sets `MODEL` on two lines +- **THEN** parsing fails, citing both line numbers + +#### Scenario: Unknown keyword + +- **WHEN** a Spinloop contains `HARNESS pi` +- **THEN** parsing fails listing the accepted keywords + +#### Scenario: Naming a fleet + +- **WHEN** a Spinloop contains `FLEET ./fleet.yaml` +- **THEN** parsing fails as an unknown keyword, listing the accepted keywords, + and no special migration message is printed + +#### Scenario: Missing provider + +- **WHEN** a Spinloop has no `PROVIDER` instruction +- **THEN** parsing fails saying the PROVIDER instruction is missing + +#### Scenario: Naming a cloud endpoint + +- **WHEN** a Spinloop contains `REMOTE ./remote.json` on any line +- **THEN** parsing fails on that line, since the `REMOTE` instruction was + removed: the environment is now named with `cloud deploy --env ` at + deploy time and `--env ` at the commands that act on it + +#### Scenario: Declaring local environment variables + +- **WHEN** a Spinloop contains `ENV AWS_PROFILE=dev` and `ENV AWS_REGION=eu-west-2` + on separate lines +- **THEN** it parses, yielding both key/value pairs in the selection, and the + repetition is not treated as a duplicate-instruction error + +#### Scenario: Malformed ENV value + +- **WHEN** a Spinloop contains an `ENV` instruction whose value has no `=` or an + empty key +- **THEN** parsing fails, naming the offending line + +#### Scenario: Setting the number of parallel slots + +- **WHEN** a Spinloop contains `PARALLEL 2` +- **THEN** it parses, yielding a parallel count of 2 in the selection + +#### Scenario: A non-numeric or non-positive PARALLEL is caught on use + +- **WHEN** a Spinloop contains `PARALLEL 0`, `PARALLEL -1`, or `PARALLEL abc` +- **THEN** parsing accepts the raw value, exactly as it does for `CONTEXT`, and + the command that goes on to use it (`serve`, `cloud deploy`, a fleet wake) + fails naming the value, rather than silently treating it as a slot count + +#### Scenario: Declaring engine parameters + +- **WHEN** a Spinloop contains `PARAM jinja = 1` and + `PARAM hf = unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL # the weights` on separate + lines +- **THEN** it parses, yielding both entries in the selection, and the repetition + is not treated as a duplicate-instruction error + +#### Scenario: A PARAM may repeat a key + +- **WHEN** a Spinloop sets `PARAM ngl = 99` on one line and `PARAM ngl = 97` on + a later line +- **THEN** it parses, and the later line's value is the one that stands + +#### Scenario: A bare PARAM names a flag without a value + +- **WHEN** a Spinloop contains `PARAM jinja` on a line by itself +- **THEN** it parses as the bare form of `PARAM jinja =`, yielding an entry + whose empty value names a bare flag + +#### Scenario: Malformed PARAM value + +- **WHEN** a Spinloop contains a `PARAM` instruction with an empty key, or a + key containing whitespace, with or without a value +- **THEN** parsing fails, naming the offending line + +### Requirement: Applying and unapplying a Spinloop + +`spinloop harness apply` SHALL apply the Spinloop's selection exactly as the equivalent +`spinloop harness add` would, and `spinloop harness unapply` SHALL remove what the Spinloop selects +exactly as the equivalent `spinloop harness remove` would. A command-line `--output`/`-o` +on `apply` SHALL override the Spinloop's `OUTPUT` instruction, and `--providers` +SHALL override the catalogue it resolves against (a Spinloop never names a +catalogue). `apply` SHALL ignore a `PRESET` instruction — it is consumed only +by `spinloop serve` — and SHALL ignore `PARAM` instructions the same way: they +name engine flags that only a serve or a deploy consumes. + +#### Scenario: Apply equals add + +- **WHEN** a Spinloop with `PROVIDER ollama` and `MODEL llama3.2` is applied +- **THEN** the harness config matches what `spinloop harness add -p ollama -m llama3.2` + would have produced + +#### Scenario: Output override + +- **WHEN** a Spinloop sets `OUTPUT 32k` and the user runs + `spinloop harness apply --output 16k` +- **THEN** the applied output limit is 16000 tokens + +#### Scenario: Preset is not apply's business + +- **WHEN** a Spinloop with a `PRESET` instruction is applied +- **THEN** the harness config is written as if the instruction were absent + +#### Scenario: Params are not apply's business + +- **WHEN** a Spinloop with `PARAM` instructions is applied +- **THEN** the harness config is written as if the instructions were absent diff --git a/openspec/changes/add-param-keyword/tasks.md b/openspec/changes/add-param-keyword/tasks.md new file mode 100644 index 00000000..07b8cb4a --- /dev/null +++ b/openspec/changes/add-param-keyword/tasks.md @@ -0,0 +1,81 @@ +# Tasks: PARAM keyword + +## 1. Grammar — `internal/spinloop` + +- [x] 1.1 Add the `Param{Key, Value}` type and a `Params []Param` field to + `Selection` (file order, like `Env`) +- [x] 1.2 Parse `PARAM`: repeatable, value is the rest of the line split at + the first `=` (key a single non-empty token, value may be empty or hold + spaces); special-case it before the single-value field-count check, as + `ENV` is; parse errors name the line +- [x] 1.3 Add `PARAM` to the unknown-keyword error's accepted list +- [x] 1.4 Render `PARAM` lines in `Format` (after `PRESET`, before `ENV`) +- [x] 1.5 Tests: parse with trailing comment, repetition, repeated key, + malformed values (no `=`, empty key, whitespace in key), unknown-keyword + list, `Format` round-trip +- [x] 1.6 Accept a bare `PARAM key` — the ` = value` left off — as the bare + form of `PARAM key =` (key is the rest of the line, value empty); the + empty-key and whitespace-in-key errors apply to both forms. `Format` renders + an empty-valued PARAM bare + +## 2. Engine command — `internal/preset`, `cmd/spinloop/serve.go` + +- [x] 2.1 Extract `Dialect.Merge(layers ...[]Param) []Param` from + `Dialect.Flags` (identical later-wins merge); `Flags` renders the merge +- [x] 2.2 Add `modelSourceFlags(runner) []string` (llamacpp `hf-repo`/ + `model`/`hf-file`, mtplx `model`, vllm none) and a `modelFlags` field on + `serveEngine` initialised from it; add `paramModel(engine, params)` — the + last PARAM naming the model in the engine's dialect +- [x] 2.3 `buildServeArgv`: convert `sel.Params` to `[]preset.Param` once; + preset path passes it as an override layer to `CommandIn` ahead of the + instruction params; preset-less path renders + `dialect.Merge(selParams, params)` +- [x] 2.4 Widen the `needsModel` check: `MODEL` or a PARAM naming the model + (engines with model flags only); the error names the three ways +- [x] 2.5 Tests: PARAM renders per dialect (aliases, booleans, passthrough); + PARAM beats preset; instructions beat PARAM; PARAM model for llamacpp and + mtplx; vllm still needs `MODEL`; PARAM `ctx-size` unscaled by `PARALLEL`; + `PARALLEL` beats PARAM's own value; preset-less PARAM-only command; + `--dry-run` output +- [x] 2.6 Mention `PARAM` in `serve`'s help where `PRESET` is named + +## 3. Cloud deploy — `cmd/spinloop/cloud.go` + +- [x] 3.1 `deployConfig`: convert `sel.Params` to `[]preset.Param`; extend the + model, context and parallel fallbacks with the PARAM layer (after the + preset's); the model fallback reads the runner's model-source keys +- [x] 3.2 Append `dropOwned(owns, paramLayer)` as a ServeArgs layer after the + section layer; pass the PARAM layer to `companionsFrom` +- [x] 3.3 Update the "nothing to deploy"/"no context size" errors to name a + PARAM as an alternative +- [x] 3.4 Tests: deploy with PARAM only (model/context/parallel from PARAM, + remaining PARAMs in ServeArgs minus owned flags); PARAM beats preset in the + derived config; owned PARAM flags dropped for cloud, kept for node wake; + node wake derives the same way; local-path PARAM model still refused by the + cloud + +## 4. Examples + +- [x] 4.1 `examples/llamacpp/qwen3.8-27b-mtp/`: `Spinloop` (the issue's + Q5_K_XL + `draft-mtp` configuration, PARAM-based, no preset file) and a + `README.md` following the other llamacpp examples' shape +- [x] 4.2 `examples/vllm/qwen3.8-27b/`: `Spinloop` (first vLLM example: + `PROVIDER vllm`, `MODEL`, `ALIAS`, `CONTEXT`, PARAMs with value-taking + flags) and a `README.md` +- [x] 4.3 `examples/llamacpp/qwen3.8-27b-mtp/`: `PARAM jinja` written in the + bare form, shown in the README + +## 5. Docs + +- [x] 5.1 `docs/spinloop-file.md`: `PARAM` in the keyword table, its syntax + and repetition rules, the precedence rule (instructions > PARAM > preset), + and the model-sourcing note +- [x] 5.2 Serve docs/help: `PARAM` alongside `PRESET` wherever the preset is + described as the way to pass engine options +- [x] 5.3 The bare PARAM form (`PARAM jinja`) in `spinloop-file.md` and the + serve docs' PARAM section + +## 6. Verify + +- [x] 6.1 `gofmt -l`, `go vet ./...`, `go test ./...` green +- [x] 6.2 Coverage `go test ./... -cover` at or above 80% overall