diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c34e2729..d0a340ecd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(server): preserve delimiters in string metadata overrides by @allenflux in #2380 + ## [0.3.36] - feat: update llama.cpp to ggml-org/llama.cpp@0c1e57098 diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 8aa929202..ca03143fe 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -236,9 +236,9 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: assert isinstance(settings.kv_overrides, list) kv_overrides = {} for kv in settings.kv_overrides: - key, value = kv.split("=") + key, value = kv.split("=", 1) if ":" in value: - value_type, value = value.split(":") + value_type, value = value.split(":", 1) if value_type == "bool": kv_overrides[key] = value.lower() in ["true", "1"] elif value_type == "int": diff --git a/tests/test_server_model.py b/tests/test_server_model.py new file mode 100644 index 000000000..daa0f51c1 --- /dev/null +++ b/tests/test_server_model.py @@ -0,0 +1,58 @@ +from unittest.mock import Mock + +import pytest + +import llama_cpp +from llama_cpp.server.model import LlamaProxy +from llama_cpp.server.settings import ModelSettings + + +@pytest.mark.parametrize( + "value", + [ + "user: ", + "{{ messages[0]['role'] == 'user' }}", + "user: name=value", + "", + ":", + "=", + "用户: name=value", + ], +) +def test_server_kv_overrides_preserve_string_values(monkeypatch, value): + create_model = Mock() + monkeypatch.setattr(llama_cpp, "Llama", create_model) + settings = ModelSettings( + model="unused.gguf", + kv_overrides=[f"tokenizer.chat_template=str:{value}"], + ) + + result = LlamaProxy.load_llama_from_model_settings(settings) + + assert result is create_model.return_value + assert create_model.call_args.kwargs["kv_overrides"] == { + "tokenizer.chat_template": value + } + + +def test_server_kv_overrides_preserve_numeric_and_bool_values(monkeypatch): + create_model = Mock() + monkeypatch.setattr(llama_cpp, "Llama", create_model) + settings = ModelSettings( + model="unused.gguf", + kv_overrides=[ + "tokenizer.ggml.add_bos_token=bool:true", + "tokenizer.ggml.add_eos_token=bool:false", + "llama.context_length=int:2048", + "llama.rope.freq_base=float:10000.5", + ], + ) + + LlamaProxy.load_llama_from_model_settings(settings) + + assert create_model.call_args.kwargs["kv_overrides"] == { + "tokenizer.ggml.add_bos_token": True, + "tokenizer.ggml.add_eos_token": False, + "llama.context_length": 2048, + "llama.rope.freq_base": 10000.5, + }