diff --git a/src/utils/gbnfJson/terminals/GbnfStringValue.ts b/src/utils/gbnfJson/terminals/GbnfStringValue.ts index 0768722f..a341b2d6 100644 --- a/src/utils/gbnfJson/terminals/GbnfStringValue.ts +++ b/src/utils/gbnfJson/terminals/GbnfStringValue.ts @@ -1,6 +1,14 @@ import {GbnfTerminal} from "../GbnfTerminal.js"; import {GbnfGrammarGenerator} from "../GbnfGrammarGenerator.js"; +// the control characters (U+0000 - U+001F) that a JSON string cannot contain unescaped, +// after `\t`, `\r`, `\n`, `\b` and `\f` were already replaced with their short escape sequence +const remainingControlCharactersRegex = /[\u0000-\u001f]/g; + +function escapeControlCharacter(char: string): string { + const hex = char.charCodeAt(0).toString(16); + return "\\\\u" + hex.padStart(4, "0"); +} export class GbnfStringValue extends GbnfTerminal { public readonly value: string; @@ -19,6 +27,9 @@ export class GbnfStringValue extends GbnfTerminal { .replaceAll("\t", "\\\\t") .replaceAll("\r", "\\\\r") .replaceAll("\n", "\\\\n") + .replaceAll("\b", "\\\\b") + .replaceAll("\f", "\\\\f") + .replaceAll(remainingControlCharactersRegex, escapeControlCharacter) .replaceAll('"', "\\\\" + '\\"'), '\\"', '"' diff --git a/test/standalone/utils/GbnfStringValue.test.ts b/test/standalone/utils/GbnfStringValue.test.ts new file mode 100644 index 00000000..d312684a --- /dev/null +++ b/test/standalone/utils/GbnfStringValue.test.ts @@ -0,0 +1,76 @@ +import {describe, expect, test} from "vitest"; +import {GbnfStringValue} from "../../../src/utils/gbnfJson/terminals/GbnfStringValue.js"; +import {getGbnfGrammarForGbnfJsonSchema} from "../../../src/utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.js"; + + +// the grammar literal for a string value is the JSON representation of the value, +// with `\` and `"` escaped again, since it is placed inside a GBNF string literal +function getExpectedGrammarLiteral(value: string) { + return '"' + JSON.stringify(value).replaceAll("\\", "\\\\") + .replaceAll('"', '\\"') + '"'; +} + +// reverses the escaping of a GBNF string literal, to get the text that the grammar makes the model generate +function getGeneratedTextOfGrammarLiteral(literal: string) { + return literal.slice(1, -1).replaceAll('\\"', '"') + .replaceAll("\\\\", "\\"); +} + +const rawControlCharacterRegex = /[\u0000-\u001f]/; + +describe("utils", () => { + describe("GbnfStringValue", () => { + test("escapes backspace and form feed", () => { + expect(new GbnfStringValue("a\bb").getGrammar()).toBe(getExpectedGrammarLiteral("a\bb")); + expect(new GbnfStringValue("a\fb").getGrammar()).toBe(getExpectedGrammarLiteral("a\fb")); + expect(new GbnfStringValue("a\bb").getGrammar()).toBe('"\\"a\\\\bb\\""'); + expect(new GbnfStringValue("a\fb").getGrammar()).toBe('"\\"a\\\\fb\\""'); + }); + + test("escapes the other control characters as \\u00XX", () => { + expect(new GbnfStringValue("a\u0000b").getGrammar()).toBe('"\\"a\\\\u0000b\\""'); + expect(new GbnfStringValue("a\u0001b").getGrammar()).toBe('"\\"a\\\\u0001b\\""'); + expect(new GbnfStringValue("a\u001fb").getGrammar()).toBe('"\\"a\\\\u001fb\\""'); + }); + + test("keeps the existing escaping of tab, carriage return, line feed, quote and backslash", () => { + const value = "t\tr\rn\nq\"b\\"; + + expect(new GbnfStringValue(value).getGrammar()).toBe(getExpectedGrammarLiteral(value)); + }); + + test("does not escape characters that are valid in a JSON string", () => { + expect(new GbnfStringValue("a b\u007fé
").getGrammar()).toBe('"\\"a b\u007fé
\\""'); + }); + + test("every control character is escaped the same way JSON.stringify does and has no raw form in the grammar", () => { + for (let code = 0; code < 0x20; code++) { + const value = "a" + String.fromCharCode(code) + "b"; + const grammarLiteral = new GbnfStringValue(value).getGrammar(); + + expect(grammarLiteral).not.toMatch(rawControlCharacterRegex); + expect(grammarLiteral).toBe(getExpectedGrammarLiteral(value)); + expect(JSON.parse(getGeneratedTextOfGrammarLiteral(grammarLiteral))).toBe(value); + } + }); + + test("enum, const and property names with control characters in a JSON schema grammar", () => { + const enumGrammar = getGbnfGrammarForGbnfJsonSchema({enum: ["a\bb"]}); + expect(enumGrammar).toContain('val0 ::= "\\"a\\\\bb\\""'); + expect(enumGrammar).not.toMatch(/\u0008/); + + const constGrammar = getGbnfGrammarForGbnfJsonSchema({const: "x\u001f"}); + expect(constGrammar).toContain('"\\"x\\\\u001f\\""'); + expect(constGrammar.includes(String.fromCharCode(0x1f))).toBe(false); + + const objectGrammar = getGbnfGrammarForGbnfJsonSchema({ + type: "object", + properties: { + "k\f": {type: "string"} + } + }); + expect(objectGrammar).toContain('"\\"k\\\\f\\""'); + expect(objectGrammar).not.toMatch(/\u000c/); + }); + }); +});