From 9dbba737263bd2162156682f0df2e45bd519fa71 Mon Sep 17 00:00:00 2001 From: Mayank Dhingra Date: Fri, 30 Oct 2026 11:48:05 -0400 Subject: [PATCH] fix: preserve native Unicode in JSON Pointers Keep literal Unicode characters out of the byte-oriented escape codec. Decode existing ASCII escape spellings and UTF-16 surrogate units in separate spans so native object keys retain their code points. Share the decoder with from_parts and cover pointer builders, URI handling and JSON Patch operations that otherwise edit a different key. Signed-off-by: Mayank Dhingra --- docs/pointers.md | 21 +++ jsonpath/pointer.py | 43 +++--- tests/test_json_pointer_unicode.py | 210 +++++++++++++++++++++++++++++ 3 files changed, 259 insertions(+), 15 deletions(-) create mode 100644 tests/test_json_pointer_unicode.py diff --git a/docs/pointers.md b/docs/pointers.md index e29f39f..51a4cb7 100644 --- a/docs/pointers.md +++ b/docs/pointers.md @@ -37,6 +37,27 @@ JSON Pointers are a fundamental component of JSON Patch ([RFC 6902](https://data print(repr(rjp.to("/items/0/name"))) # JSONPointer('/items/#0') ``` +## Unicode and escape decoding + +Native Unicode characters in a pointer retain their code points. For example, +`JSONPointer("/café")` points to the key `"café"`, not to a re-encoded spelling. +Unicode normalization is not applied, so composed and decomposed keys remain +distinct. + +The existing `unicode_escape=True` compatibility option also decodes escape +spellings before resolving the pointer. Disable it when a pointer already +contains the literal characters you want, including backslash sequences that +were decoded from JSON text: + +```python +pointer = JSONPointer(r"/literal\n", unicode_escape=False) +assert pointer.resolve({r"literal\n": 7}) == 7 +``` + +Percent decoding remains separate. For example, +`JSONPointer("/caf%C3%A9", uri_decode=True)` resolves the key `"café"`. + + ## `resolve(data)` Resolve this pointer against _data_. _data_ can be a file-like object or string containing JSON formatted data, or a Python [`Mapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.Mapping) or [`Sequence`](https://docs.python.org/3/library/collections.abc.html#collections.abc.Sequence), like a dictionary or list. diff --git a/jsonpath/pointer.py b/jsonpath/pointer.py index 261d613..412db9d 100644 --- a/jsonpath/pointer.py +++ b/jsonpath/pointer.py @@ -2,7 +2,6 @@ from __future__ import annotations -import codecs import re from functools import reduce from operator import getitem @@ -38,6 +37,13 @@ def __str__(self) -> str: UNDEFINED = _Undefined() +# Match only ASCII escape spellings, leaving native Unicode untouched. +_ESCAPES = re.compile( + r"\\(?:[0-7]{1,3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}" + r"|U[0-9A-Fa-f]{8}|N\{[\x00-\x7f]*?\}|[\x00-\x7f]|$)" +) +_SURROGATES = re.compile(r"[\ud800-\udfff]+") + class JSONPointer: """Identify a single, specific value in JSON-like data, as per RFC 6901. @@ -264,14 +270,26 @@ def _encode(parts: Iterable[Union[int, str]]) -> str: ) return "" - def _unicode_escape(self, s: str) -> str: - # UTF-16 escape sequences - possibly surrogate pairs - inside UTF-8 - # encoded strings. As per https://datatracker.ietf.org/doc/html/rfc4627 - # section 2.5. - return ( - codecs.decode(s.replace("\\/", "/"), "unicode-escape") - .encode("utf-16", "surrogatepass") - .decode("utf-16") + @staticmethod + def _unicode_escape(s: str) -> str: + # The codec accepts bytes. Encoding a complete native string as UTF-8 + # makes those bytes look like separate Latin-1 characters to it. + s = s.replace("\\/", "/") + if s.isascii(): + decoded = s.encode("ascii").decode("unicode-escape") + else: + if match := _SURROGATES.search(s): + raise UnicodeEncodeError( + "utf-8", s, match.start(), match.end(), "surrogates not allowed" + ) + decoded = _ESCAPES.sub( + lambda match: match[0].encode("ascii").decode("unicode-escape"), s + ) + # Keep the existing pairing and validation of UTF-16 surrogate units + # without encoding the surrounding native Unicode characters. + return _SURROGATES.sub( + lambda match: match[0].encode("utf-16", "surrogatepass").decode("utf-16"), + decoded, ) @classmethod @@ -318,12 +336,7 @@ def from_parts( if uri_decode: _parts = (unquote(p) for p in _parts) if unicode_escape: - _parts = ( - codecs.decode(p.replace("\\/", "/"), "unicode-escape") - .encode("utf-16", "surrogatepass") - .decode("utf-16") - for p in _parts - ) + _parts = (cls._unicode_escape(p) for p in _parts) __parts = tuple(_parts) diff --git a/tests/test_json_pointer_unicode.py b/tests/test_json_pointer_unicode.py new file mode 100644 index 0000000..149da1a --- /dev/null +++ b/tests/test_json_pointer_unicode.py @@ -0,0 +1,210 @@ +"""Native Unicode pointers and JSON Patch targets retain their characters.""" + +from __future__ import annotations + +import json +from typing import Dict +from typing import List +from urllib.parse import quote + +import pytest + +from jsonpath import JSONPatch +from jsonpath import JSONPointer +from jsonpath import finditer +from jsonpath import patch +from jsonpath import pointer + +_VALUE = 7 +_OTHER_VALUE = 9 + +KEYS = [ + "café", + "漢字", + "😀", + "e\N{COMBINING ACUTE ACCENT}", + "café/part~name", + "café%2F", + 'café"quoted', + "café\n\t\0", + "café\\😀", + "café\\\\😀", +] + + +def _token(key: str) -> str: + return key.replace("~", "~0").replace("/", "~1") + + +@pytest.mark.parametrize("key", KEYS) +def test_native_unicode_pointer(key: str) -> None: + # A raw pointer has already been decoded by its transport. Backslashes + # are literal when compatibility escape decoding is disabled. + data = {key: _VALUE} + value = JSONPointer("/" + _token(key), unicode_escape=False) + assert value.resolve(data) == _VALUE + assert str(value) == "/" + _token(key) + + +@pytest.mark.parametrize("key", KEYS) +def test_unicode_with_compatibility_escapes(key: str) -> None: + # JSON's encoder provides the escape spelling; this is not another + # implementation of the pointer's decoder. Native Unicode stays literal. + encoded = json.dumps(_token(key), ensure_ascii=False)[1:-1] + value = JSONPointer("/" + encoded, unicode_escape=True) + assert value.resolve({key: _VALUE}) == _VALUE + assert str(value) == "/" + _token(key) + + +@pytest.mark.parametrize("key", ["café", "漢字", "😀", "e\N{COMBINING ACUTE ACCENT}"]) +def test_default_pointer_and_convenience_api(key: str) -> None: + data = {key: _VALUE} + assert JSONPointer("/" + key).resolve(data) == _VALUE + assert pointer.resolve("/" + key, data) == _VALUE + assert JSONPointer.from_parts([key]).resolve(data) == _VALUE + assert (JSONPointer("") / key).resolve(data) == _VALUE + assert JSONPointer("").join(key).resolve(data) == _VALUE + + +@pytest.mark.parametrize("unicode_escape", [False, True]) +@pytest.mark.parametrize("key", KEYS) +def test_uri_decoding_preserves_native_unicode( + key: str, *, unicode_escape: bool +) -> None: + encoded = _token(key) + if unicode_escape: + encoded = json.dumps(encoded, ensure_ascii=False)[1:-1] + value = JSONPointer( + quote("/" + encoded, safe="/~"), + unicode_escape=unicode_escape, + uri_decode=True, + ) + assert value.resolve({key: _VALUE}) == _VALUE + + +def test_unicode_normalization_is_not_applied() -> None: + data = {"é": 1, "e\N{COMBINING ACUTE ACCENT}": 2} + assert JSONPointer("/é").resolve(data) == 1 + assert ( + JSONPointer("/e\N{COMBINING ACUTE ACCENT}").resolve(data) + == data["e\N{COMBINING ACUTE ACCENT}"] + ) + + +def test_escaped_unicode_and_native_unicode_can_be_mixed() -> None: + value = JSONPointer("/café/" + r"\ud83d\ude00") + assert value.resolve({"café": {"😀": _VALUE}}) == _VALUE + assert str(value) == "/café/😀" + + +def test_unicode_escape_flag_preserves_literal_escape_spelling() -> None: + data = {r"café\u0061": _VALUE, "caféa": _OTHER_VALUE} + assert JSONPointer(r"/café\u0061", unicode_escape=False).resolve(data) == _VALUE + assert ( + JSONPointer(r"/café\u0061", unicode_escape=True).resolve(data) == _OTHER_VALUE + ) + + +def test_percent_decoding_is_separate_from_unicode_escapes() -> None: + data = {"café%2F": _VALUE, "café%252F": _OTHER_VALUE} + assert JSONPointer("/café%252F", uri_decode=True).resolve(data) == _VALUE + assert JSONPointer("/café%252F", uri_decode=False).resolve(data) == _OTHER_VALUE + + +def test_a_pointer_from_a_query_match_keeps_its_native_parts() -> None: + data = {"café": {"😀": _VALUE}} + match = next(iter(finditer("$['café']['😀']", data))) + assert match.pointer().resolve(data) == _VALUE + assert str(match.pointer()) == "/café/😀" + + +@pytest.mark.parametrize( + "operation", ["add", "replace", "remove", "copy", "move", "test"] +) +@pytest.mark.parametrize("encoding", ["objects", "unicode-json", "ascii-json"]) +@pytest.mark.parametrize("unicode_escape", [False, True]) +def test_patch_targets_the_native_key( + operation: str, encoding: str, *, unicode_escape: bool +) -> None: + # Both keys exist so a lossy decoder silently changes the wrong member + # instead of raising. JSON input is decoded before pointer processing. + data = {"café": {"value": 1}, "café": {"value": _OTHER_VALUE}} + operations: List[Dict[str, object]] + if operation in ("copy", "move"): + operations = [{"op": operation, "from": "/café", "path": "/target"}] + elif operation == "test": + operations = [{"op": operation, "path": "/café/value", "value": 1}] + elif operation == "remove": + operations = [{"op": operation, "path": "/café"}] + else: + operations = [{"op": operation, "path": "/café/value", "value": 2}] + + value = ( + operations + if encoding == "objects" + else json.dumps(operations, ensure_ascii=encoding == "ascii-json") + ) + result = patch.apply(value, data, unicode_escape=unicode_escape) + assert result is data + assert data["café"] == {"value": _OTHER_VALUE} + if operation in ("remove", "move"): + assert "café" not in data + else: + assert data["café"] == {"value": 1 if operation in ("copy", "test") else 2} + if operation in ("copy", "move"): + assert data["target"] == {"value": 1} + + +def test_patched_and_atomic_operations_preserve_native_targets() -> None: + source = {"café": [1], "café": [_OTHER_VALUE]} + operations = JSONPatch().replace("/café/0", 2) + assert operations.patched(source) == {"café": [2], "café": [_OTHER_VALUE]} + assert source == {"café": [1], "café": [_OTHER_VALUE]} + assert operations.atomic(source) is source + assert source == {"café": [2], "café": [_OTHER_VALUE]} + + +@pytest.mark.parametrize( + "suffix", + [r"\u12", r"\xZZ", r"\U00110000", r"\N{NO SUCH NAME}", r"\ud800", "\\"], +) +def test_malformed_compatibility_escapes_are_still_refused(suffix: str) -> None: + with pytest.raises(UnicodeDecodeError): + JSONPointer("/café/" + suffix) + + +@pytest.mark.parametrize("key", [r"café\😀", r"café\é", r"漢\字"]) +def test_backslash_before_native_unicode_is_literal(key: str) -> None: + assert JSONPointer("/" + key).resolve({key: _VALUE}) == _VALUE + + +@pytest.mark.parametrize( + ("encoded", "key"), + [ + (r"\x61", "a"), + (r"\141", "a"), + (r"\U0001f600", "😀"), + (r"\N{GRINNING FACE}", "😀"), + (r"\ud83d\ude00", "😀"), + (r"\\u0061", r"\u0061"), + ], +) +def test_compatibility_escapes_with_a_native_prefix(encoded: str, key: str) -> None: + assert JSONPointer("/café/" + encoded).resolve({"café": {key: _VALUE}}) == _VALUE + + +@pytest.mark.parametrize("units", [chr(0xD800), chr(0xDC00), chr(0xD800) + chr(0xDC00)]) +@pytest.mark.parametrize("unicode_escape", [False, True]) +def test_literal_surrogates_keep_the_existing_validation( + units: str, *, unicode_escape: bool +) -> None: + key = "café/" + units + value = "/" + _token(key) + if unicode_escape: + with pytest.raises(UnicodeEncodeError): + JSONPointer(value, unicode_escape=unicode_escape) + else: + assert ( + JSONPointer(value, unicode_escape=unicode_escape).resolve({key: _VALUE}) + == _VALUE + )