From f5473a9f5fd1c02494d2c7a9d45aa0109c214257 Mon Sep 17 00:00:00 2001 From: Allan Yang Date: Wed, 7 Oct 2026 12:10:42 -0400 Subject: [PATCH] Widen ESCAPED_REGEX: comments, Liquid tags, multi-line markup, quoted braces Follow-up to #97. Keeps Regexp.linear_time? true. Zero output changes on Shopify Core's English locale files except 19 Liquid email subjects where {% if %} tags are now preserved instead of pseudolocalized. --- CHANGELOG.md | 1 + lib/pseudolocalization/pseudolocalizer.rb | 12 ++++--- test/pseudolocalization_test.rb | 41 ++++++++++++++++++++--- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67262e5..36b4972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased ### Fixed - Fixed slow matching in `ESCAPED_REGEX` on strings with many unclosed `<`, `{{`, `%{` or `&` (CWE-1333). Lazy wildcards became character classes bounded by each token's own delimiters. Output is unchanged or improved for well-formed input; a token containing a nested opener of its own kind (`x`, `{{ a {b} }}`, `&a&`) is no longer swallowed up to the first closer. Quoted HTML attribute values may now contain `<` and `>`. +- `ESCAPED_REGEX` now also preserves HTML comments (``), Liquid tags (`{% %}`), markup spanning a newline, and `{`/`}` inside quoted Liquid strings; a `<` not followed by a letter, `/`, `!` or `?` is treated as text; a URL ends at `<` or `>`; only well-formed character references (`&name;`, `'`, `'`) are kept. Output is unchanged for every string in Shopify Core's English locale files except 19 Liquid email subjects, where `{% if … %}` tags are now preserved instead of pseudolocalized. ## [0.9.1] - 2024-06-27 ### Added diff --git a/lib/pseudolocalization/pseudolocalizer.rb b/lib/pseudolocalization/pseudolocalizer.rb index 648f3ea..2ec300f 100644 --- a/lib/pseudolocalization/pseudolocalizer.rb +++ b/lib/pseudolocalization/pseudolocalizer.rb @@ -3,11 +3,13 @@ module I18n class Pseudolocalizer ESCAPED_REGEX = Regexp.new("(#{ [ - "<[^<>\"'\\n]*(?:(?:\"[^\"\\n]*\"|'[^'\\n]*')[^<>\"'\\n]*)*>", - "{{[^{}\\n]*}}", - "%{[^{}\\n]*}", - "https?:\/\/\\S+", - "&[^\\s&;]*;" + "", # HTML comment + "<[a-zA-Z/!?][^<>\"']*(?:(?:\"[^\"]*\"|'[^']*')[^<>\"']*)*>", # HTML tag, end tag, doctype + "{{[^{}\"']*(?:(?:\"[^\"]*\"|'[^']*')[^{}\"']*)*}}", # Liquid output + "{%-?\\s*[a-zA-Z#][^%\"']*(?:(?:\"[^\"]*\"|'[^']*')[^%\"']*)*%}", # Liquid tag + "%{[^{}\\n]*}", # Ruby I18n interpolation + "https?:\/\/[^\\s<>]+", # URL + "&(?:[a-zA-Z][a-zA-Z0-9]*|#[0-9]+|#[xX][0-9a-fA-F]+);", # HTML character reference ].join('|') })") diff --git a/test/pseudolocalization_test.rb b/test/pseudolocalization_test.rb index 8c82105..05167cc 100644 --- a/test/pseudolocalization_test.rb +++ b/test/pseudolocalization_test.rb @@ -66,9 +66,42 @@ def test_it_works_with_templates assert_equal('Ḥḛḛḽḽṓṓ, ẁṓṓṛḽḍ %{firstname} %{lastname}!', @backend.translate(:en, 'Hello, world %{firstname} %{lastname}!', {})) end - def test_markup_never_spans_a_newline - assert_equal("αα <ṡṗααṇ\nͼḽααṡṡ=\"ẋ\">ḅ", @backend.translate(:en, "a b", {})) - assert_equal("αα {{\n ṇααṃḛḛ }} ḅ", @backend.translate(:en, "a {{\n name }} b", {})) + def test_markup_may_span_a_newline + assert_equal("αα ḅ", @backend.translate(:en, "a b", {})) + assert_equal("αα {{\n name }} ḅ", @backend.translate(:en, "a {{\n name }} b", {})) + assert_equal("αα ḅ", @backend.translate(:en, "a b", {})) + end + + def test_a_less_than_sign_that_does_not_start_a_tag_is_text + assert_equal('3<4 ααṇḍ 5>2', @backend.translate(:en, '3<4 and 5>2', {})) + assert_equal('αα < ḅ
ͼ > ḍ', @backend.translate(:en, 'a < b
c > d', {})) + assert_equal("Ṫααḡ '<%{tag_name}>' ḭḭṡ ṇṓṓṭ ṗḛḛṛṃḭḭṭṭḛḛḍ", @backend.translate(:en, "Tag '<%{tag_name}>' is not permitted", {})) + end + + def test_html_comments_and_doctype_are_preserved + assert_equal(" ẋ", @backend.translate(:en, " x", {})) + assert_equal(' ẋ', @backend.translate(:en, ' x', {})) + end + + def test_liquid_strings_may_contain_braces + assert_equal('{{ x | default: "{" }} ẋ', @backend.translate(:en, '{{ x | default: "{" }} x', {})) + assert_equal("{{ d | date: '%B %e, %Y' }} ẋ", @backend.translate(:en, "{{ d | date: '%B %e, %Y' }} x", {})) + end + + def test_liquid_tags_are_preserved + assert_equal('ṎṎṛḍḛḛṛ {{ name }}{% if customer.name %} ṗḽααͼḛḛḍ ḅẏẏ {{ customer.name }}{% endif %}', + @backend.translate(:en, 'Order {{ name }}{% if customer.name %} placed by {{ customer.name }}{% endif %}', {})) + assert_equal('{%- if a contains "50%" -%}ẋ{% endif %}', @backend.translate(:en, '{%- if a contains "50%" -%}x{% endif %}', {})) + assert_equal("ṡẏẏṇṭααẋ ('{{', '}}', '{%' ṓṓṛ '%}')", @backend.translate(:en, "syntax ('{{', '}}', '{%' or '%}')", {})) + end + + def test_a_url_ends_at_a_tag + assert_equal('https://x.com/a?b=1&c=2
Ṅṓṓẁ', @backend.translate(:en, 'https://x.com/a?b=1&c=2
Now', {})) + assert_equal("https://x.io/Don't_Look_Up ẋ", @backend.translate(:en, "https://x.io/Don't_Look_Up x", {})) + end + + def test_only_well_formed_character_references_are_preserved + assert_equal('& ' ' ½ &ϝṓṓṓṓ-ḅααṛ; ḀḀṪ&Ṫ', @backend.translate(:en, '& ' ' ½ &foo-bar; AT&T', {})) end def test_unterminated_openers_are_pseudolocalized_as_text @@ -91,7 +124,7 @@ def test_quoted_attribute_values_can_contain_angle_brackets end def test_many_unterminated_openers_complete_in_linear_time - ['<', '<"', "<'", '{{', '%{', '&'].each do |opener| + ['<', '