Widen ESCAPED_REGEX: comments, Liquid tags, multi-line markup, quoted braces

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## Unreleased
 ### Fixed
 - Fixed slow matching in `ESCAPED_REGEX` on strings with many unclosed `<`, `{{`, `%{` or `&` (CWE-1333). Lazy wildcards became character classes bounded by each token's own delimiters. Output is unchanged or improved for well-formed input; a token containing a nested opener of its own kind (`<b <i>x</i>`, `{{ a {b} }}`, `&a&amp;`) is no longer swallowed up to the first closer. Quoted HTML attribute values may now contain `<` and `>`.
+- `ESCAPED_REGEX` now also preserves HTML comments (`<!-- -->`), Liquid tags (`{% %}`), markup spanning a newline, and `{`/`}` inside quoted Liquid strings; a `<` not followed by a letter, `/`, `!` or `?` is treated as text; a URL ends at `<` or `>`; only well-formed character references (`&name;`, `&#39;`, `&#x27;`) are kept. Output is unchanged for every string in Shopify Core's English locale files except 19 Liquid email subjects, where `{% if … %}` tags are now preserved instead of pseudolocalized.
 
 ## [0.9.1] - 2024-06-27
 ### Added

lib/pseudolocalization/pseudolocalizer.rb

@@ -3,11 +3,13 @@ module I18n
     class Pseudolocalizer
       ESCAPED_REGEX = Regexp.new("(#{
         [
-          "<[^<>\"'\\n]*(?:(?:\"[^\"\\n]*\"|'[^'\\n]*')[^<>\"'\\n]*)*>",
-          "{{[^{}\\n]*}}",
-          "%{[^{}\\n]*}",
-          "https?:\/\/\\S+",
-          "&[^\\s&;]*;"
+          "<!--(?:[^-]|-[^-])*-->",                                            # HTML comment
+          "<[a-zA-Z/!?][^<>\"']*(?:(?:\"[^\"]*\"|'[^']*')[^<>\"']*)*>",         # HTML tag, end tag, doctype
+          "{{[^{}\"']*(?:(?:\"[^\"]*\"|'[^']*')[^{}\"']*)*}}",                 # Liquid output
+          "{%-?\\s*[a-zA-Z#][^%\"']*(?:(?:\"[^\"]*\"|'[^']*')[^%\"']*)*%}",    # Liquid tag
+          "%{[^{}\\n]*}",                                                      # Ruby I18n interpolation
+          "https?:\/\/[^\\s<>]+",                                             # URL
+          "&(?:[a-zA-Z][a-zA-Z0-9]*|#[0-9]+|#[xX][0-9a-fA-F]+);",              # HTML character reference
         ].join('|')
       })")
 

test/pseudolocalization_test.rb

@@ -66,9 +66,42 @@ def test_it_works_with_templates
     assert_equal('Ḥḛḛḽḽṓṓ, ẁṓṓṛḽḍ %{firstname} %{lastname}!', @backend.translate(:en, 'Hello, world %{firstname} %{lastname}!', {}))
   end
 
-  def test_markup_never_spans_a_newline
-    assert_equal("αα <ṡṗααṇ\nͼḽααṡṡ=\"ẋ\">ḅ</span>", @backend.translate(:en, "a <span\nclass=\"x\">b</span>", {}))
-    assert_equal("αα {{\n ṇααṃḛḛ }} ḅ", @backend.translate(:en, "a {{\n name }} b", {}))
+  def test_markup_may_span_a_newline
+    assert_equal("αα <span\nclass=\"x\">ḅ</span>", @backend.translate(:en, "a <span\nclass=\"x\">b</span>", {}))
+    assert_equal("αα {{\n name }} ḅ", @backend.translate(:en, "a {{\n name }} b", {}))
+    assert_equal("αα <!--\n note\n--> ḅ", @backend.translate(:en, "a <!--\n note\n--> b", {}))
+  end
+
+  def test_a_less_than_sign_that_does_not_start_a_tag_is_text
+    assert_equal('3<4 ααṇḍ 5>2', @backend.translate(:en, '3<4 and 5>2', {}))
+    assert_equal('αα < ḅ <br/> ͼ > ḍ', @backend.translate(:en, 'a < b <br/> c > d', {}))
+    assert_equal("Ṫααḡ '<%{tag_name}>' ḭḭṡ ṇṓṓṭ ṗḛḛṛṃḭḭṭṭḛḛḍ", @backend.translate(:en, "Tag '<%{tag_name}>' is not permitted", {}))
+  end
+
+  def test_html_comments_and_doctype_are_preserved
+    assert_equal("<!-- don't > \"do\" that --> ẋ", @backend.translate(:en, "<!-- don't > \"do\" that --> x", {}))
+    assert_equal('<!DOCTYPE html> ẋ', @backend.translate(:en, '<!DOCTYPE html> x', {}))
+  end
+
+  def test_liquid_strings_may_contain_braces
+    assert_equal('{{ x | default: "{" }} ẋ', @backend.translate(:en, '{{ x | default: "{" }} x', {}))
+    assert_equal("{{ d | date: '%B %e, %Y' }} ẋ", @backend.translate(:en, "{{ d | date: '%B %e, %Y' }} x", {}))
+  end
+
+  def test_liquid_tags_are_preserved
+    assert_equal('ṎṎṛḍḛḛṛ {{ name }}{% if customer.name %} ṗḽααͼḛḛḍ ḅẏẏ {{ customer.name }}{% endif %}',
+      @backend.translate(:en, 'Order {{ name }}{% if customer.name %} placed by {{ customer.name }}{% endif %}', {}))
+    assert_equal('{%- if a contains "50%" -%}ẋ{% endif %}', @backend.translate(:en, '{%- if a contains "50%" -%}x{% endif %}', {}))
+    assert_equal("ṡẏẏṇṭααẋ ('{{', '}}', '{%' ṓṓṛ '%}')", @backend.translate(:en, "syntax ('{{', '}}', '{%' or '%}')", {}))
+  end
+
+  def test_a_url_ends_at_a_tag
+    assert_equal('https://x.com/a?b=1&c=2<br>Ṅṓṓẁ', @backend.translate(:en, 'https://x.com/a?b=1&c=2<br>Now', {}))
+    assert_equal("https://x.io/Don't_Look_Up ẋ", @backend.translate(:en, "https://x.io/Don't_Look_Up x", {}))
+  end
+
+  def test_only_well_formed_character_references_are_preserved
+    assert_equal('&amp; &#39; &#x27; &frac12; &ϝṓṓṓṓ-ḅααṛ; ḀḀṪ&Ṫ', @backend.translate(:en, '&amp; &#39; &#x27; &frac12; &foo-bar; AT&T', {}))
   end
 
   def test_unterminated_openers_are_pseudolocalized_as_text
@@ -91,7 +124,7 @@ def test_quoted_attribute_values_can_contain_angle_brackets
   end
 
   def test_many_unterminated_openers_complete_in_linear_time
-    ['<', '<"', "<'", '{{', '%{', '&'].each do |opener|
+    ['<', '</"', "</'", '<!--', '<!-- -', '{{', '{{"', "{{'", '{%', '{% "', '%{', '&', '&#'].each do |opener|
       input = opener * 100_000
       started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       assert_equal(input, @backend.translate(:en, input, {}))