From 348a2915f089ec6463438a560328cc6957541a11 Mon Sep 17 00:00:00 2001 From: Allan Yang Date: Tue, 29 Sep 2026 12:07:31 -0400 Subject: [PATCH 1/5] Bound ESCAPED_REGEX so unterminated openers cannot trigger polynomial matching The token-preserving regex used lazy wildcards (`<.*?>`, `{{.*?}}`, `%{.*?}`, `&\S*?;`). On a backtracking engine, a string with many opening delimiters and no closer (e.g. "<" * n) makes each attempt scan to the end of the input, fail, and retry from the next opener, which is O(n^2) in the input length (CWE-1333). Replace each wildcard with a negated character class bounded by the token's own delimiters, so an unterminated opener fails after one step. `\n` stays excluded to mirror `.`, so a token still never spans a line. Behaviour is unchanged for well-formed markup, Liquid tags, templates, URLs and entities (property-tested: 0 differences over 74,499 random strings with no nested delimiter). The only observable change is that a token containing a nested delimiter of its own kind (`x`, `{{ a {b} }}`, `&a&`) is no longer swallowed up to the first closer. Note that Ruby >= 3.2 already matched the previous pattern in linear time thanks to regex memoization (Regexp.linear_time? returns true for it); this change makes the bound hold on every engine the gem supports (required_ruby_version >= 2.7). Co-authored-by: Allan Yang --- CHANGELOG.md | 3 +- lib/pseudolocalization/pseudolocalizer.rb | 21 ++++++++-- test/pseudolocalization_test.rb | 48 +++++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc376e3..7992c37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased -Nil. +### Fixed +- `ESCAPED_REGEX` no longer does polynomial work on strings with many unterminated `<`, `{{`, `%{` or `&` openers (CWE-1333). The lazy wildcards were replaced with negated character classes bounded by each token's own delimiters. Well-formed markup, Liquid tags, templates, URLs and entities tokenize identically; the only observable change is that a token containing a nested delimiter of its own kind (e.g. `x`, `{{ a {b} }}`, `&a&`) is no longer swallowed up to the first closer. ## [0.9.1] - 2024-06-27 ### Added diff --git a/lib/pseudolocalization/pseudolocalizer.rb b/lib/pseudolocalization/pseudolocalizer.rb index e7c81ab..1e3e551 100644 --- a/lib/pseudolocalization/pseudolocalizer.rb +++ b/lib/pseudolocalization/pseudolocalizer.rb @@ -1,13 +1,26 @@ module Pseudolocalization module I18n class Pseudolocalizer + # Tokens that must be preserved verbatim: HTML tags, Liquid output tags, + # Ruby interpolation templates, URLs, and HTML entities. + # + # Each bracketed token uses a negated character class that stops at its + # own delimiters instead of a lazy wildcard (`<.*?>`). With a wildcard, + # an opener that is never closed (e.g. `"<" * n`) makes a backtracking + # engine scan to the end of the string, fail, and retry from the next + # opener, which is quadratic in the input length (CWE-1333). Bounding + # the interior means an unterminated opener fails after one step. + # + # `\n` is excluded to mirror `.`, so a token still never spans a line. + # Ruby >= 3.2 already matches the previous pattern in linear time via + # regex memoization; this change makes that true on every engine. ESCAPED_REGEX = Regexp.new("(#{ [ - "<.*?>", - "{{.*?}}", - "%{.*?}", + "<[^<>\\n]*>", + "{{[^{}\\n]*}}", + "%{[^{}\\n]*}", "https?:\/\/\\S+", - "&\\S*?;" + "&[^\\s&;]*;" ].join('|') })") diff --git a/test/pseudolocalization_test.rb b/test/pseudolocalization_test.rb index 19045c9..d5bc621 100644 --- a/test/pseudolocalization_test.rb +++ b/test/pseudolocalization_test.rb @@ -66,6 +66,54 @@ def test_it_works_with_templates assert_equal('Ḥḛḛḽḽṓṓ, ẁṓṓṛḽḍ %{firstname} %{lastname}!', @backend.translate(:en, 'Hello, world %{firstname} %{lastname}!', {})) end + def test_markup_never_spans_a_newline + # `.` never matched "\n" and the bounded character classes exclude it too, + # so a tag or Liquid tag split across lines is still treated as text. + assert_equal("αα <ṡṗααṇ\nͼḽααṡṡ=\"ẋ\">ḅ", @backend.translate(:en, "a b", {})) + assert_equal("αα {{\n ṇααṃḛḛ }} ḅ", @backend.translate(:en, "a {{\n name }} b", {})) + end + + def test_unterminated_openers_are_pseudolocalized_as_text + assert_equal('αα <ḅ ͼ', @backend.translate(:en, 'a ͼ ḍ', @backend.translate(:en, 'a c d', {})) + assert_equal('αα {{ ḅ | ϝ: {ͼ} }} ḍ', @backend.translate(:en, 'a {{ b | f: {c} }} d', {})) + assert_equal('αα %{ḅ{ͼ} ḍ', @backend.translate(:en, 'a %{b{c} d', {})) + assert_equal('αα &ḅ& ͼ', @backend.translate(:en, 'a &b& c', {})) + + # This includes a raw `<` inside a quoted attribute value. HTML serializers + # emit it as `<`; a literal `<` there is treated like any other nested + # opener. (`>` inside an attribute value already ended the tag before.) + assert_equal('<αα ṭḭḭṭḽḛḛ="αα < b">ḽḭḭṇḳ', @backend.translate(:en, 'link', {})) + end + + def test_many_unterminated_openers_complete_in_linear_time + # Regression guard for CWE-1333. With the former lazy wildcards (`<.*?>`), + # every unterminated opener re-scanned the rest of the string, so 100_000 + # openers took tens of seconds on a backtracking engine (Ruby < 3.2). + # The bounded pattern finishes in milliseconds on every engine. + require 'benchmark' + + ['<', '{{', '%{', '&'].each do |opener| + input = opener * 100_000 + elapsed = Benchmark.realtime { assert_equal(input, @backend.translate(:en, input, {})) } + assert_operator elapsed, :<, 2.0, "#{opener.inspect} * 100_000 took #{elapsed.round(2)}s" + end + end + + def test_escaped_regex_is_linear_time + skip 'Regexp.linear_time? was added in Ruby 3.2' unless Regexp.respond_to?(:linear_time?) + + assert Regexp.linear_time?(Pseudolocalization::I18n::Pseudolocalizer::ESCAPED_REGEX) + end + def test_it_allows_ignoring_cetain_keys @backend.ignores = ['Ignore*', /Clifford.$/] From 8270ad63f3b5644b2c070bfbafa8f450134b9f3a Mon Sep 17 00:00:00 2001 From: Allan Yang Date: Tue, 29 Sep 2026 12:34:05 -0400 Subject: [PATCH 2/5] Trim comments to the essentials The rationale lives in the PR and CHANGELOG. Keep one note at the regex so the bounded classes and the `\n` exclusion aren't "simplified" away, and one line per new test saying what it pins. Co-authored-by: Allan Yang --- CHANGELOG.md | 2 +- lib/pseudolocalization/pseudolocalizer.rb | 15 ++------------- test/pseudolocalization_test.rb | 17 +++++------------ 3 files changed, 8 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7992c37..f207023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased ### Fixed -- `ESCAPED_REGEX` no longer does polynomial work on strings with many unterminated `<`, `{{`, `%{` or `&` openers (CWE-1333). The lazy wildcards were replaced with negated character classes bounded by each token's own delimiters. Well-formed markup, Liquid tags, templates, URLs and entities tokenize identically; the only observable change is that a token containing a nested delimiter of its own kind (e.g. `x`, `{{ a {b} }}`, `&a&`) is no longer swallowed up to the first closer. +- Fixed slow matching in `ESCAPED_REGEX` on strings with many unclosed `<`, `{{`, `%{` or `&` (CWE-1333). Lazy wildcards became character classes bounded by each token's own delimiters. Output is unchanged for well-formed input; a token containing a nested opener of its own kind (`x`, `{{ a {b} }}`, `&a&`) is no longer swallowed up to the first closer. ## [0.9.1] - 2024-06-27 ### Added diff --git a/lib/pseudolocalization/pseudolocalizer.rb b/lib/pseudolocalization/pseudolocalizer.rb index 1e3e551..549aa29 100644 --- a/lib/pseudolocalization/pseudolocalizer.rb +++ b/lib/pseudolocalization/pseudolocalizer.rb @@ -1,19 +1,8 @@ module Pseudolocalization module I18n class Pseudolocalizer - # Tokens that must be preserved verbatim: HTML tags, Liquid output tags, - # Ruby interpolation templates, URLs, and HTML entities. - # - # Each bracketed token uses a negated character class that stops at its - # own delimiters instead of a lazy wildcard (`<.*?>`). With a wildcard, - # an opener that is never closed (e.g. `"<" * n`) makes a backtracking - # engine scan to the end of the string, fail, and retry from the next - # opener, which is quadratic in the input length (CWE-1333). Bounding - # the interior means an unterminated opener fails after one step. - # - # `\n` is excluded to mirror `.`, so a token still never spans a line. - # Ruby >= 3.2 already matches the previous pattern in linear time via - # regex memoization; this change makes that true on every engine. + # Bounded on purpose: an unclosed opener fails in one step instead of + # rescanning the rest of the string. `\n` is excluded to match `.`. ESCAPED_REGEX = Regexp.new("(#{ [ "<[^<>\\n]*>", diff --git a/test/pseudolocalization_test.rb b/test/pseudolocalization_test.rb index d5bc621..0593b04 100644 --- a/test/pseudolocalization_test.rb +++ b/test/pseudolocalization_test.rb @@ -67,8 +67,7 @@ def test_it_works_with_templates end def test_markup_never_spans_a_newline - # `.` never matched "\n" and the bounded character classes exclude it too, - # so a tag or Liquid tag split across lines is still treated as text. + # Same as before the fix: `.` never matched "\n". assert_equal("αα <ṡṗααṇ\nͼḽααṡṡ=\"ẋ\">ḅ", @backend.translate(:en, "a b", {})) assert_equal("αα {{\n ṇααṃḛḛ }} ḅ", @backend.translate(:en, "a {{\n name }} b", {})) end @@ -81,24 +80,18 @@ def test_unterminated_openers_are_pseudolocalized_as_text end def test_a_nested_opener_ends_the_outer_token - # A token containing a delimiter of its own kind is not swallowed up to the - # first closer; the inner well-formed token is preserved instead. + # The one behaviour change: an opener inside an unclosed token ends it, + # instead of the token swallowing everything up to the first closer. assert_equal('αα <ḅ ͼ ḍ', @backend.translate(:en, 'a c d', {})) assert_equal('αα {{ ḅ | ϝ: {ͼ} }} ḍ', @backend.translate(:en, 'a {{ b | f: {c} }} d', {})) assert_equal('αα %{ḅ{ͼ} ḍ', @backend.translate(:en, 'a %{b{c} d', {})) assert_equal('αα &ḅ& ͼ', @backend.translate(:en, 'a &b& c', {})) - - # This includes a raw `<` inside a quoted attribute value. HTML serializers - # emit it as `<`; a literal `<` there is treated like any other nested - # opener. (`>` inside an attribute value already ended the tag before.) + # A raw `<` inside an attribute value counts too (HTML tools write `<`). assert_equal('<αα ṭḭḭṭḽḛḛ="αα < b">ḽḭḭṇḳ', @backend.translate(:en, 'link', {})) end def test_many_unterminated_openers_complete_in_linear_time - # Regression guard for CWE-1333. With the former lazy wildcards (`<.*?>`), - # every unterminated opener re-scanned the rest of the string, so 100_000 - # openers took tens of seconds on a backtracking engine (Ruby < 3.2). - # The bounded pattern finishes in milliseconds on every engine. + # CWE-1333 guard. The old `<.*?>` took seconds here on Ruby < 3.2. require 'benchmark' ['<', '{{', '%{', '&'].each do |opener| From 09d3b50ce1a5e89fee8db448e1b72f0b21ad2ad9 Mon Sep 17 00:00:00 2001 From: Allan Yang Date: Wed, 30 Sep 2026 19:06:10 +0000 Subject: [PATCH 3/5] Time the linear-time test without the benchmark gem benchmark stopped being a default gem in Ruby 3.5, so under bundler on Ruby 3.5+ `require 'benchmark'` raises LoadError unless the gem is in the Gemfile. Process.clock_gettime(CLOCK_MONOTONIC) is core Ruby and is what Benchmark.realtime uses internally, so the measurement is unchanged. --- test/pseudolocalization_test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/pseudolocalization_test.rb b/test/pseudolocalization_test.rb index 0593b04..99d375a 100644 --- a/test/pseudolocalization_test.rb +++ b/test/pseudolocalization_test.rb @@ -92,11 +92,11 @@ def test_a_nested_opener_ends_the_outer_token def test_many_unterminated_openers_complete_in_linear_time # CWE-1333 guard. The old `<.*?>` took seconds here on Ruby < 3.2. - require 'benchmark' - ['<', '{{', '%{', '&'].each do |opener| input = opener * 100_000 - elapsed = Benchmark.realtime { assert_equal(input, @backend.translate(:en, input, {})) } + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + assert_equal(input, @backend.translate(:en, input, {})) + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started assert_operator elapsed, :<, 2.0, "#{opener.inspect} * 100_000 took #{elapsed.round(2)}s" end end From cf2542f5bd39ab3ae2cd82f13cb6844b93347396 Mon Sep 17 00:00:00 2001 From: Allan Yang Date: Wed, 30 Sep 2026 19:44:39 +0000 Subject: [PATCH 4/5] Remove comments that only make sense in the context of this PR Per review: the history of the pattern lives in the PR and CHANGELOG, and the test names already say what each test pins. Co-authored-by: Allan Yang --- lib/pseudolocalization/pseudolocalizer.rb | 2 -- test/pseudolocalization_test.rb | 5 ----- 2 files changed, 7 deletions(-) diff --git a/lib/pseudolocalization/pseudolocalizer.rb b/lib/pseudolocalization/pseudolocalizer.rb index 549aa29..be652b4 100644 --- a/lib/pseudolocalization/pseudolocalizer.rb +++ b/lib/pseudolocalization/pseudolocalizer.rb @@ -1,8 +1,6 @@ module Pseudolocalization module I18n class Pseudolocalizer - # Bounded on purpose: an unclosed opener fails in one step instead of - # rescanning the rest of the string. `\n` is excluded to match `.`. ESCAPED_REGEX = Regexp.new("(#{ [ "<[^<>\\n]*>", diff --git a/test/pseudolocalization_test.rb b/test/pseudolocalization_test.rb index 99d375a..b29d41f 100644 --- a/test/pseudolocalization_test.rb +++ b/test/pseudolocalization_test.rb @@ -67,7 +67,6 @@ def test_it_works_with_templates end def test_markup_never_spans_a_newline - # Same as before the fix: `.` never matched "\n". assert_equal("αα <ṡṗααṇ\nͼḽααṡṡ=\"ẋ\">ḅ", @backend.translate(:en, "a b", {})) assert_equal("αα {{\n ṇααṃḛḛ }} ḅ", @backend.translate(:en, "a {{\n name }} b", {})) end @@ -80,18 +79,14 @@ def test_unterminated_openers_are_pseudolocalized_as_text end def test_a_nested_opener_ends_the_outer_token - # The one behaviour change: an opener inside an unclosed token ends it, - # instead of the token swallowing everything up to the first closer. assert_equal('αα <ḅ ͼ ḍ', @backend.translate(:en, 'a c d', {})) assert_equal('αα {{ ḅ | ϝ: {ͼ} }} ḍ', @backend.translate(:en, 'a {{ b | f: {c} }} d', {})) assert_equal('αα %{ḅ{ͼ} ḍ', @backend.translate(:en, 'a %{b{c} d', {})) assert_equal('αα &ḅ& ͼ', @backend.translate(:en, 'a &b& c', {})) - # A raw `<` inside an attribute value counts too (HTML tools write `<`). assert_equal('<αα ṭḭḭṭḽḛḛ="αα < b">ḽḭḭṇḳ', @backend.translate(:en, 'link', {})) end def test_many_unterminated_openers_complete_in_linear_time - # CWE-1333 guard. The old `<.*?>` took seconds here on Ruby < 3.2. ['<', '{{', '%{', '&'].each do |opener| input = opener * 100_000 started = Process.clock_gettime(Process::CLOCK_MONOTONIC) From adb5201e0b09a6acaa15a43f3e5f6b14290b20da Mon Sep 17 00:00:00 2001 From: Allan Yang Date: Wed, 30 Sep 2026 20:13:11 +0000 Subject: [PATCH 5/5] Let quoted HTML attribute values contain < and > A quoted attribute value is skipped as a unit, so and stay intact, as they did before the regex change. Every branch still fails at the next unquoted <, so matching stays linear. Co-authored-by: Allan Yang --- CHANGELOG.md | 2 +- lib/pseudolocalization/pseudolocalizer.rb | 2 +- test/pseudolocalization_test.rb | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f207023..67262e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased ### Fixed -- Fixed slow matching in `ESCAPED_REGEX` on strings with many unclosed `<`, `{{`, `%{` or `&` (CWE-1333). Lazy wildcards became character classes bounded by each token's own delimiters. Output is unchanged for well-formed input; a token containing a nested opener of its own kind (`x`, `{{ a {b} }}`, `&a&`) is no longer swallowed up to the first closer. +- Fixed slow matching in `ESCAPED_REGEX` on strings with many unclosed `<`, `{{`, `%{` or `&` (CWE-1333). Lazy wildcards became character classes bounded by each token's own delimiters. Output is unchanged or improved for well-formed input; a token containing a nested opener of its own kind (`x`, `{{ a {b} }}`, `&a&`) is no longer swallowed up to the first closer. Quoted HTML attribute values may now contain `<` and `>`. ## [0.9.1] - 2024-06-27 ### Added diff --git a/lib/pseudolocalization/pseudolocalizer.rb b/lib/pseudolocalization/pseudolocalizer.rb index be652b4..648f3ea 100644 --- a/lib/pseudolocalization/pseudolocalizer.rb +++ b/lib/pseudolocalization/pseudolocalizer.rb @@ -3,7 +3,7 @@ module I18n class Pseudolocalizer ESCAPED_REGEX = Regexp.new("(#{ [ - "<[^<>\\n]*>", + "<[^<>\"'\\n]*(?:(?:\"[^\"\\n]*\"|'[^'\\n]*')[^<>\"'\\n]*)*>", "{{[^{}\\n]*}}", "%{[^{}\\n]*}", "https?:\/\/\\S+", diff --git a/test/pseudolocalization_test.rb b/test/pseudolocalization_test.rb index b29d41f..8c82105 100644 --- a/test/pseudolocalization_test.rb +++ b/test/pseudolocalization_test.rb @@ -83,11 +83,15 @@ def test_a_nested_opener_ends_the_outer_token assert_equal('αα {{ ḅ | ϝ: {ͼ} }} ḍ', @backend.translate(:en, 'a {{ b | f: {c} }} d', {})) assert_equal('αα %{ḅ{ͼ} ḍ', @backend.translate(:en, 'a %{b{c} d', {})) assert_equal('αα &ḅ& ͼ', @backend.translate(:en, 'a &b& c', {})) - assert_equal('<αα ṭḭḭṭḽḛḛ="αα < b">ḽḭḭṇḳ', @backend.translate(:en, 'link', {})) + end + + def test_quoted_attribute_values_can_contain_angle_brackets + assert_equal('ḽḭḭṇḳ', @backend.translate(:en, 'link', {})) + assert_equal("ḽḭḭṇḳ", @backend.translate(:en, "link", {})) end def test_many_unterminated_openers_complete_in_linear_time - ['<', '{{', '%{', '&'].each do |opener| + ['<', '<"', "<'", '{{', '%{', '&'].each do |opener| input = opener * 100_000 started = Process.clock_gettime(Process::CLOCK_MONOTONIC) assert_equal(input, @backend.translate(:en, input, {}))