From 22bf6eaa381e28d912c73f3d45208a95b5f9be83 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 03:43:22 +1100 Subject: [PATCH 01/11] Simplify parsing of string constant in patchedast Now that all supported python version supports ast end_col_offset, we can rely on the end node address instead of regex to find the node region. This is much simpler and way less error prone than using regex. --- rope/refactor/patchedast.py | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index a30b0440..0eeab10d 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -67,7 +67,7 @@ def __init__(self, source, children=False): self.source = _Source(source) self.children = children self.lines = codeanalyze.SourceLinesAdapter(source) - self.ast_adapter = codeanalyze.ASTLinesAdapter(source) + self.ast_adapter = self.source.ast_adapter self.children_stack = [] Number = object() @@ -115,9 +115,7 @@ def _handle(self, node, base_children, eat_parens=False, eat_spaces=False): token_start = child.region[0] else: if child is self.String: - region = self.source.consume_string( - end=self._find_next_statement_start() - ) + region = self.source.consume_node(node) elif child is self.Number: region = self.source.consume_number() elif child == self.empty_tuple: @@ -397,10 +395,7 @@ def end_quote_char(): QUOTE_CHARS = ['"""', "'''", '"', "'"] offset = self.source.offset - start, end = self.source.consume_string( - end=self._find_next_statement_start(), - ) - self.source.offset = offset + start, end = self.ast_adapter[node] children = [] children.append(start_quote_char()) @@ -909,6 +904,7 @@ class _Source: def __init__(self, source): self.source = source self.offset = 0 + self.ast_adapter = codeanalyze.ASTLinesAdapter(source) def consume(self, token, skip_comment=True): try: @@ -930,21 +926,11 @@ def consume_joined_string(self, token): self.offset = new_offset + len(token) return (new_offset, self.offset) - def consume_string(self, end=None): - if _Source._string_pattern is None: - string_pattern = codeanalyze.get_string_pattern() - formatted_string_pattern = codeanalyze.get_formatted_string_pattern() - original = r"(?:{})|(?:{})".format( - string_pattern, - formatted_string_pattern, - ) - pattern = r"({})((\s|\\\n|#[^\n]*\n)*({}))*".format( - original, - original, - ) - _Source._string_pattern = re.compile(pattern) - repattern = _Source._string_pattern - return self._consume_pattern(repattern, end) + def consume_node(self, node): + start, end = self.ast_adapter[node] + if self.offset < end: + self.offset = end + return start, end def consume_number(self): if _Source._number_pattern is None: @@ -1022,5 +1008,4 @@ def _get_number_pattern(self): integer = r"\-?(0[xo][\da-fA-F]+|\d+)" return r"(%s(\.\d*)?|(\.\d+))([eE][-+]?\d+)?[jJ]?" % integer - _string_pattern = None _number_pattern = None From d6e3ffe9a277461dc4b6882abb5ebcdeb46ba304 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 03:52:14 +1100 Subject: [PATCH 02/11] Simplify parsing of number constant in patchedast Use consume_node() instead of regex pattern matching. --- rope/refactor/patchedast.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 0eeab10d..1c0cc7c9 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -117,7 +117,7 @@ def _handle(self, node, base_children, eat_parens=False, eat_spaces=False): if child is self.String: region = self.source.consume_node(node) elif child is self.Number: - region = self.source.consume_number() + region = self.source.consume_node(node) elif child == self.empty_tuple: region = self.source.consume_empty_tuple() elif child == self.with_or_comma_context_manager: @@ -932,12 +932,6 @@ def consume_node(self, node): self.offset = end return start, end - def consume_number(self): - if _Source._number_pattern is None: - _Source._number_pattern = re.compile(self._get_number_pattern()) - repattern = _Source._number_pattern - return self._consume_pattern(repattern) - def consume_empty_tuple(self): return self._consume_pattern(re.compile(r"\(\s*\)")) @@ -1002,10 +996,3 @@ def find_backwards(self, pattern, offset): def __getitem__(self, index): return self.source[index] - - def _get_number_pattern(self): - # HACK: It is merely an approaximation and does the job - integer = r"\-?(0[xo][\da-fA-F]+|\d+)" - return r"(%s(\.\d*)?|(\.\d+))([eE][-+]?\d+)?[jJ]?" % integer - - _number_pattern = None From ad1106b554ef6a43ca99737fa1668380a211f8a3 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 03:53:02 +1100 Subject: [PATCH 03/11] Remove deprecated node handlers that has now become ast.Constant() These node handlers were used in very old python version that was no longer supported. The relevant handler are now implemented by _PatchingASTWalker._Constant(). --- rope/refactor/patchedast.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 1c0cc7c9..0651be0e 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -364,15 +364,6 @@ def _Constant(self, node): assert False - def _Num(self, node): - self._handle(node, [self.Number]) - - def _Str(self, node): - self._handle(node, [self.String]) - - def _Bytes(self, node): - self._handle(node, [self.String]) - def _JoinedStr(self, node): def start_quote_char(): possible_quotes = [ @@ -434,9 +425,6 @@ def _Dict(self, node): children.append("}") self._handle(node, children) - def _Ellipsis(self, node): - self._handle(node, ["..."]) - def _Expr(self, node): self._handle(node, [node.value]) @@ -648,6 +636,7 @@ def _Return(self, node): self._handle(node, children) def _Index(self, node): + assert False self._handle(node, [node.value]) def _Subscript(self, node): From 6e843397592daecbe54d1689fa0cf1a731027414 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 03:59:16 +1100 Subject: [PATCH 04/11] Remove deprecated _Index node handler In current version of python this is now handled by _Subscript() --- rope/refactor/patchedast.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 0651be0e..05a248f6 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -635,10 +635,6 @@ def _Return(self, node): children.append(node.value) self._handle(node, children) - def _Index(self, node): - assert False - self._handle(node, [node.value]) - def _Subscript(self, node): self._handle(node, [node.value, "[", node.slice, "]"]) From 49ec8002bdf548944a9b2caab62347ecbdca3a34 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 04:08:29 +1100 Subject: [PATCH 05/11] Simplify handling of Constant node Constant nodes are leaf nodes that doesn't have any children, they're simply consumed and does not need further recursive processing. --- rope/refactor/patchedast.py | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 05a248f6..7248e558 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -70,8 +70,7 @@ def __init__(self, source, children=False): self.ast_adapter = self.source.ast_adapter self.children_stack = [] - Number = object() - String = object() + Constant = object() with_or_comma_context_manager = object() empty_tuple = object() @@ -114,9 +113,7 @@ def _handle(self, node, base_children, eat_parens=False, eat_spaces=False): self(child) token_start = child.region[0] else: - if child is self.String: - region = self.source.consume_node(node) - elif child is self.Number: + if child is self.Constant: region = self.source.consume_node(node) elif child == self.empty_tuple: region = self.source.consume_empty_tuple() @@ -346,23 +343,7 @@ def _Delete(self, node): self._handle(node, ["del"] + self._child_nodes(node.targets, ",")) def _Constant(self, node): - if isinstance(node.value, (str, bytes)): - self._handle(node, [self.String]) - return - - if any(node.value is v for v in [True, False, None]): - self._handle(node, [str(node.value)]) - return - - if isinstance(node.value, numbers.Number): - self._handle(node, [self.Number]) - return - - if node.value is Ellipsis: - self._handle(node, ["..."]) - return - - assert False + self._handle(node, [self.Constant]) def _JoinedStr(self, node): def start_quote_char(): From 7deea3175ba9328e15da7b3e5cc5ff0df3732c56 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 04:11:54 +1100 Subject: [PATCH 06/11] Simplify parsing of empty_tuple, remove the empty_tuple sentinel --- rope/refactor/patchedast.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 7248e558..03bee884 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -72,7 +72,6 @@ def __init__(self, source, children=False): Constant = object() with_or_comma_context_manager = object() - empty_tuple = object() def __call__(self, node): method = getattr(self, "_" + node.__class__.__name__, None) @@ -115,8 +114,6 @@ def _handle(self, node, base_children, eat_parens=False, eat_spaces=False): else: if child is self.Constant: region = self.source.consume_node(node) - elif child == self.empty_tuple: - region = self.source.consume_empty_tuple() elif child == self.with_or_comma_context_manager: region = self.source.consume_with_or_comma_context_manager() elif isinstance(node, (ast.JoinedStr, ast.FormattedValue)): @@ -689,7 +686,8 @@ def _Tuple(self, node): if node.elts: self._handle(node, self._child_nodes(node.elts, ","), eat_parens=True) else: - self._handle(node, [self.empty_tuple]) + empty_tuple = self.ast_adapter.get_source_segment(node) + self._handle(node, [empty_tuple]) def _UnaryOp(self, node): children = self._get_op(node.op) @@ -898,9 +896,6 @@ def consume_node(self, node): self.offset = end return start, end - def consume_empty_tuple(self): - return self._consume_pattern(re.compile(r"\(\s*\)")) - def consume_with_or_comma_context_manager(self): repattern = re.compile(r"with|,") return self._consume_pattern(repattern) From 957e3ee598aa50926e5916560992672841999cf1 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 04:14:54 +1100 Subject: [PATCH 07/11] Renamed Constant sentinel to AtomicNode --- rope/refactor/patchedast.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 03bee884..e9f4984b 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -70,7 +70,7 @@ def __init__(self, source, children=False): self.ast_adapter = self.source.ast_adapter self.children_stack = [] - Constant = object() + AtomicNode = object() with_or_comma_context_manager = object() def __call__(self, node): @@ -112,7 +112,7 @@ def _handle(self, node, base_children, eat_parens=False, eat_spaces=False): self(child) token_start = child.region[0] else: - if child is self.Constant: + if child is self.AtomicNode: region = self.source.consume_node(node) elif child == self.with_or_comma_context_manager: region = self.source.consume_with_or_comma_context_manager() @@ -340,7 +340,7 @@ def _Delete(self, node): self._handle(node, ["del"] + self._child_nodes(node.targets, ",")) def _Constant(self, node): - self._handle(node, [self.Constant]) + self._handle(node, [self.AtomicNode]) def _JoinedStr(self, node): def start_quote_char(): From 525f4f79f647b6ac5fc0291a02c6ddc48b7bae76 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 04:40:22 +1100 Subject: [PATCH 08/11] Split out handling of withitems to their own level This simplifies the logic of dealing with comma in the withitems, we no longer need the with_or_comma_context_manager sentinel and special handling. --- rope/refactor/patchedast.py | 20 ++++++++------------ ropetest/refactor/patchedasttest.py | 24 ++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index e9f4984b..e21d67ff 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -71,7 +71,6 @@ def __init__(self, source, children=False): self.children_stack = [] AtomicNode = object() - with_or_comma_context_manager = object() def __call__(self, node): method = getattr(self, "_" + node.__class__.__name__, None) @@ -114,8 +113,6 @@ def _handle(self, node, base_children, eat_parens=False, eat_spaces=False): else: if child is self.AtomicNode: region = self.source.consume_node(node) - elif child == self.with_or_comma_context_manager: - region = self.source.consume_with_or_comma_context_manager() elif isinstance(node, (ast.JoinedStr, ast.FormattedValue)): region = self.source.consume_joined_string(child) else: @@ -718,16 +715,19 @@ def _While(self, node): children.extend(node.orelse) self._handle(node, children) + def _withitem(self, node): + children = [] + children.extend([node.context_expr]) + if node.optional_vars: + children.extend(["as", node.optional_vars]) + self._handle(node, children) + def _handle_with_node(self, node, is_async): children = [] if is_async: children.extend(["async"]) - for item in node.items: - children.extend([self.with_or_comma_context_manager, item.context_expr]) - if item.optional_vars: - children.extend(["as", item.optional_vars]) - children.append(":") + children.extend(["with", *self._child_nodes(node.items, ","), ":"]) children.extend(node.body) self._handle(node, children) @@ -896,10 +896,6 @@ def consume_node(self, node): self.offset = end return start, end - def consume_with_or_comma_context_manager(self): - repattern = re.compile(r"with|,") - return self._consume_pattern(repattern) - def _good_token(self, token, offset, start=None): """Checks whether consumed token is in comments""" if start is None: diff --git a/ropetest/refactor/patchedasttest.py b/ropetest/refactor/patchedasttest.py index 353933fb..f3320f45 100644 --- a/ropetest/refactor/patchedasttest.py +++ b/ropetest/refactor/patchedasttest.py @@ -1221,7 +1221,23 @@ def test_with_node(self): checker = _ResultChecker(self, ast_frag) checker.check_children( "With", - ["with", " ", "Name", " ", "as", " ", "Name", "", ":", "\n ", "Pass"], + ["with", " ", "withitem", "", ":", "\n ", "Pass"], + ) + checker.check_children( + "withitem", + ["Name", " ", "as", " ", "Name"], + ) + + def test_with_node_multiple_items(self): + source = dedent("""\ + with a as b, c as d: + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children( + "With", + ["with", " ", "withitem", "", ",", " ", "withitem", "", ":", "\n ", "Pass"], ) def test_async_with_node(self): @@ -1234,7 +1250,11 @@ async def afunc(): checker = _ResultChecker(self, ast_frag) checker.check_children( "AsyncWith", - ["async", " ", "with", " ", "Name", " ", "as", " ", "Name", "", ":", "\n ", "Pass"], + ["async", " ", "with", " ", "withitem", "", ":", "\n ", "Pass"], + ) + checker.check_children( + "withitem", + ["Name", " ", "as", " ", "Name"], ) def test_try_finally_node(self): From 916855cf75e00da521833db13035417d8e2d21cb Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 04:44:29 +1100 Subject: [PATCH 09/11] Remove unused method _Source._consume_pattern() Now that all the consume_string, consume_number, consume_with_or_comma_context_manager, and consume_empty_tuple have been removed, this method is no longer needed. --- rope/refactor/patchedast.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index e21d67ff..0455dc12 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -917,18 +917,6 @@ def _get_location(self): lines = self.source[: self.offset].split("\n") return (len(lines), len(lines[-1])) - def _consume_pattern(self, repattern, end=None): - while True: - if end is None: - end = len(self.source) - match = repattern.search(self.source, self.offset, end) - if self._good_token(match.group(), match.start()): - break - else: - self._skip_comment() - self.offset = match.end() - return match.start(), match.end() - def till_token(self, token): new_offset = self.source.index(token, self.offset) return self[self.offset : new_offset] From 89fcdc69f5b580a91cf410a52cede9d534c0f47c Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 04:46:58 +1100 Subject: [PATCH 10/11] Remove more unused methods These has long been unused. --- rope/refactor/patchedast.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 0455dc12..70b7143b 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -917,10 +917,6 @@ def _get_location(self): lines = self.source[: self.offset].split("\n") return (len(lines), len(lines[-1])) - def till_token(self, token): - new_offset = self.source.index(token, self.offset) - return self[self.offset : new_offset] - def rfind_token(self, token, start, end): index = start while True: @@ -933,11 +929,5 @@ def rfind_token(self, token, start, end): except ValueError: return None - def from_offset(self, offset): - return self[offset : self.offset] - - def find_backwards(self, pattern, offset): - return self.source.rindex(pattern, 0, offset) - def __getitem__(self, index): return self.source[index] From 522d0c0819efdacb8c61edfb3e7c6365b7937015 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 8 Oct 2026 05:12:20 +1100 Subject: [PATCH 11/11] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 743e2690..b3e7311e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # **Upcoming release** -- #895 patchedast Starred and keyword now consumes their syntactically expected and ** (lieryan) +- #895 patchedast Starred and keyword now consumes their syntactically expected and ** (@lieryan) +- #896 patchedast cleanup and refactoring (@lieryan) # Release 1.15.0