From ee566e8230b84c417ed2e8da4a28b9e55eddf9b7 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:55:53 -0700 Subject: [PATCH 1/2] feat(PARSE-02): add binding-aware compiler front end --- IMPLEMENTATION_PLAN.md | 4 + README.md | 4 + docs/compiler-front-end.md | 31 +++ docs/compiler-language.md | 4 +- src/JavaScript/compiler.py | 385 ++++++++++++++++++++++++++++++++ tests/test_compiler_frontend.py | 100 +++++++++ 6 files changed, 527 insertions(+), 1 deletion(-) create mode 100644 docs/compiler-front-end.md create mode 100644 src/JavaScript/compiler.py create mode 100644 tests/test_compiler_frontend.py diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 2219c45..73ec1ed 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -370,6 +370,10 @@ Date, RegExp, Map/Set or prototype/descriptor expansion first. first syntax/semantic decisions and reference corpus. It does not claim that translation is implemented yet. +[The compiler front-end API](docs/compiler-front-end.md) implements parsing, +lexical binding, validation and immutable typed IR in the development checkout; +JavaScript emission remains a separate milestone. + ## Compiler contracts and artifacts Preserve ordinary supported Python truthiness, integers, Unicode, evaluation diff --git a/README.md b/README.md index f320f1a..b21f0f0 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,10 @@ globals and other unlisted APIs remain future work. Development is tracked in the [issues](https://github.com/outside-labs/Python-JavaScript/issues) and [Project](https://github.com/orgs/outside-labs/projects/11). +The development checkout now includes a [binding-aware compiler front end](docs/compiler-front-end.md) +that validates source and produces typed IR. JavaScript emission and browser +execution remain subsequent milestones. + ## Development ```sh diff --git a/docs/compiler-front-end.md b/docs/compiler-front-end.md new file mode 100644 index 0000000..6cdeb02 --- /dev/null +++ b/docs/compiler-front-end.md @@ -0,0 +1,31 @@ +# Compiler front end + +The development checkout provides JavaScript.compiler.parse_source(source, +filename=...) for the adopted python-browser/1 profile. It parses, binds and +validates source without running target code, then returns an immutable ModuleIR. +It does not emit or execute JavaScript yet; EMIT-01 owns that next step. + +The API uses only the declared Python runtime and standard library. A ModuleIR +records profile/schema identity, SHA-256 of UTF-8 input, stable lexical binding +IDs and typed nodes with original source spans. to_dict returns an independent +JSON-serializable snapshot; integer constants are decimal strings tagged as +python-int so an IR consumer cannot lose precision through JSON numbers. + +Bindings distinguish module constants/functions, function parameters/locals, +builtin APIs and explicit JavaScript registry aliases. Alias resolution follows +binding identity: a parameter named len or Array is not rewritten to a builtin +or imported intrinsic. Branch and loop value types merge conservatively; source +order, guards, defaults and errors remain represented for the emitter. + +CompileError exposes a Diagnostic with code, phase, message and SourceSpan. +Locations use one-based character columns, converting Python AST's UTF-8 byte +offsets. PJS100 covers syntax/features, PJS110 binding/import boundaries and +PJS120 unsupported operations/calls. Unknown syntax/imports are refused before +any output publication or target execution. Syntax, binding and validation +phases remain distinct. + +Use the adopted language contract for supported forms. Initial module bindings +are unconditional immutable literals; for loops are function-local with simple +targets and no else clause. Source is UTF-8 and limited to 1 MiB. This front end +does not load Python application imports, provide browser APIs or certify that +arbitrary dependencies can be translated. diff --git a/docs/compiler-language.md b/docs/compiler-language.md index 972de51..91f42d6 100644 --- a/docs/compiler-language.md +++ b/docs/compiler-language.md @@ -20,7 +20,7 @@ and arbitrary user objects are outside the first profile. | Scalars | None, bool, arbitrary-precision int and Unicode str literals; tuples of supported values. Floats/bytes/complex wait for separate numeric/binary contracts. | | Bindings | Simple names, immutable module constants, ordinary local assignments and top-level plain functions. Preserve lexical binding identity and evaluation order. | | Functions | Positional arguments and immutable literal defaults; return, direct calls and recursion. Keyword/variadic calls, nested closures, decorators, annotations and async functions are deferred. | -| Control flow | if/elif/else, for over approved strings/tuples, break/continue, ordinary return and expression statements. while/comprehensions/generators are deferred initially. | +| Control flow | if/elif/else, function-local for over approved strings/tuples, break/continue, ordinary return and expression statements; loop-else is deferred. while/comprehensions/generators are deferred initially. | | Operations | Supported scalar arithmetic/comparisons, Python and/or/not, identity on supported values, string membership and supported indexing. Mixed or unknown operand forms need helpers or a source-located rejection. | | Builtins/errors | len on adopted strings/tuples, print of adopted values, approved isinstance checks and explicit builtin TypeError raising needed by the fixture. No eval/exec, reflection or undeclared I/O. | | JavaScript API | Resolve explicit imports/aliases through a versioned symbol/method registry. Start with Array.of and length as a binding fixture; other exports/methods need adopted signatures and parity cases before lowering. | @@ -66,6 +66,8 @@ source-located diagnostics, not an interpreter/server fallback. ## Front end and diagnostics +Source input is UTF-8 with an explicit 1 MiB front-end limit. + Use ast -> bind -> validate -> typed IR -> lower -> emit. Never import or execute application/dependency modules to compile them. Carry original file/line/column spans and stable binding identities through every pass. Validate imports, diff --git a/src/JavaScript/compiler.py b/src/JavaScript/compiler.py new file mode 100644 index 0000000..5fa0d37 --- /dev/null +++ b/src/JavaScript/compiler.py @@ -0,0 +1,385 @@ +"""Binding-aware front end for the experimental python-browser/1 profile.""" + +import ast +from dataclasses import dataclass +import hashlib +import symtable +from typing import Literal + + +ValueType = Literal['statement', 'python-value', 'python-none', 'python-bool', + 'python-int', 'python-str', 'python-tuple', 'python-exception', + 'python-type', 'callable', 'javascript-type', 'javascript-array', + 'javascript-number', 'javascript-callable'] + + +@dataclass(frozen=True) +class SourceSpan: + filename: str + line: int + column: int + end_line: int + end_column: int + + +@dataclass(frozen=True) +class Diagnostic: + code: str + message: str + span: SourceSpan + phase: Literal['parse', 'bind', 'validate'] = 'validate' + + +class CompileError(ValueError): + def __init__(self, diagnostic: Diagnostic): + self.diagnostic = diagnostic + span = diagnostic.span + super().__init__(f'{span.filename}:{span.line}:{span.column}: {diagnostic.code}: {diagnostic.message}') + + +@dataclass(frozen=True) +class Binding: + id: str + name: str + scope: str + kind: Literal['constant', 'function', 'parameter', 'local', 'javascript', 'builtin'] + target: str | None = None + + +@dataclass(frozen=True) +class IRNode: + kind: str + value_type: ValueType + span: SourceSpan + fields: tuple[tuple[str, object], ...] = () + + def field(self, name, default=None): + return dict(self.fields).get(name, default) + + def to_dict(self): + def encode(value): + if isinstance(value, IRNode): + return value.to_dict() + if isinstance(value, tuple): + return [encode(item) for item in value] + return value + return {'kind': self.kind, 'value_type': self.value_type, + 'span': dict(vars(self.span)), + 'fields': {key: encode(value) for key, value in self.fields}} + + +@dataclass(frozen=True) +class ModuleIR: + filename: str + source_sha256: str + bindings: tuple[Binding, ...] + body: tuple[IRNode, ...] + profile: str = 'python-browser/1' + schema_version: int = 1 + + def to_dict(self): + return {'profile': self.profile, 'schema_version': self.schema_version, + 'filename': self.filename, 'source_sha256': self.source_sha256, + 'bindings': [dict(vars(binding)) for binding in self.bindings], + 'body': [node.to_dict() for node in self.body]} + + +_BUILTINS = {'len', 'print', 'isinstance', 'str', 'int', 'bool', 'tuple', 'TypeError'} +_TYPES = {'str', 'int', 'bool', 'tuple'} +_BINARY = {ast.Add: '+', ast.Sub: '-', ast.Mult: '*', ast.FloorDiv: '//', ast.Mod: '%'} +_COMPARE = {ast.Eq: '==', ast.NotEq: '!=', ast.Lt: '<', ast.LtE: '<=', ast.Gt: '>', + ast.GtE: '>=', ast.Is: 'is', ast.IsNot: 'is not', ast.In: 'in', ast.NotIn: 'not in'} + + +def _literal_type(value): + if value is None: + return 'python-none' + return {bool: 'python-bool', int: 'python-int', str: 'python-str', tuple: 'python-tuple'}.get(type(value)) + + +def _immutable(node): + try: + value = ast.literal_eval(node) + except (ValueError, TypeError, SyntaxError): + return False + def accepted(value): + return _literal_type(value) is not None and (not isinstance(value, tuple) or all(accepted(item) for item in value)) + return accepted(value) + + +class _FrontEnd: + def __init__(self, source, filename): + self.filename = filename + self.lines = source.splitlines() or [''] + self.bindings = {} + self.root = {} + self.locals = {} + self.types = {} + try: + self.tree = ast.parse(source, filename=filename) + self.table = symtable.symtable(source, filename, 'exec') + except SyntaxError as error: + span = SourceSpan(filename, error.lineno or 1, error.offset or 1, + error.end_lineno or error.lineno or 1, + error.end_offset or error.offset or 1) + raise CompileError(Diagnostic('PJS100', error.msg, span, 'parse')) from None + self._declare_module() + + def span(self, node): + def column(line, offset): + text = self.lines[line - 1] if 0 < line <= len(self.lines) else '' + return len(text.encode('utf-8')[:offset].decode('utf-8')) + 1 + line, end_line = node.lineno, node.end_lineno or node.lineno + return SourceSpan(self.filename, line, column(line, node.col_offset), + end_line, column(end_line, node.end_col_offset or node.col_offset)) + + def fail(self, code, message, node): + raise CompileError(Diagnostic(code, message, self.span(node), 'bind' if code == 'PJS110' else 'validate')) + + def node(self, original, kind, value_type='statement', **fields): + return IRNode(kind, value_type, self.span(original), tuple(fields.items())) + + def declare(self, name, scope, kind, target=None): + binding = Binding(scope + ':' + name, name, scope, kind, target) + self.bindings[binding.id] = binding + return binding + + def _declare_module(self): + for node in self.tree.body: + names = [] + kind, target = 'constant', None + if isinstance(node, ast.FunctionDef): + names, kind = [node.name], 'function' + elif isinstance(node, ast.Assign): + if len(node.targets) != 1 or not isinstance(node.targets[0], ast.Name) or not _immutable(node.value): + self.fail('PJS100', 'Module bindings require single-name immutable literals.', node) + names = [node.targets[0].id] + self.types['module:' + names[0]] = _literal_type(ast.literal_eval(node.value)) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + kind = 'javascript' + if isinstance(node, ast.Import): + if len(node.names) != 1 or node.names[0].name != 'JavaScript': + self.fail('PJS110', 'Only the approved JavaScript registry may be imported.', node) + names, target = [node.names[0].asname or 'JavaScript'], 'JavaScript' + else: + if node.level or node.module != 'JavaScript' or len(node.names) != 1 or node.names[0].name != 'Array': + self.fail('PJS110', 'Only the approved JavaScript.Array binding may be imported.', node) + names, target = [node.names[0].asname or 'Array'], 'JavaScript.Array' + for name in names: + if name in self.root: + self.fail('PJS100', 'Module rebinding is outside the immutable profile.', node) + self.root[name] = self.declare(name, 'module', kind, target) + for child in self.table.get_children(): + if child.get_name() in self.root and self.root[child.get_name()].kind == 'function': + scope = 'function:' + child.get_name() + self.locals[scope] = { + symbol.get_name(): self.declare(symbol.get_name(), scope, + 'parameter' if symbol.is_parameter() else 'local') + for symbol in sorted(child.get_symbols(), key=lambda symbol: symbol.get_name()) + if symbol.is_local() + } + + def resolve(self, name, scope, node): + if scope and name in self.locals[scope]: + return self.locals[scope][name] + if name in self.root: + return self.root[name] + if name in _BUILTINS: + identifier = 'builtin:' + name + if identifier not in self.bindings: + self.declare(name, 'builtin', 'builtin', identifier) + return self.bindings[identifier] + self.fail('PJS110', 'Unresolved or unsupported binding: ' + name, node) + + def expression(self, node, scope): + if isinstance(node, ast.Constant): + kind = _literal_type(node.value) + if kind is None: + self.fail('PJS100', 'Literal type is deferred in this profile.', node) + value = str(node.value) if type(node.value) is int else node.value + return self.node(node, 'constant', kind, value=value) + if isinstance(node, ast.Name): + binding = self.resolve(node.id, scope, node) + kind = self.types.get(binding.id, 'python-value') + target = binding.target + if binding.kind == 'function': + kind, target = 'callable', binding.id + elif binding.kind == 'javascript': + kind = 'javascript-type' + elif binding.kind == 'builtin': + kind = 'python-type' if binding.name in _TYPES else 'callable' + return self.node(node, 'name', kind, binding=binding.id, target=target) + if isinstance(node, ast.Tuple): + return self.node(node, 'tuple', 'python-tuple', values=tuple(self.expression(value, scope) for value in node.elts)) + if isinstance(node, ast.Attribute): + value = self.expression(node.value, scope) + target = {( 'JavaScript', 'Array'): 'JavaScript.Array', + ('JavaScript.Array', 'of'): 'JavaScript.Array.of'}.get((value.field('target'), node.attr)) + if target: + return self.node(node, 'attribute', 'javascript-callable' if target.endswith('.of') else 'javascript-type', + value=value, name=node.attr, target=target) + if value.value_type == 'javascript-array' and node.attr == 'length': + return self.node(node, 'attribute', 'javascript-number', value=value, name='length') + self.fail('PJS110' if (value.field('target') or '').startswith('JavaScript') else 'PJS120', + 'Attribute is outside the approved JavaScript API.', node) + if isinstance(node, ast.Call): + if node.keywords or any(isinstance(argument, ast.Starred) for argument in node.args): + self.fail('PJS100', 'Keyword and starred calls are deferred.', node) + function = self.expression(node.func, scope) + target = function.field('target') + allowed = {'builtin:len': ('python-int', 1), 'builtin:print': ('python-none', None), + 'builtin:isinstance': ('python-bool', 2), 'builtin:TypeError': ('python-exception', 1), + 'JavaScript.Array.of': ('javascript-array', None)} + if target in allowed: + kind, count = allowed[target] + if count is not None and len(node.args) != count: + self.fail('PJS120', 'Unsupported argument count.', node) + elif target in self.bindings and self.bindings[target].kind == 'function': + kind = 'python-value' + else: + self.fail('PJS120', 'Call does not resolve to an approved direct callable.', node) + arguments = tuple(self.expression(argument, scope) for argument in node.args) + if target == 'builtin:isinstance' and arguments[1].field('target') not in {'builtin:' + name for name in _TYPES}: + self.fail('PJS120', 'isinstance requires an approved builtin type binding.', node) + return self.node(node, 'call', kind, function=function, arguments=arguments, target=target) + if isinstance(node, ast.BinOp): + operator = _BINARY.get(type(node.op)) + if operator is None: + self.fail('PJS100', 'Binary operator is deferred.', node) + left, right = self.expression(node.left, scope), self.expression(node.right, scope) + kinds = {left.value_type, right.value_type} + if kinds <= {'python-int', 'python-bool'}: + kind = 'python-int' + elif operator == '+' and kinds == {'python-str'}: + kind = 'python-str' + elif 'python-value' in kinds and kinds <= {'python-value', 'python-int', 'python-bool', 'python-str'}: + kind = 'python-value' + else: + self.fail('PJS120', 'Operand types require a separately adopted operation.', node) + return self.node(node, 'binary', kind, operator=operator, left=left, right=right) + if isinstance(node, ast.UnaryOp): + operator = {ast.Not: 'not', ast.UAdd: '+', ast.USub: '-'}.get(type(node.op)) + if operator is None: + self.fail('PJS100', 'Unary operator is deferred.', node) + value = self.expression(node.operand, scope) + if operator != 'not' and value.value_type not in {'python-int', 'python-bool', 'python-value'}: + self.fail('PJS120', 'Unary numeric operation requires a Python integer.', node) + return self.node(node, 'unary', 'python-bool' if operator == 'not' else 'python-int', operator=operator, value=value) + if isinstance(node, ast.BoolOp): + values = tuple(self.expression(value, scope) for value in node.values) + kinds = {value.value_type for value in values} + return self.node(node, 'boolean', next(iter(kinds)) if len(kinds) == 1 else 'python-value', + operator='and' if isinstance(node.op, ast.And) else 'or', values=values) + if isinstance(node, ast.Compare): + operators = tuple(_COMPARE.get(type(operator)) for operator in node.ops) + if None in operators: + self.fail('PJS100', 'Comparison is deferred.', node) + values = tuple(self.expression(value, scope) for value in [node.left, *node.comparators]) + for index, operator in enumerate(operators): + if operator in {'is', 'is not'}: + for value in values[index:index + 2]: + if value.value_type in {'python-int', 'python-str', 'javascript-number'} or ( + value.kind == 'tuple' + ): + self.fail('PJS120', 'Literal interning identity is outside this profile.', node) + return self.node(node, 'compare', 'python-bool', operators=operators, values=values) + if isinstance(node, ast.Subscript): + if isinstance(node.slice, ast.Slice): + self.fail('PJS100', 'Slicing is deferred.', node) + value, index = self.expression(node.value, scope), self.expression(node.slice, scope) + if value.value_type not in {'python-str', 'python-tuple', 'python-value'} or index.value_type not in {'python-int', 'python-bool', 'python-value'}: + self.fail('PJS120', 'Indexing requires an adopted sequence and integer.', node) + return self.node(node, 'index', 'python-str' if value.value_type == 'python-str' else 'python-value', value=value, index=index) + self.fail('PJS100', 'Unsupported expression: ' + type(node).__name__, node) + + def statements(self, nodes, scope=None, loop=0): + return tuple(self.statement(node, scope, loop) for node in nodes) + + def statement(self, node, scope, loop): + if isinstance(node, (ast.Import, ast.ImportFrom)) and scope is None: + if node not in self.tree.body: + self.fail('PJS100', 'Imports require unconditional module bindings.', node) + names = [alias.asname or alias.name for alias in node.names] + return self.node(node, 'import', bindings=tuple(self.root[name].id for name in names)) + if isinstance(node, ast.FunctionDef): + if node not in self.tree.body or scope or node.decorator_list or node.type_params or node.returns or any(arg.annotation for arg in [*node.args.posonlyargs, *node.args.args]): + self.fail('PJS100', 'Nested, decorated, generic or annotated functions are deferred.', node) + if node.args.vararg or node.args.kwarg or node.args.kwonlyargs: + self.fail('PJS100', 'Variadic and keyword-only parameters are deferred.', node) + if not all(_immutable(default) for default in node.args.defaults): + self.fail('PJS100', 'Defaults must be immutable literals.', node) + function_scope = 'function:' + node.name + defaults = tuple(self.expression(default, None) for default in node.args.defaults) + original = self.types.copy() + body = self.statements(node.body, function_scope) + self.types = original + return self.node(node, 'function', binding=self.root[node.name].id, + parameters=tuple(self.locals[function_scope][arg.arg].id for arg in [*node.args.posonlyargs, *node.args.args]), + defaults=defaults, body=body) + if isinstance(node, ast.Assign): + if any(not isinstance(target, ast.Name) for target in node.targets): + self.fail('PJS100', 'Only simple-name assignment is supported.', node) + if scope is None and (node not in self.tree.body or not _immutable(node.value)): + self.fail('PJS100', 'Module bindings must be unconditional immutable literals.', node) + value = self.expression(node.value, scope) + targets = tuple(self.resolve(target.id, scope, target).id for target in node.targets) + for target in targets: + self.types[target] = value.value_type + return self.node(node, 'assign', targets=targets, value=value) + if isinstance(node, ast.Expr): + return self.node(node, 'expression', value=self.expression(node.value, scope)) + if isinstance(node, ast.Return): + if scope is None: + self.fail('PJS100', 'Return requires a function.', node) + return self.node(node, 'return', value=self.expression(node.value, scope) if node.value else None) + if isinstance(node, ast.If): + test = self.expression(node.test, scope) + original = self.types.copy() + body = self.statements(node.body, scope, loop) + left = self.types.copy() + self.types = original.copy() + otherwise = self.statements(node.orelse, scope, loop) + right = self.types + self.types = {key: left.get(key) if left.get(key) == right.get(key) else 'python-value' + for key in left.keys() | right.keys()} + return self.node(node, 'if', test=test, body=body, otherwise=otherwise) + if isinstance(node, ast.For): + if scope is None or node.orelse or not isinstance(node.target, ast.Name): + self.fail('PJS100', 'For requires a function, simple target and no else clause.', node) + value = self.expression(node.iter, scope) + if value.value_type not in {'python-str', 'python-tuple', 'python-value'}: + self.fail('PJS120', 'For requires an adopted iterable.', node) + target = self.resolve(node.target.id, scope, node.target).id + original = self.types.copy() + self.types[target] = 'python-str' if value.value_type == 'python-str' else 'python-value' + body = self.statements(node.body, scope, loop + 1) + self.types = {key: kind if kind == original.get(key) else 'python-value' for key, kind in (original | self.types).items()} + return self.node(node, 'for', target=target, value=value, body=body) + if isinstance(node, (ast.Break, ast.Continue)): + if not loop: + self.fail('PJS100', 'Loop control requires a loop.', node) + return self.node(node, 'break' if isinstance(node, ast.Break) else 'continue') + if isinstance(node, ast.Raise): + value = self.expression(node.exc, scope) if node.exc else None + if node.cause or value is None or value.value_type != 'python-exception': + self.fail('PJS120', 'Only explicit adopted builtin errors may be raised.', node) + return self.node(node, 'raise', value=value) + if isinstance(node, ast.Pass): + return self.node(node, 'pass') + self.fail('PJS100', 'Unsupported statement: ' + type(node).__name__, node) + + +def parse_source(source: str, *, filename: str = '') -> ModuleIR: + """Validate source and return immutable typed IR without executing target code.""" + if not isinstance(source, str) or not isinstance(filename, str): + raise TypeError('source and filename must be strings') + try: + encoded = source.encode('utf-8') + except UnicodeEncodeError: + raise CompileError(Diagnostic('PJS100', 'Source must be valid UTF-8.', SourceSpan(filename, 1, 1, 1, 1), 'parse')) from None + if len(encoded) > 1_048_576: + raise CompileError(Diagnostic('PJS100', 'Source exceeds the 1 MiB profile limit.', SourceSpan(filename, 1, 1, 1, 1), 'parse')) + frontend = _FrontEnd(source, filename) + body = frontend.statements(frontend.tree.body) + return ModuleIR(filename, hashlib.sha256(encoded).hexdigest(), + tuple(frontend.bindings[key] for key in sorted(frontend.bindings)), body) diff --git a/tests/test_compiler_frontend.py b/tests/test_compiler_frontend.py new file mode 100644 index 0000000..7fdf1a6 --- /dev/null +++ b/tests/test_compiler_frontend.py @@ -0,0 +1,100 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from JavaScript.compiler import CompileError, parse_source + + +CORPUS = Path(__file__).parent / 'fixtures' / 'compiler_contract.json' + + +class CompilerFrontEndTests(unittest.TestCase): + def test_adopted_corpus_is_accepted_or_diagnosed(self): + corpus = json.loads(CORPUS.read_text(encoding='utf-8')) + for case in corpus['cases']: + with self.subTest(case=case['id']): + self.assertEqual(parse_source(case['source']).profile, 'python-browser/1') + for case in corpus['boundaries']: + with self.subTest(boundary=case['id']): + with self.assertRaises(CompileError) as caught: + parse_source(case['source'], filename='boundary.py') + self.assertEqual(caught.exception.diagnostic.code, case['diagnostic']) + self.assertEqual(caught.exception.diagnostic.span.filename, 'boundary.py') + + def test_large_integers_determinism_and_serialization_are_lossless(self): + source = 'print(9007199254740993)' + module = parse_source(source, filename='example.py') + encoded = json.dumps(module.to_dict(), sort_keys=True) + self.assertEqual(encoded, json.dumps(parse_source(source, filename='example.py').to_dict(), sort_keys=True)) + constant = module.body[0].field('value').field('arguments')[0] + self.assertEqual(constant.value_type, 'python-int') + self.assertEqual(constant.field('value'), '9007199254740993') + exported = module.to_dict() + exported['body'][0]['span']['line'] = 99 + exported['bindings'][0]['name'] = 'changed' + self.assertEqual(module.body[0].span.line, 1) + self.assertNotEqual(module.bindings[0].name, 'changed') + + def test_aliases_resolve_by_binding_and_shadowing_is_not_intrinsic(self): + for statement in ['import JavaScript as JS', 'from JavaScript import Array as A']: + expression = 'JS.Array.of(1)' if statement.startswith('import ') else 'A.of(1)' + module = parse_source(statement + '\ndef selected():\n return ' + expression + '.length') + encoded = json.dumps(module.to_dict()) + self.assertIn('JavaScript.Array.of', encoded) + self.assertIn('javascript-array', encoded) + for source in [ + 'from JavaScript import Array as A\ndef selected(A):\n return A.of(1)', + 'def selected(len):\n return len(1)', + ]: + with self.assertRaises(CompileError) as caught: + parse_source(source) + self.assertEqual(caught.exception.diagnostic.code, 'PJS120') + own = parse_source('def len(value):\n return value + 1\nprint(len(2))') + self.assertIn('module:len', {binding.id for binding in own.bindings}) + self.assertNotIn('builtin:len', {binding.id for binding in own.bindings}) + + def test_scopes_recursion_and_branch_types_are_preserved(self): + source = ('VALUE = 4\ndef first(value):\n if value:\n item = 1\n' + ' else:\n item = "text"\n return item + VALUE\n' + 'def second(value):\n return first(value)') + module = parse_source(source) + parameters = [binding for binding in module.bindings if binding.kind == 'parameter'] + self.assertEqual(len({binding.id for binding in parameters}), 2) + self.assertEqual({binding.name for binding in parameters}, {'value'}) + parse_source('def factorial(n):\n if n:\n return n * factorial(n - 1)\n return 1') + + def test_source_locations_count_characters_after_unicode(self): + source = 'é = 1; eval("text")' + with self.assertRaises(CompileError) as caught: + parse_source(source, filename='unicode.py') + diagnostic = caught.exception.diagnostic + self.assertEqual(diagnostic.code, 'PJS110') + self.assertEqual(diagnostic.phase, 'bind') + self.assertEqual(diagnostic.span.column, source.index('eval') + 1) + self.assertEqual(diagnostic.span.line, 1) + self.assertIn('unicode.py:1:', str(caught.exception)) + + def test_unsupported_operations_and_initialization_fail_without_execution(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / 'must-not-exist' + source = f'open({str(target)!r}, "w").write("executed")' + with self.assertRaises(CompileError): + parse_source(source) + self.assertFalse(target.exists()) + for source in [ + 'values = []', 'value = len("x")', 'def f(value=list()):\n return value', + 'def f():\n while True:\n pass', 'def f():\n return 1.5', + 'def f():\n return "text" + 1', 'def f():\n return missing', + 'def f():\n return (1, 2)[:1]', 'def f():\n import os', + 'print(1 is 1)', 'print((1, 2) is (1, 2))', + 'if False:\n import os', 'if True:\n def nested():\n pass', + ]: + with self.subTest(source=source), self.assertRaises(CompileError): + parse_source(source) + with self.assertRaises(CompileError): + parse_source('def invalid(') + with self.assertRaises(CompileError): + parse_source('\ud800') + with self.assertRaises(CompileError): + parse_source('#' * 1_048_577) From 6cf391bbaf56878b4393509ce7bb154b8117b365 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:22:58 -0700 Subject: [PATCH 2/2] feat(EMIT-01): emit deterministic Python ES modules --- IMPLEMENTATION_PLAN.md | 5 + README.md | 4 + docs/compiler-emitter.md | 91 +++++++++ docs/compiler-language.md | 7 +- pyproject.toml | 1 + src/JavaScript/_compiler_runtime.py | 271 ++++++++++++++++++++++++++ src/JavaScript/emitter.py | 254 ++++++++++++++++++++++++ tests/fixtures/compiler_contract.json | 5 + tests/test_compiler_emitter.py | 163 ++++++++++++++++ 9 files changed, 797 insertions(+), 4 deletions(-) create mode 100644 docs/compiler-emitter.md create mode 100644 src/JavaScript/_compiler_runtime.py create mode 100644 src/JavaScript/emitter.py create mode 100644 tests/test_compiler_emitter.py diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 73ec1ed..7731846 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -434,3 +434,8 @@ and publishes no package. - [Fetch standard](https://fetch.spec.whatwg.org/) - [Python AST](https://docs.python.org/3/library/ast.html) - [PyPI trusted publishing](https://docs.pypi.org/trusted-publishers/) + +The development checkout now provides the [ES module emitter](docs/compiler-emitter.md) +for the approved single-module subset, with selected helpers, source maps and a +versioned artifact manifest. MOD-01 and browser profile integration follow their +explicit dependencies. diff --git a/README.md b/README.md index b21f0f0..c74b7d0 100644 --- a/README.md +++ b/README.md @@ -104,3 +104,7 @@ Node.js 24 runs the native ECMAScript differential fixture. That test is skipped when Node is absent locally; CI and release builds require it. Runtime code needs only Python and the two declared companions. Tests make no external network requests. + +The development [compiler emitter](docs/compiler-emitter.md) produces deterministic +ES modules for the approved single-module Python profile. This API is not part +of the published 0.1.0 distribution. diff --git a/docs/compiler-emitter.md b/docs/compiler-emitter.md new file mode 100644 index 0000000..a2a995d --- /dev/null +++ b/docs/compiler-emitter.md @@ -0,0 +1,91 @@ +# Experimental ES module emitter + +The development checkout exposes `JavaScript.emitter.compile_source`. It +validates source through the python-browser/1 front end, lowers the immutable IR +and returns a CompiledArtifact without executing input or writing files. The +published 0.1.0 package predates this compiler implementation. + +```python +from JavaScript.emitter import compile_source + +artifact = compile_source( + "def count(text):\n return len(text)", + filename="counter.py", +) +print(artifact.manifest["application_exports"]) # ['count'] +``` + +`files` is a tuple of immutable OutputFile(path, data, role) values. The caller +can inspect or pass those bytes to the future pocket-python application builder. +The manifest bytes are separate from their own file/hash closure and can be +saved as PYTHON-JAVASCRIPT.json. Every recorded file has exact size, SHA-256 and +role; imports, notices and selected helper dependencies are explicit. Current +single-module output has no static imports. The compiler ABI is +python-browser-esm/1, independently of package and application-profile versions. + +The emitted entrypoint is app.mjs by default, with a named `createModule` export: + +```javascript +import {createModule} from './app.mjs'; +const app = createModule({write: text => console.log(text)}); +console.log(app.count('A😀')); // 2n +``` + +Each factory call initializes a fresh module in Python source order. Top-level +print/call statements execute at that time. Functions are assigned when their +definition executes; immutable literal defaults are created once per definition, +including tuple identity across calls. Function-local unbound values and global +use before definition raise source-located Python errors. Imports/aliases resolve +through the validated JavaScript registry; only Array.of/length are adopted. +The default output sink uses console.log with the print newline removed. + +External function arguments accept BigInt, bool, null and native JavaScript +strings. Number is rejected to avoid ambiguous rounded integers. An explicit +`{type:'str', codepoints:[...]}` preserves Python surrogate/code-point distinctions; +`{type:'tuple', items:[...]}` constructs a Python tuple. Cycles, arbitrary host +objects and input depth over 64 fail explicitly. Native string input iterates +Unicode code points at the boundary. Exported strings/tuples retain their frozen +str/codepoints or tuple/items wrappers; they are not JavaScript arrays. The host +protocol adapter will serialize these values separately. + +Python integers use BigInt, including signed floor division/modulo and numeric +bool behavior. Strings retain code points, negative indexing, membership and +lexicographic ordering. Tuples retain truthiness, iteration, equality/order and +alias identity. Chained comparisons and boolean operators evaluate each operand +once, in order, with short-circuiting and original operand results. + +Print formats supported values using Python spelling and tuple/string repr. +The compiler generates a compact nonprintable-range table from its CPython +Unicode database and records that database version, so a newer JavaScript +Unicode database cannot silently change repr. This is build-time data, not +runtime Python delegation. Raw surrogate text output requires a separately +adopted sink and fails explicitly; escaped surrogate tuple repr is supported. +Integer/string interning identity remains outside the profile. Dynamic values +outside an adopted operation fail with a source-located TypeError rather than +coercing to native JavaScript behavior. + +Compiled calls use an explicit depth limit of 128 and report RecursionError +before depending on host stack exhaustion. This is a profile resource limit, +not CPython's configurable recursion limit. Arbitrary compiled applications are +not instruction/memory sandboxed; the independent VM has its own budget model. +Unsupported syntax/imports fail compilation before an artifact is returned. +Ordinary error classes are preserved; argument-count wording is profile-specific. + +Source maps use version 3 and original logical filename, lines/columns and +sourcesContent. Runtime errors also carry original location records. Pass +source_map=False to omit map bytes and their sourceMappingURL together. Use a +logical filename when artifacts are intended for distribution; maps intentionally +include the selected source. module_name must be a portable .mjs basename. + +Only selected helpers and their declared dependency closure enter the output. +NOTICES.txt records original helper provenance; no third-party interpreter or +runtime is embedded. The repository has no declared code license; this notice +does not choose or grant one. Redistribution licensing remains a release decision. + +Run `python -m unittest discover -s tests -p 'test_compiler*.py' -v` with the +installed development checkout and Node 24.19.0. Tests compare CPython and emitted +Node output/errors/evaluation order, the single-module text_metrics application, +precision/Unicode/tuple/default behavior and an independent Node source-map +consumer. Local modules, async browser capabilities and the final browser parity +gate remain MOD-01, ASYNC-01/BROWSER-01 and QA-03. This compiler executes only +ahead-of-time approved source; it does not parse new source in the browser. diff --git a/docs/compiler-language.md b/docs/compiler-language.md index 91f42d6..1c09c6d 100644 --- a/docs/compiler-language.md +++ b/docs/compiler-language.md @@ -1,10 +1,9 @@ # Compiler source language contract PARSE-01 adopts `python-browser/1` as the initial language profile, using Python -3.14 grammar and semantics for ordinary Python constructs. This contract is -future compiler work; the released package still provides Python-side -JavaScript compatibility objects. The reference corpus checks Python behavior, -not successful translation. +3.14 grammar and semantics for ordinary Python constructs. The development checkout implements its front end and +[ES module emitter](compiler-emitter.md). The released 0.1.0 package predates +these APIs and provides Python-side JavaScript compatibility objects. ## First application and accepted source diff --git a/pyproject.toml b/pyproject.toml index 8d7461b..7bd129b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ source-include = [ "tests/fixtures/conformance.mjs", "tests/fixtures/compiler_contract.json", "docs/compatibility.md", + "docs/compiler*.md", "scripts/install_distribution.py", "CHANGELOG.md", "RELEASING.md", diff --git a/src/JavaScript/_compiler_runtime.py b/src/JavaScript/_compiler_runtime.py new file mode 100644 index 0000000..1d4dbb9 --- /dev/null +++ b/src/JavaScript/_compiler_runtime.py @@ -0,0 +1,271 @@ +"""Original, dependency-selected ECMAScript helpers for python-browser/1.""" + +from functools import lru_cache +import json + + +HELPERS = { + 'error': ((), r''' +class $PyError extends Error { + constructor(name, message, location) { super(message); this.name = name; this.location = location; } +} +function $fail(name, message, location) { throw new $PyError(name, message, location); } +'''), + 'binding': (('error',), r''' +const $UNBOUND = Symbol('unbound'); +function $get(value, name, local, location) { + if (value === $UNBOUND) $fail(local ? 'UnboundLocalError' : 'NameError', + local ? `cannot access local variable '${name}' where it is not associated with a value` : `name '${name}' is not defined`, location); + return value; +} +'''), + 'str': ((), r''' +class $Str { + constructor(codepoints) { this.type = 'str'; this.codepoints = Object.freeze(codepoints); Object.freeze(this); } +} +function $str(points) { return new $Str(points); } +function $text(value) { return value.codepoints.map(point => String.fromCodePoint(point)).join(''); } +'''), + 'tuple': ((), r''' +class $Tuple { + constructor(items) { this.type = 'tuple'; this.items = Object.freeze(items); Object.freeze(this); } +} +const $EMPTY = new $Tuple([]); +function $tuple(items) { return items.length ? new $Tuple(items) : $EMPTY; } +'''), + 'numeric': (('error',), r''' +function $number(value, location) { + if (typeof value === 'bigint') return value; + if (typeof value === 'boolean') return value ? 1n : 0n; + $fail('TypeError', 'Operation requires an adopted Python integer.', location); +} +'''), + 'truth': (('error', 'str', 'tuple'), r''' +function $truth(value, location) { + if (value === null || typeof value === 'boolean' || typeof value === 'bigint') return Boolean(value); + if (value instanceof $Str) return value.codepoints.length !== 0; + if (value instanceof $Tuple) return value.items.length !== 0; + $fail('TypeError', 'Truth testing this value is outside python-browser/1.', location); +} +'''), + 'binary': (('numeric', 'str'), r''' +function $binary(operator, left, right, location) { + if (operator === '+' && left instanceof $Str && right instanceof $Str) return $str([...left.codepoints, ...right.codepoints]); + left = $number(left, location); right = $number(right, location); + if (operator === '+') return left + right; + if (operator === '-') return left - right; + if (operator === '*') return left * right; + if (right === 0n) $fail('ZeroDivisionError', operator === '//' ? 'division by zero' : 'integer modulo by zero', location); + let quotient = left / right; + if (left % right !== 0n && (left < 0n) !== (right < 0n)) quotient -= 1n; + return operator === '//' ? quotient : left - quotient * right; +} +'''), + 'unary': (('numeric', 'truth'), r''' +function $unary(operator, value, location) { + if (operator === 'not') return !$truth(value, location); + value = $number(value, location); return operator === '-' ? -value : value; +} +'''), + 'equal': (('numeric', 'str', 'tuple'), r''' +function $equal(left, right, location) { + if (left === right) return true; + if (['bigint', 'boolean'].includes(typeof left) && ['bigint', 'boolean'].includes(typeof right)) return $number(left, location) === $number(right, location); + if (left instanceof $Str && right instanceof $Str) return left.codepoints.length === right.codepoints.length && left.codepoints.every((point, index) => point === right.codepoints[index]); + if (left instanceof $Tuple && right instanceof $Tuple) return left.items.length === right.items.length && left.items.every((value, index) => $equal(value, right.items[index], location)); + return false; +} +function $order(left, right, location) { + if (['bigint', 'boolean'].includes(typeof left) && ['bigint', 'boolean'].includes(typeof right)) { + left = $number(left, location); right = $number(right, location); return left < right ? -1 : left > right ? 1 : 0; + } + const strings = left instanceof $Str && right instanceof $Str; + const tuples = left instanceof $Tuple && right instanceof $Tuple; + if (!strings && !tuples) $fail('TypeError', 'Ordering requires matching adopted sequence or numeric values.', location); + const a = strings ? left.codepoints : left.items, b = strings ? right.codepoints : right.items; + for (let index = 0; index < Math.min(a.length, b.length); index++) { + if (strings ? a[index] !== b[index] : !$equal(a[index], b[index], location)) return strings ? (a[index] < b[index] ? -1 : 1) : $order(a[index], b[index], location); + } + return a.length < b.length ? -1 : a.length > b.length ? 1 : 0; +} +'''), + 'compare': (('equal',), r''' +function $compare(operator, left, right, location) { + if (operator === '==' || operator === '!=') return $equal(left, right, location) === (operator === '=='); + if (operator === 'is' || operator === 'is not') { + if (left !== null && right !== null && typeof left !== 'boolean' && typeof right !== 'boolean' && + (typeof left === 'bigint' || typeof right === 'bigint' || left instanceof $Str || right instanceof $Str || typeof left === 'number' || typeof right === 'number')) { + $fail('TypeError', 'Integer/string interning identity is outside python-browser/1.', location); + } + return (left === right) === (operator === 'is'); + } + if (operator === 'in' || operator === 'not in') { + let found = false; + if (right instanceof $Tuple) found = right.items.some(value => $equal(left, value, location)); + else if (right instanceof $Str && left instanceof $Str) { + outer: for (let start = 0; start <= right.codepoints.length - left.codepoints.length; start++) { + for (let offset = 0; offset < left.codepoints.length; offset++) if (left.codepoints[offset] !== right.codepoints[start + offset]) continue outer; + found = true; break; + } + } else $fail('TypeError', 'Membership requires adopted string/tuple values.', location); + return found === (operator === 'in'); + } + const order = $order(left, right, location); + return operator === '<' ? order < 0 : operator === '<=' ? order <= 0 : operator === '>' ? order > 0 : order >= 0; +} +'''), + 'index': (('numeric', 'str', 'tuple'), r''' +function $index(value, index, location) { + const string = value instanceof $Str; + if (!string && !(value instanceof $Tuple)) $fail('TypeError', 'Indexing requires a Python str or tuple.', location); + const items = string ? value.codepoints : value.items; + index = $number(index, location); if (index < 0n) index += BigInt(items.length); + if (index < 0n || index >= BigInt(items.length)) $fail('IndexError', string ? 'string index out of range' : 'tuple index out of range', location); + return string ? $str([items[Number(index)]]) : items[Number(index)]; +} +'''), + 'iterate': (('error', 'str', 'tuple'), r''' +function $iterate(value, location) { + if (value instanceof $Str) return value.codepoints.map(point => $str([point])); + if (value instanceof $Tuple) return value.items; + $fail('TypeError', 'Iteration requires a Python str or tuple.', location); +} +'''), + 'call': (('error',), r''' +class $Callable { + constructor(name, count, defaults, body) { Object.assign(this, {name, count, defaults, body}); Object.freeze(this); } +} +let $depth = 0; +function $fn(name, count, defaults, body) { return new $Callable(name, count, defaults, body); } +function $call(fn, args, location) { + if (!(fn instanceof $Callable)) $fail('TypeError', 'Value is not an adopted callable.', location); + if (fn.count !== null) { + if (args.length < fn.count - fn.defaults.length || args.length > fn.count) $fail('TypeError', `${fn.name}() received an unsupported argument count`, location); + for (let index = args.length; index < fn.count; index++) args.push(fn.defaults[index - (fn.count - fn.defaults.length)]); + } + if ($depth >= 128) $fail('RecursionError', 'Maximum compiled call depth (128) exceeded.', location); + $depth++; + try { return fn.body(args, location); } finally { $depth--; } +} +'''), + 'len': (('call', 'str', 'tuple'), r''' +const $len = $fn('len', 1, [], (args, location) => { + const value = args[0]; + if (value instanceof $Str) return BigInt(value.codepoints.length); + if (value instanceof $Tuple) return BigInt(value.items.length); + $fail('TypeError', 'len() requires an adopted Python str or tuple.', location); +}); +'''), + 'types': (('call', 'str', 'tuple'), r''' +const $types = Object.freeze({str: Symbol('str'), int: Symbol('int'), bool: Symbol('bool'), tuple: Symbol('tuple')}); +const $isinstance = $fn('isinstance', 2, [], (args, location) => { + const [value, type] = args; + if (type === $types.str) return value instanceof $Str; + if (type === $types.tuple) return value instanceof $Tuple; + if (type === $types.int) return typeof value === 'bigint' || typeof value === 'boolean'; + if (type === $types.bool) return typeof value === 'boolean'; + $fail('TypeError', 'isinstance() requires an adopted builtin type.', location); +}); +'''), + 'exception': (('call', 'str'), r''' +const $TypeError = $fn('TypeError', 1, [], (args, location) => { + if (!(args[0] instanceof $Str)) $fail('TypeError', 'TypeError() requires a str message in this profile.', location); + return new $PyError('TypeError', $text(args[0]), location); +}); +'''), + 'repr': (('error', 'str', 'tuple'), r''' +const $nonprintable = __NONPRINTABLE__; +function $printable(point) { + let low = 0, high = $nonprintable.length; + while (low < high) { const mid = (low + high) >> 1; if ($nonprintable[mid][1] < point) low = mid + 1; else high = mid; } + return low === $nonprintable.length || point < $nonprintable[low][0]; +} +function $repr(value, location, quoteString = false) { + if (value === null) return 'None'; + if (typeof value === 'boolean') return value ? 'True' : 'False'; + if (typeof value === 'bigint' || typeof value === 'number') return value.toString(); + if (value instanceof $Str) { + if (!quoteString) { + if (value.codepoints.some(point => point >= 0xd800 && point <= 0xdfff)) $fail('TypeError', 'Surrogate text output requires a separately adopted sink.', location); + return $text(value); + } + const quote = !value.codepoints.includes(39) || value.codepoints.includes(34) ? "'" : '"'; + return quote + value.codepoints.map(point => { + if (point === 92 || point === quote.codePointAt(0)) return '\\' + String.fromCodePoint(point); + if (point === 9) return '\\t'; if (point === 10) return '\\n'; if (point === 13) return '\\r'; + if (!$printable(point)) return (point < 256 ? '\\x' : point < 65536 ? '\\u' : '\\U') + point.toString(16).padStart(point < 256 ? 2 : point < 65536 ? 4 : 8, '0'); + return String.fromCodePoint(point); + }).join('') + quote; + } + if (value instanceof $Tuple) return '(' + value.items.map(item => $repr(item, location, true)).join(', ') + (value.items.length === 1 ? ',' : '') + ')'; + if (value instanceof $PyError) return value.message; + $fail('TypeError', 'Printing this value is outside python-browser/1.', location); +} +'''), + 'print': (('call', 'repr'), r''' +const $print = $fn('print', null, [], (args, location) => { write(args.map(value => $repr(value, location)).join(' ') + '\n'); return null; }); +'''), + 'javascript': (('call',), r''' +class $JSArray { constructor(values) { this.values = values; } } +const $of = $fn('Array.of', null, [], args => new $JSArray(args)); +const $JArray = Object.freeze({of: $of}); +const $JS = Object.freeze({Array: $JArray}); +function $attribute(value, name, location) { + if (value === $JS && name === 'Array') return $JArray; + if (value === $JArray && name === 'of') return $of; + if (value instanceof $JSArray && name === 'length') return value.values.length; + $fail('TypeError', 'Attribute is outside the approved JavaScript registry.', location); +} +'''), + 'input': (('call', 'str', 'tuple'), r''' +function $input(value, location, depth = 0, seen = new Set()) { + if (depth > 64) $fail('TypeError', 'Input nesting exceeds 64.', location); + if (value === null || ['boolean', 'bigint'].includes(typeof value) || value instanceof $Str || value instanceof $Tuple) return value; + if (typeof value === 'string') return $str(Array.from(value, character => character.codePointAt(0))); + if (typeof value !== 'object' || seen.has(value)) $fail('TypeError', 'Input requires an adopted Python value.', location); + if (value.type === 'str' && Array.isArray(value.codepoints) && value.codepoints.length <= 1048576 && Array.from(value.codepoints).every(point => Number.isInteger(point) && point >= 0 && point <= 0x10ffff)) return $str([...value.codepoints]); + if (value.type === 'tuple' && Array.isArray(value.items)) { + seen.add(value); const items = Array.from(value.items, item => $input(item, location, depth + 1, seen)); seen.delete(value); return $tuple(items); + } + $fail('TypeError', 'Use BigInt, string, bool, null or explicit str/tuple input; JavaScript Number/Array is not Python int/tuple.', location); +} +function $expose(value, location) { + return value instanceof $Callable ? (...args) => $call(value, args.map(argument => $input(argument, location)), location) : value; +} +'''), +} + + +@lru_cache(maxsize=1) +def _nonprintable_ranges(): + ranges, start = [], None + for point in range(0x110000): + if not chr(point).isprintable(): + if start is None: + start = point + elif start is not None: + ranges.append([start, point - 1]) + start = None + if start is not None: + ranges.append([start, 0x10ffff]) + return json.dumps(ranges, separators=(',', ':')) + + +def helper_closure(required): + ordered = [] + def visit(name): + if name in ordered: + return + for dependency in HELPERS[name][0]: + visit(dependency) + ordered.append(name) + for name in sorted(required): + visit(name) + return tuple(ordered) + + +def helper_source(name): + source = HELPERS[name][1].strip() + if name == 'repr': + source = source.replace('__NONPRINTABLE__', _nonprintable_ranges()) + return source diff --git a/src/JavaScript/emitter.py b/src/JavaScript/emitter.py new file mode 100644 index 0000000..db59905 --- /dev/null +++ b/src/JavaScript/emitter.py @@ -0,0 +1,254 @@ +"""Deterministic ES modules for the validated python-browser/1 source profile.""" + +from dataclasses import dataclass +import hashlib +from importlib.metadata import version +import json +import re +import unicodedata + +from .compiler import parse_source +from ._compiler_runtime import helper_closure, helper_source + + +def _json(value): + return json.dumps(value, ensure_ascii=True, separators=(',', ':')) + + +@dataclass(frozen=True) +class OutputFile: + path: str + data: bytes + role: str + + +@dataclass(frozen=True) +class CompiledArtifact: + files: tuple[OutputFile, ...] + manifest_bytes: bytes + + @property + def manifest(self): + """Return a fresh snapshot; callers cannot mutate artifact metadata.""" + return json.loads(self.manifest_bytes) + + +def _vlq(value): + alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + value = (-value << 1 | 1) if value < 0 else value << 1 + encoded = '' + while True: + digit = value & 31 + value >>= 5 + encoded += alphabet[digit | (32 if value else 0)] + if not value: + return encoded + + +class _Emitter: + def __init__(self, module): + self.module = module + self.bindings = {binding.id: binding for binding in module.bindings} + self.names = {binding.id: '$v' + str(index) for index, binding in enumerate(module.bindings)} + self.required = set() + self.lines = [] + self.counter = 0 + + def use(self, helper): + self.required.add(helper) + + def temporary(self): + self.counter += 1 + return '$t' + str(self.counter) + + def location(self, node): + return _json(vars(node.span)) + + def add(self, text, indent=1, node=None): + self.lines.append((' ' * indent + text, node.span if node else None)) + + def expression(self, node): + fields, location = dict(node.fields), self.location(node) + kind = node.kind + if kind == 'constant': + if node.value_type == 'python-int': + return fields['value'] + 'n' + if node.value_type == 'python-str': + self.use('str') + return '$str(' + _json([ord(character) for character in fields['value']]) + ')' + return _json(fields['value']) + if kind == 'tuple': + self.use('tuple') + return '$tuple([' + ','.join(map(self.expression, fields['values'])) + '])' + if kind == 'name': + binding = self.bindings[fields['binding']] + if binding.kind == 'builtin': + name = binding.name + if name in {'str', 'int', 'bool', 'tuple'}: + self.use('types') + return '$types.' + name + helper, value = {'len': ('len', '$len'), 'print': ('print', '$print'), + 'isinstance': ('types', '$isinstance'), 'TypeError': ('exception', '$TypeError')}[name] + self.use(helper) + return value + self.use('binding') + return f'$get({self.names[binding.id]},{_json(binding.name)},{str(binding.kind in {"local", "parameter"}).lower()},{location})' + if kind == 'attribute': + self.use('javascript') + return f'$attribute({self.expression(fields["value"])},{_json(fields["name"])},{location})' + if kind == 'call': + self.use('call') + return f'$call({self.expression(fields["function"])},[{",".join(map(self.expression, fields["arguments"]))}],{location})' + if kind in {'binary', 'unary', 'index'}: + self.use(kind) + if kind == 'binary': + values = [_json(fields['operator']), self.expression(fields['left']), self.expression(fields['right'])] + elif kind == 'unary': + values = [_json(fields['operator']), self.expression(fields['value'])] + else: + values = [self.expression(fields['value']), self.expression(fields['index'])] + return '$' + kind + '(' + ','.join([*values, location]) + ')' + if kind == 'boolean': + self.use('truth') + name = self.temporary() + values = fields['values'] + pieces = [f'let {name}={self.expression(values[0])};'] + for value in values[1:]: + condition = f'$truth({name},{location})' + if fields['operator'] == 'and': + condition = '!' + condition + pieces.extend([f'if({condition})return {name};', f'{name}={self.expression(value)};']) + return '(()=>{' + ''.join(pieces) + f'return {name};' + '})()' + if kind == 'compare': + self.use('compare') + left, right = self.temporary(), self.temporary() + pieces = [f'let {left}={self.expression(fields["values"][0])};let {right};'] + for operator, value in zip(fields['operators'], fields['values'][1:]): + pieces.extend([f'{right}={self.expression(value)};', + f'if(!$compare({_json(operator)},{left},{right},{location}))return false;', f'{left}={right};']) + return '(()=>{' + ''.join(pieces) + 'return true;})()' + raise AssertionError('Unrecognized validated expression: ' + kind) + + def statements(self, nodes, indent=1): + for node in nodes: + self.statement(node, indent) + + def statement(self, node, indent): + fields, kind = dict(node.fields), node.kind + if kind == 'import': + self.use('javascript') + for identifier in fields['bindings']: + value = '$JS' if self.bindings[identifier].target == 'JavaScript' else '$JArray' + self.add(f'{self.names[identifier]}={value};', indent, node) + elif kind == 'function': + self.use('call') + binding = self.bindings[fields['binding']] + defaults = '[' + ','.join(map(self.expression, fields['defaults'])) + ']' + self.add(f'{self.names[binding.id]}=$fn({_json(binding.name)},{len(fields["parameters"])},{defaults},($args)=>{{', indent, node) + locals_ = [item for item in self.module.bindings if item.scope == 'function:' + binding.name] + if locals_: + self.use('binding') + self.add('let ' + ','.join(self.names[item.id] + '=$UNBOUND' for item in locals_) + ';', indent + 1) + for index, parameter in enumerate(fields['parameters']): + self.add(f'{self.names[parameter]}=$args[{index}];', indent + 1) + self.statements(fields['body'], indent + 1) + self.add('return null;', indent + 1) + self.add('});', indent) + elif kind == 'assign': + temporary = self.temporary() + self.add(f'const {temporary}={self.expression(fields["value"])};', indent, node) + for target in fields['targets']: + self.add(f'{self.names[target]}={temporary};', indent, node) + elif kind in {'expression', 'return', 'raise'}: + value = self.expression(fields['value']) if fields['value'] else 'null' + prefix = {'expression': '', 'return': 'return ', 'raise': 'throw '}[kind] + self.add(prefix + value + ';', indent, node) + elif kind == 'if': + self.use('truth') + self.add(f'if($truth({self.expression(fields["test"])},{self.location(node)})){{', indent, node) + self.statements(fields['body'], indent + 1) + if fields['otherwise']: + self.add('}else{', indent) + self.statements(fields['otherwise'], indent + 1) + self.add('}', indent) + elif kind == 'for': + self.use('iterate') + temporary = self.temporary() + self.add(f'for(const {temporary} of $iterate({self.expression(fields["value"])},{self.location(node)})){{', indent, node) + self.add(f'{self.names[fields["target"]]}={temporary};', indent + 1) + self.statements(fields['body'], indent + 1) + self.add('}', indent) + elif kind in {'break', 'continue'}: + self.add(kind + ';', indent, node) + elif kind == 'pass': + self.add(';', indent, node) + else: + raise AssertionError('Unrecognized validated statement: ' + kind) + + def render(self, source, module_name, source_map): + roots = [binding for binding in self.module.bindings if binding.scope == 'module'] + if roots: + self.use('binding') + self.add('let ' + ','.join(self.names[binding.id] + '=$UNBOUND' for binding in roots) + ';') + self.statements(self.module.body) + exports = [binding for binding in roots if binding.kind != 'javascript'] + items = [] + for binding in exports: + value = self.names[binding.id] + if binding.kind == 'function': + self.use('input') + definition = next(node for node in self.module.body if node.kind == 'function' and node.field('binding') == binding.id) + value = f'$expose({value},{self.location(definition)})' + items.append(f'[{_json(binding.name)}]:{value}') + self.add('return Object.freeze({' + ','.join(items) + '});') + helpers = helper_closure(self.required) + lines = [('export function createModule({write = text => console.log(text.slice(0,-1))} = {}) {', None), + (" if(typeof write !== 'function')throw new TypeError('write must be a function');", None)] + for helper in helpers: + lines.extend((' ' + line, None) for line in helper_source(helper).splitlines()) + lines.extend(self.lines) + lines.append(('}', None)) + mappings, previous_line, previous_column = [], 0, 0 + for text, span in lines: + if span: + column = len(text) - len(text.lstrip()) + mappings.append(''.join(_vlq(value) for value in [column, 0, span.line - 1 - previous_line, span.column - 1 - previous_column])) + previous_line, previous_column = span.line - 1, span.column - 1 + else: + mappings.append('') + code = '\n'.join(text for text, _ in lines) + '\n' + output = [OutputFile(module_name, code.encode('utf-8'), 'executable-module')] + if source_map: + code += '//# sourceMappingURL=' + module_name + '.map\n' + output[0] = OutputFile(module_name, code.encode('utf-8'), 'executable-module') + data = {'version': 3, 'file': module_name, 'sources': [self.module.filename], + 'sourcesContent': [source], 'names': [], 'mappings': ';'.join(mappings)} + output.append(OutputFile(module_name + '.map', (_json(data) + '\n').encode('utf-8'), 'source-map')) + return output, helpers, [binding.name for binding in exports] + + +def compile_source(source: str, *, filename: str = 'app.py', module_name: str = 'app.mjs', + source_map: bool = True) -> CompiledArtifact: + """Compile one approved module to immutable artifacts; do not execute or publish it.""" + if not isinstance(module_name, str) or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_-]*\.mjs', module_name): + raise ValueError('module_name must be a portable .mjs basename') + if type(source_map) is not bool: + raise TypeError('source_map must be bool') + module = parse_source(source, filename=filename) + output, helpers, exports = _Emitter(module).render(source, module_name, source_map) + compiler_version = version('Python-JavaScript') + notice = ('Generated by Python-JavaScript ' + compiler_version + '\n' + 'https://github.com/outside-labs/Python-JavaScript\n' + 'Original compiler helper implementation; no third-party runtime is embedded.\n' + 'This provenance notice grants no additional license rights.\n') + output.append(OutputFile('NOTICES.txt', notice.encode('utf-8'), 'notice')) + files = tuple(sorted(output, key=lambda item: item.path)) + manifest = {'kind': 'python-javascript-compiled', 'schema_version': 1, + 'compiler': {'name': 'Python-JavaScript', 'version': compiler_version, 'abi': 'python-browser-esm/1'}, + 'language_profile': module.profile, 'target': 'ecmascript-2022-module', + 'unicode_version': unicodedata.unidata_version, 'entrypoint': module_name, + 'entrypoint_export': 'createModule', 'application_exports': exports, + 'sources': [{'path': filename, 'sha256': module.source_sha256, 'size': len(source.encode('utf-8'))}], + 'files': [{'path': item.path, 'size': len(item.data), 'sha256': hashlib.sha256(item.data).hexdigest(), 'role': item.role} for item in files], + 'imports': [], 'helpers': list(helpers), 'notices': ['NOTICES.txt']} + return CompiledArtifact(files, (_json(manifest) + '\n').encode('utf-8')) diff --git a/tests/fixtures/compiler_contract.json b/tests/fixtures/compiler_contract.json index 01491c4..133e1e4 100644 --- a/tests/fixtures/compiler_contract.json +++ b/tests/fixtures/compiler_contract.json @@ -37,6 +37,11 @@ "id": "alias", "source": "from JavaScript import Array as A\nprint(A.of(1, 2).length)", "stdout": "2\n" + }, + { + "id": "text-metrics-single-module", + "source": "def character_count(text):\n return len(text)\n\n\n\nSEPARATORS = \" \\t\\n\\r\\v\\f\"\n\n\ndef metrics(text):\n if not isinstance(text, str):\n raise TypeError(\"text must be str\")\n characters = character_count(text)\n words = 0\n lines = 0\n inside_word = False\n if text:\n lines = 1\n for character in text:\n if character == \"\\n\":\n lines = lines + 1\n if character in SEPARATORS:\n inside_word = False\n elif not inside_word:\n words = words + 1\n inside_word = True\n return characters, words, lines\n\nprint(metrics(\"\"))\nprint(metrics(\"A😀 é\\nline two\"))\n", + "stdout": "(0, 0, 0)\n(14, 4, 2)\n" } ], "boundaries": [ diff --git a/tests/test_compiler_emitter.py b/tests/test_compiler_emitter.py new file mode 100644 index 0000000..f1e9ae9 --- /dev/null +++ b/tests/test_compiler_emitter.py @@ -0,0 +1,163 @@ +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from JavaScript.compiler import CompileError +from JavaScript.emitter import compile_source + + +CORPUS = Path(__file__).parent / 'fixtures' / 'compiler_contract.json' + + +def run_node(artifact, invocation='createModule({write: text => stdout += text});'): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for file in artifact.files: + (root / file.path).write_bytes(file.data) + runner = ("import {createModule} from './app.mjs'; let stdout=''; let error=null;\n" + "try { " + invocation + " } catch (caught) { error={name:caught.name,message:caught.message,location:caught.location}; }\n" + "console.log(JSON.stringify({stdout,error}));") + (root / 'run.mjs').write_text(runner, encoding='utf-8') + process = subprocess.run(['node', str(root / 'run.mjs')], check=True, + capture_output=True, text=True, encoding='utf-8', timeout=10) + return json.loads(process.stdout) + + +class CompilerEmitterTests(unittest.TestCase): + def assert_parity(self, source): + oracle = subprocess.run([sys.executable, '-I', '-X', 'utf8', '-c', source], + capture_output=True, text=True, encoding='utf-8', timeout=5) + result = run_node(compile_source(source, filename='case.py')) + self.assertEqual(result['stdout'], oracle.stdout.replace('\r\n', '\n')) + if oracle.returncode: + self.assertIsNotNone(result['error']) + self.assertEqual(result['error']['name'], oracle.stderr.splitlines()[-1].split(':', 1)[0]) + self.assertEqual(result['error']['location']['filename'], 'case.py') + else: + self.assertIsNone(result['error'], result['error']) + return result + + def test_corpus_executes_as_emitted_modules(self): + corpus = json.loads(CORPUS.read_text(encoding='utf-8')) + for case in corpus['cases']: + with self.subTest(case=case['id']): + result = self.assert_parity(case['source']) + self.assertEqual(result['stdout'], case['stdout']) + for case in corpus['boundaries']: + with self.subTest(boundary=case['id']), self.assertRaises(CompileError): + compile_source(case['source']) + + def test_single_evaluation_short_circuit_order_and_loop_control(self): + self.assert_parity('''def mark(value): + print(value) + return value +def selected(): + total = 0 + for value in (1, 2, 3, 4): + if value == 2: + continue + if value == 4: + break + total = total + value + return total +print(mark(1) < mark(2) < mark(3)) +print(mark(3) < mark(2) < mark(1)) +print(mark('') and mark('never')) +print(mark(()) or mark('fallback')) +print(selected())''') + + def test_scalar_tuple_unicode_and_registry_semantics(self): + self.assert_parity('''import JavaScript as JS +def selected(): + original = (1,) + alias = original + return original is alias, () == (), not (), True + 2, '😀'[-1], '😀' in 'A😀', 2 in (1, 2) +print(JS.Array.of(1, 2).length) +print(selected()) +print(('a\\nb', "a'b", '\\x08', '\\u00a0', '\\ud800')) +print(('\\U0001fae9', '\\u0301')) +print(len('\\ud83d\\ude00'), len('😀'), '\\ud83d\\ude00' == '😀') +print(-7 // 3, 7 // -3, -7 % 3, 7 % -3, (1, 2) < (1, 3))''') + + def test_defaults_preserve_definition_time_identity_and_recursion(self): + self.assert_parity('''def same(value=(1, 2)): + return value +def selected(): + first = same() + second = same() + return first is second +def factorial(value): + if value: + return value * factorial(value - 1) + return 1 +print(selected(), factorial(6))''') + result = run_node(compile_source('def forever():\n return forever()\nforever()')) + self.assertEqual(result['error']['name'], 'RecursionError') + + def test_runtime_errors_keep_source_and_output_before_failure(self): + sources = [ + 'print("before")\n1 // 0', 'print("before")\n(1,)[5]', + 'def selected(value):\n if value:\n item = 1\n return item\nselected(False)', + 'print(later())\ndef later():\n return 1', + 'def selected(value):\n return value + 1\nselected()', + 'def selected(value):\n if not isinstance(value, str):\n raise TypeError("text must be str")\n return len(value)\nselected(1)', + 'A.of(1)\nfrom JavaScript import Array as A', + ] + for source in sources: + with self.subTest(source=source): + self.assert_parity(source) + error = run_node(compile_source('def selected():\n return 1 // 0\nselected()', filename='original.py'))['error'] + self.assertEqual(error['location']['filename'], 'original.py') + self.assertEqual(error['location']['line'], 2) + self.assertEqual(error['location']['column'], 12) + + def test_exports_enforce_inputs_and_independent_module_state(self): + artifact = compile_source('def size(text):\n return len(text)\ndef echo(value):\n return value') + result = run_node(artifact, "const app=createModule({write: text => stdout+=text}); stdout=app.size('A😀').toString();") + self.assertEqual(result['stdout'], '2') + result = run_node(artifact, "const app=createModule(); app.echo(9007199254740992);") + self.assertEqual(result['error']['name'], 'TypeError') + result = run_node(artifact, "const app=createModule(); const value={type:'tuple',items:[]}; value.items.push(value); app.echo(value);") + self.assertEqual(result['error']['name'], 'TypeError') + + def test_manifest_determinism_closure_hashes_and_optional_maps(self): + source = 'def selected(value):\n return value + 1' + artifact = compile_source(source) + self.assertEqual(artifact, compile_source(source)) + manifest = artifact.manifest + self.assertEqual(manifest['kind'], 'python-javascript-compiled') + self.assertEqual(manifest['imports'], []) + self.assertIn('binary', manifest['helpers']) + self.assertNotIn('javascript', manifest['helpers']) + self.assertNotIn('print', manifest['helpers']) + self.assertEqual(manifest['application_exports'], ['selected']) + for record, file in zip(manifest['files'], artifact.files): + self.assertEqual(record['path'], file.path) + self.assertEqual(record['size'], len(file.data)) + self.assertEqual(record['sha256'], hashlib.sha256(file.data).hexdigest()) + source_map = json.loads(next(file.data for file in artifact.files if file.role == 'source-map')) + self.assertEqual(source_map['sourcesContent'], [source]) + self.assertEqual(source_map['version'], 3) + self.assertTrue(any(source_map['mappings'].split(';'))) + code = next(file.data.decode() for file in artifact.files if file.role == 'executable-module') + generated_line = next(index for index, line in enumerate(code.splitlines()) if 'return $binary' in line) + consumer = "import {SourceMap} from 'node:module'; const input=JSON.parse(process.argv[1]); console.log(JSON.stringify(new SourceMap(input.map).findEntry(input.line,4)));" + result = subprocess.run(['node', '--input-type=module', '-e', consumer, + json.dumps({'map': source_map, 'line': generated_line})], + check=True, capture_output=True, text=True, timeout=5) + mapping = json.loads(result.stdout) + self.assertEqual(mapping['originalLine'], 1) + self.assertEqual(mapping['originalColumn'], 4) + self.assertEqual(mapping['originalSource'], 'app.py') + manifest['files'][0]['sha256'] = 'changed' + self.assertNotEqual(artifact.manifest['files'][0]['sha256'], 'changed') + no_map = compile_source(source, source_map=False) + self.assertNotIn('sourceMappingURL', next(file.data.decode() for file in no_map.files if file.role == 'executable-module')) + self.assertNotIn('source-map', {file.role for file in no_map.files}) + for invalid in ['../app.mjs', 'app.js', 'app.mjs\nexecute', 'é.mjs']: + with self.assertRaises(ValueError): + compile_source('pass', module_name=invalid)