From 6863850fd78bb039cb64bf83275978c7811b5cd9 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:28:22 -0700 Subject: [PATCH 1/2] feat(MOD-01): compile verified local Python modules --- IMPLEMENTATION_PLAN.md | 11 ++ README.md | 4 +- docs/compiler-language.md | 4 +- docs/compiler-modules.md | 91 ++++++++++ src/JavaScript/_compiler_runtime.py | 17 +- src/JavaScript/compiler.py | 30 +++- src/JavaScript/emitter.py | 27 ++- src/JavaScript/modules.py | 261 ++++++++++++++++++++++++++++ tests/test_compiler_modules.py | 162 +++++++++++++++++ 9 files changed, 591 insertions(+), 16 deletions(-) create mode 100644 docs/compiler-modules.md create mode 100644 src/JavaScript/modules.py create mode 100644 tests/test_compiler_modules.py diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 7731846..4e4c719 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -439,3 +439,14 @@ The development checkout now provides the [ES module emitter](docs/compiler-emit for the approved single-module subset, with selected helpers, source maps and a versioned artifact manifest. MOD-01 and browser profile integration follow their explicit dependencies. + +## Development implementation checkpoint: 2026-10-20 + +PARSE-01, PARSE-02 and EMIT-01 are merged. The development checkout implements +[verified local-module compilation](docs/compiler-modules.md) for MOD-01, with +relative imports, aliases, package re-exports, initialization order and a shared +value/callable runtime. Explicit source-import cycles and unsupported source are +rejected. Per-module maps, source hashes, static output closure and unchanged +notices are retained. Published 0.1.0 artifacts remain unchanged. GitHub Issues +and Project fields record live delivery status; the forward table records the +original adopted dependency order. ASYNC-01 and BROWSER-01 follow this foundation. diff --git a/README.md b/README.md index c74b7d0..c048f20 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ the [issues](https://github.com/outside-labs/Python-JavaScript/issues) and [Project](https://github.com/orgs/outside-labs/projects/11). The development checkout now includes a [binding-aware compiler front end](docs/compiler-front-end.md) -that validates source and produces typed IR. JavaScript emission and browser -execution remain subsequent milestones. +that validates source and produces typed IR. The emitter and [local-module compiler](docs/compiler-modules.md) +produce executable ES modules; browser adapters remain a subsequent milestone. ## Development diff --git a/docs/compiler-language.md b/docs/compiler-language.md index 1c09c6d..cbe5759 100644 --- a/docs/compiler-language.md +++ b/docs/compiler-language.md @@ -9,7 +9,7 @@ these APIs and provides Python-side JavaScript compatibility objects. Start with a single-module equivalent of Shimify's text_metrics application: Unicode character count, explicitly selected ASCII word separators, LF-based -line count and a tuple of Python integer results. MOD-01 later adds its local +line count and a tuple of Python integer results. The [local-module profile](compiler-modules.md) adds its helper import and package re-export. The selected input domain is builtin str; non-string input raises TypeError("text must be str"). String subclass overrides and arbitrary user objects are outside the first profile. @@ -23,7 +23,7 @@ and arbitrary user objects are outside the first profile. | Operations | Supported scalar arithmetic/comparisons, Python and/or/not, identity on supported values, string membership and supported indexing. Mixed or unknown operand forms need helpers or a source-located rejection. | | Builtins/errors | len on adopted strings/tuples, print of adopted values, approved isinstance checks and explicit builtin TypeError raising needed by the fixture. No eval/exec, reflection or undeclared I/O. | | JavaScript API | Resolve explicit imports/aliases through a versioned symbol/method registry. Start with Array.of and length as a binding fixture; other exports/methods need adopted signatures and parity cases before lowering. | -| Imports | Only approved JavaScript registry imports in the single-module profile. Local modules follow MOD-01; Python stdlib/third-party/native imports require explicit future adapters. | +| Imports | Only approved JavaScript registry imports in the single-module profile. Named local imports use the separate modules profile; Python stdlib/third-party/native imports require explicit future adapters. | Every AST form and operation is supported, rejected or deferred explicitly. A Python-side JavaScript API existing today does not automatically make it diff --git a/docs/compiler-modules.md b/docs/compiler-modules.md new file mode 100644 index 0000000..bbaabc7 --- /dev/null +++ b/docs/compiler-modules.md @@ -0,0 +1,91 @@ +# Compiling local source modules + +The development checkout provides `JavaScript.modules.compile_modules`. The +published 0.1.0 package predates this API. `python-browser-modules/1` extends the +[single-module language](compiler-language.md) with explicit named source imports. + +```python +from pathlib import Path +from JavaScript.modules import SourceModule, compile_modules + +sources = ( + SourceModule.from_text("app", "from helpers import size\nprint(size('A😀'))"), + SourceModule.from_text("helpers", "def size(text):\n return len(text)"), +) +artifact = compile_modules(sources, entrypoint="app") +destination = Path("compiled") +for output in artifact.files: + path = destination / output.path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(output.data) +(destination / "manifest.json").write_bytes(artifact.manifest_bytes) +``` + +Load `createModule` from the generated `app.mjs` and call it to initialize a new +application context. Its `write` option receives Python print output, including +newlines. The entrypoint is imported under its declared identity; it is not +executed a second time as `__main__`. + +## Inputs and imports + +Supply an immutable tuple of `SourceModule` records. Each record contains its +qualified module name, exact immutable UTF-8 bytes, SHA-256, package flag and +logical relative `.py` filename. Direct construction validates the supplied +hash; `from_text` computes it. Compilation rechecks records. Logical filenames +are source-map identities, not paths the compiler opens. No target module is +imported or executed during compilation. + +The allowlist contains at most 32 modules, 1 MiB per module and 8 MiB total. +Module identities use ordinary ASCII Python names and cannot shadow the stdlib +or the JavaScript registry. Module names and portable logical filenames must be +unique, including when case folded. Every parent package needs an explicitly +declared `is_package=True` initializer. Only reachable modules and their parent +initializers are parsed and emitted; supplied unused modules are omitted. + +Supported source imports are unconditional top-level named imports, including +aliases, relative imports and package re-exports: + +```python +from text_metrics import metrics as selected +from .helpers import character_count as count_characters +``` + +Exports can be supported plain functions, immutable module constants or +re-exported source bindings. JavaScript registry imports retain their separate +binding rules and cannot be re-exported through the Python source namespace. +Plain local `import module`, wildcard, conditional/function-local, dynamic, +stdlib/native-extension and undeclared imports fail with source-located +diagnostics. Explicit source-import cycles are deterministically rejected. +Required parent initializer ordering supports a package re-exporting a child; +it does not authorize general partially initialized module cycles. + +Import statements execute in source order, each module initializes once per +application context, and parents initialize before children. Reading a binding +before its defining statement still raises the adopted Python error. One shared +runtime preserves imported callable, tuple and string identities, immutable +function defaults, recursion limits and the output sink. Entry exports wrap +callables once per context for JavaScript invocation; internal imports use the +original Python values. Each `createModule` call creates independent state. + +## Artifacts and limits + +The manifest has kind `python-javascript-compiled`, schema 1, compiler ABI +`python-browser-modules-esm/1` and language profile `python-browser-modules/1`. +It records selected modules/exports, Python source import edges, per-module +source hashes and filenames, the static emitted import closure, required helper +closure, file sizes/hashes and notices. Deterministically named module factories +share `runtime.mjs`; `app.mjs` statically imports every emitted dependency. +Source maps retain each original module's text. `source_map=False` omits maps. + +Original nonempty notices may be passed as immutable `OutputFile` records with +role `notice`, under portable `LICENSES/` paths. They are copied unchanged and +hashed. The compiler's provenance notice grants no additional license rights. +No runtime dependency on Shimify is introduced. Shimify may produce the verified +source bytes used here, but unchanged supported modules also compile directly. + +The text_metrics parity fixture uses its supported metrics module and the +reduced helper containing `character_count`. The original helper's unused slice +utility remains unsupported and causes rejection if supplied unchanged. This +compiler does not perform symbol extraction or silently skip unsupported code +inside a selected module. Browser DOM/Fetch integration and runtime parsing of +new source remain separate tasks. diff --git a/src/JavaScript/_compiler_runtime.py b/src/JavaScript/_compiler_runtime.py index 1d4dbb9..9a500c2 100644 --- a/src/JavaScript/_compiler_runtime.py +++ b/src/JavaScript/_compiler_runtime.py @@ -230,12 +230,27 @@ class $JSArray { constructor(values) { this.values = values; } } $fail('TypeError', 'Use BigInt, string, bool, null or explicit str/tuple input; JavaScript Number/Array is not Python int/tuple.', location); } function $expose(value, location) { - return value instanceof $Callable ? (...args) => $call(value, args.map(argument => $input(argument, location)), location) : value; + if (!(value instanceof $Callable)) return value; + if (!$exposed.has(value)) $exposed.set(value, (...args) => $call(value, args.map(argument => $input(argument, location)), location)); + return $exposed.get(value); } +const $exposed = new WeakMap(); '''), } +HELPER_BINDINGS = { + 'error': ('$PyError', '$fail'), 'binding': ('$UNBOUND', '$get'), + 'str': ('$str',), 'tuple': ('$tuple',), 'numeric': ('$number',), + 'truth': ('$truth',), 'binary': ('$binary',), 'unary': ('$unary',), + 'equal': ('$equal', '$order'), 'compare': ('$compare',), 'index': ('$index',), + 'iterate': ('$iterate',), 'call': ('$fn', '$call'), 'len': ('$len',), + 'types': ('$types', '$isinstance'), 'exception': ('$TypeError',), + 'repr': ('$repr',), 'print': ('$print',), + 'javascript': ('$JS', '$JArray', '$attribute'), 'input': ('$input', '$expose'), +} + + @lru_cache(maxsize=1) def _nonprintable_ranges(): ranges, start = [], None diff --git a/src/JavaScript/compiler.py b/src/JavaScript/compiler.py index 5fa0d37..1173d5b 100644 --- a/src/JavaScript/compiler.py +++ b/src/JavaScript/compiler.py @@ -42,7 +42,7 @@ class Binding: id: str name: str scope: str - kind: Literal['constant', 'function', 'parameter', 'local', 'javascript', 'builtin'] + kind: Literal['constant', 'function', 'parameter', 'local', 'javascript', 'builtin', 'source'] target: str | None = None @@ -84,6 +84,14 @@ def to_dict(self): 'body': [node.to_dict() for node in self.body]} +@dataclass(frozen=True) +class SourceImport: + name: str + module: str + export: str + value_type: ValueType + + _BUILTINS = {'len', 'print', 'isinstance', 'str', 'int', 'bool', 'tuple', 'TypeError'} _TYPES = {'str', 'int', 'bool', 'tuple'} _BINARY = {ast.Add: '+', ast.Sub: '-', ast.Mult: '*', ast.FloorDiv: '//', ast.Mod: '%'} @@ -108,13 +116,14 @@ def accepted(value): class _FrontEnd: - def __init__(self, source, filename): + def __init__(self, source, filename, source_imports=None): self.filename = filename self.lines = source.splitlines() or [''] self.bindings = {} self.root = {} self.locals = {} self.types = {} + self.source_imports = source_imports or {} try: self.tree = ast.parse(source, filename=filename) self.table = symtable.symtable(source, filename, 'exec') @@ -146,6 +155,15 @@ def declare(self, name, scope, kind, target=None): def _declare_module(self): for node in self.tree.body: + imports = self.source_imports.get((node.lineno, node.col_offset)) + if imports is not None: + for imported in imports: + if imported.name in self.root: + self.fail('PJS100', 'Module rebinding is outside the immutable profile.', node) + binding = self.declare(imported.name, 'module', 'source', imported.module + ':' + imported.export) + self.root[imported.name] = binding + self.types[binding.id] = imported.value_type + continue names = [] kind, target = 'constant', None if isinstance(node, ast.FunctionDef): @@ -208,6 +226,8 @@ def expression(self, node, scope): kind = 'javascript-type' elif binding.kind == 'builtin': kind = 'python-type' if binding.name in _TYPES else 'callable' + elif binding.kind == 'source': + target = binding.id if kind == 'callable' else None return self.node(node, 'name', kind, binding=binding.id, target=target) if isinstance(node, ast.Tuple): return self.node(node, 'tuple', 'python-tuple', values=tuple(self.expression(value, scope) for value in node.elts)) @@ -234,7 +254,7 @@ def expression(self, node, scope): kind, count = allowed[target] if count is not None and len(node.args) != count: self.fail('PJS120', 'Unsupported argument count.', node) - elif target in self.bindings and self.bindings[target].kind == 'function': + elif target in self.bindings and self.bindings[target].kind in {'function', 'source'}: kind = 'python-value' else: self.fail('PJS120', 'Call does not resolve to an approved direct callable.', node) @@ -299,6 +319,10 @@ def statement(self, node, scope, loop): if isinstance(node, (ast.Import, ast.ImportFrom)) and scope is None: if node not in self.tree.body: self.fail('PJS100', 'Imports require unconditional module bindings.', node) + imports = self.source_imports.get((node.lineno, node.col_offset)) + if imports is not None: + return self.node(node, 'source-import', module=imports[0].module, + bindings=tuple((self.root[imported.name].id, imported.export) for imported in imports)) names = [alias.asname or alias.name for alias in node.names] return self.node(node, 'import', bindings=tuple(self.root[name].id for name in names)) if isinstance(node, ast.FunctionDef): diff --git a/src/JavaScript/emitter.py b/src/JavaScript/emitter.py index db59905..5f25aa2 100644 --- a/src/JavaScript/emitter.py +++ b/src/JavaScript/emitter.py @@ -8,7 +8,7 @@ import unicodedata from .compiler import parse_source -from ._compiler_runtime import helper_closure, helper_source +from ._compiler_runtime import HELPER_BINDINGS, helper_closure, helper_source def _json(value): @@ -135,7 +135,12 @@ def statements(self, nodes, indent=1): def statement(self, node, indent): fields, kind = dict(node.fields), node.kind - if kind == 'import': + if kind == 'source-import': + temporary = self.temporary() + self.add(f'const {temporary}=importModule({_json(fields["module"])});', indent, node) + for identifier, exported in fields['bindings']: + self.add(f'{self.names[identifier]}={temporary}[{_json(exported)}];', indent, node) + elif kind == 'import': self.use('javascript') for identifier in fields['bindings']: value = '$JS' if self.bindings[identifier].target == 'JavaScript' else '$JArray' @@ -185,7 +190,7 @@ def statement(self, node, indent): else: raise AssertionError('Unrecognized validated statement: ' + kind) - def render(self, source, module_name, source_map): + def render(self, source, module_name, source_map, *, shared_runtime=False): roots = [binding for binding in self.module.bindings if binding.scope == 'module'] if roots: self.use('binding') @@ -195,17 +200,23 @@ def render(self, source, module_name, source_map): items = [] for binding in exports: value = self.names[binding.id] - if binding.kind == 'function': + if binding.kind == 'function' and not shared_runtime: self.use('input') definition = next(node for node in self.module.body if node.kind == 'function' and node.field('binding') == binding.id) value = f'$expose({value},{self.location(definition)})' items.append(f'[{_json(binding.name)}]:{value}') self.add('return Object.freeze({' + ','.join(items) + '});') helpers = helper_closure(self.required) - lines = [('export function createModule({write = text => console.log(text.slice(0,-1))} = {}) {', None), - (" if(typeof write !== 'function')throw new TypeError('write must be a function');", None)] - for helper in helpers: - lines.extend((' ' + line, None) for line in helper_source(helper).splitlines()) + if shared_runtime: + names = [name for helper in helpers for name in HELPER_BINDINGS[helper]] + lines = [('export function initialize(runtime, importModule) {', None)] + if names: + lines.append((' const {' + ','.join(names) + '}=runtime;', None)) + else: + lines = [('export function createModule({write = text => console.log(text.slice(0,-1))} = {}) {', None), + (" if(typeof write !== 'function')throw new TypeError('write must be a function');", None)] + for helper in helpers: + lines.extend((' ' + line, None) for line in helper_source(helper).splitlines()) lines.extend(self.lines) lines.append(('}', None)) mappings, previous_line, previous_column = [], 0, 0 diff --git a/src/JavaScript/modules.py b/src/JavaScript/modules.py new file mode 100644 index 0000000..d3853b9 --- /dev/null +++ b/src/JavaScript/modules.py @@ -0,0 +1,261 @@ +"""Compile a closed, verified set of ordinary source modules to local ES modules.""" + +import ast +from dataclasses import dataclass +import hashlib +from importlib.metadata import version +import keyword +from pathlib import PurePosixPath +import re +import sys +import unicodedata + +from .compiler import (CompileError, Diagnostic, ModuleIR, SourceImport, SourceSpan, + _FrontEnd, _immutable, _literal_type) +from .emitter import CompiledArtifact, OutputFile, _Emitter, _json +from ._compiler_runtime import HELPER_BINDINGS, helper_closure, helper_source + + +def _portable(path): + relative = PurePosixPath(path) + reserved = {'CON', 'PRN', 'AUX', 'NUL', *(f'COM{i}' for i in range(1, 10)), *(f'LPT{i}' for i in range(1, 10))} + return (bool(relative.parts) and not relative.is_absolute() and '..' not in relative.parts and str(relative) == path + and all(part.split('.')[0].upper() not in reserved and not part.endswith(('.', ' ')) + and not any(ord(character) < 32 or character in '\\:*?"<>|' for character in part) + for part in relative.parts)) + + +@dataclass(frozen=True) +class SourceModule: + name: str + data: bytes + sha256: str + is_package: bool = False + filename: str | None = None + + def __post_init__(self): + if (not isinstance(self.name, str) or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*', self.name) + or any(keyword.iskeyword(part) for part in self.name.split('.')) + or self.name == '__main__' or self.name.split('.')[0] in {*sys.stdlib_module_names, 'JavaScript'}): + raise ValueError('Use an ordinary source module name without stdlib/registry shadowing.') + if type(self.data) is not bytes or len(self.data) > 1_048_576: + raise ValueError('Source requires immutable UTF-8 bytes, at most 1 MiB per module.') + self.data.decode('utf-8') + if hashlib.sha256(self.data).hexdigest() != self.sha256: + raise ValueError('Source SHA-256 does not match its supplied bytes.') + if type(self.is_package) is not bool: + raise TypeError('is_package must be bool') + filename = self.filename + if filename is None: + filename = self.name.replace('.', '/') + ('/__init__.py' if self.is_package else '.py') + if not isinstance(filename, str) or not _portable(filename) or not filename.endswith('.py'): + raise ValueError('filename must be a portable relative logical .py path') + filename.encode('utf-8') + object.__setattr__(self, 'filename', filename) + + @classmethod + def from_text(cls, name, text, *, is_package=False, filename=None): + data = text.encode('utf-8') + return cls(name, data, hashlib.sha256(data).hexdigest(), is_package, filename) + + +class _Modules: + def __init__(self, sources, entrypoint): + if not isinstance(sources, tuple) or any(not isinstance(source, SourceModule) for source in sources): + raise TypeError('sources must be a tuple of verified SourceModule records') + if not 1 <= len(sources) <= 32 or sum(len(source.data) for source in sources) > 8_388_608: + raise ValueError('Declare 1–32 source modules, at most 8 MiB total.') + if len({source.name.casefold() for source in sources}) != len(sources): + raise ValueError('Source module identities must be unique and portable.') + if len({source.filename.casefold() for source in sources}) != len(sources): + raise ValueError('Logical source filenames must be unique and portable.') + # Recheck snapshots at the compiler boundary, including constructed records. + for source in sources: + source.__post_init__() + self.sources = {source.name: source for source in sources} + if entrypoint not in self.sources: + raise ValueError('entrypoint must name a declared source module') + self.entrypoint = entrypoint + self.trees, self.imports, self.dependencies, self.declarations = {}, {}, {}, {} + pending = [entrypoint] + while pending: + name = pending.pop() + if name in self.trees: + continue + source = self.sources[name] + try: + tree = ast.parse(source.data.decode('utf-8'), filename=source.filename) + except SyntaxError as error: + span = SourceSpan(source.filename, error.lineno or 1, error.offset or 1, + error.end_lineno or error.lineno or 1, error.end_offset or error.offset or 1) + raise CompileError(Diagnostic('PJS100', error.msg, span, 'parse')) from None + self.trees[name] = tree + self.imports[name], self.dependencies[name], self.declarations[name] = [], set(), {} + for index in range(1, len(name.split('.'))): + parent = '.'.join(name.split('.')[:index]) + if parent not in self.sources or not self.sources[parent].is_package: + self.fail(name, 'Every parent requires an explicitly supplied package initializer.') + pending.append(parent) + for node in ast.walk(tree): + if not isinstance(node, (ast.Import, ast.ImportFrom)): + continue + if node not in tree.body: + self.fail(name, 'Source imports require unconditional module bindings.', node, 'PJS100') + registry = isinstance(node, ast.Import) and all(alias.name == 'JavaScript' for alias in node.names) + registry = registry or isinstance(node, ast.ImportFrom) and not node.level and node.module == 'JavaScript' + if registry: + continue + if isinstance(node, ast.Import): + self.fail(name, 'Local modules require explicit named from-imports.', node) + base = self.absolute(name, node) + if base not in self.sources or any(alias.name == '*' for alias in node.names): + self.fail(name, 'Import requires a declared source module and explicit exported names.', node) + self.imports[name].append((node, base)) + self.dependencies[name].add(base) + pending.append(base) + for node in tree.body: + if isinstance(node, ast.FunctionDef): + self.declarations[name][node.name] = ('function', node) + elif isinstance(node, ast.Assign): + if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + self.declarations[name][node.targets[0].id] = ('constant', node) + for node, base in self.imports[name]: + for alias in node.names: + self.declarations[name][alias.asname or alias.name] = ('import', (node, base, alias.name)) + visiting, visited = set(), set() + def visit(name): + if name in visiting: + self.fail(name, 'Explicit source-import cycles are outside this module profile.') + if name in visited: + return + visiting.add(name) + for dependency in sorted(self.dependencies[name]): + visit(dependency) + visiting.remove(name) + visited.add(name) + for name in sorted(self.trees): + visit(name) + + def fail(self, name, message, node=None, code='PJS110'): + source = self.sources[name] + span = SourceSpan(source.filename, 1, 1, 1, 1) + if node is not None: + lines = source.data.decode('utf-8').splitlines() + def column(line, offset): + return len(lines[line - 1].encode('utf-8')[:offset].decode('utf-8')) + 1 + span = SourceSpan(source.filename, node.lineno, column(node.lineno, node.col_offset), + node.end_lineno, column(node.end_lineno, node.end_col_offset)) + raise CompileError(Diagnostic(code, message, span, 'bind' if code == 'PJS110' else 'validate')) + + def absolute(self, name, node): + if not node.level: + return node.module + package = name if self.sources[name].is_package else name.rpartition('.')[0] + parts = package.split('.') if package else [] + if node.level > len(parts): + self.fail(name, 'Relative import escapes its declared package.', node) + return '.'.join([*parts[:len(parts) - node.level + 1], *([node.module] if node.module else [])]) + + def export_type(self, name, exported, requester, node): + declaration = self.declarations[name].get(exported) + if declaration is None: + self.fail(requester, 'Export is outside the declared Python namespace: ' + name + ':' + exported, node) + kind, value = declaration + if kind == 'function': + return 'callable' + if kind == 'constant': + if not _immutable(value.value): + self.fail(name, 'Module bindings require immutable literals.', value, 'PJS100') + result = _literal_type(ast.literal_eval(value.value)) + if result is None: + self.fail(name, 'Literal type is outside this profile.', value, 'PJS100') + return result + original, base, target = value + return self.export_type(base, target, name, original) + + def lower(self, name): + source = self.sources[name] + imports = {(node.lineno, node.col_offset): tuple( + SourceImport(alias.asname or alias.name, base, alias.name, + self.export_type(base, alias.name, name, node)) for alias in node.names) + for node, base in self.imports[name]} + frontend = _FrontEnd(source.data.decode('utf-8'), source.filename, imports) + body = frontend.statements(frontend.tree.body) + return ModuleIR(source.filename, source.sha256, + tuple(frontend.bindings[key] for key in sorted(frontend.bindings)), body, + profile='python-browser-modules/1') + + +def compile_modules(sources: tuple[SourceModule, ...], *, entrypoint: str, + source_map: bool = True, notices: tuple[OutputFile, ...] = ()) -> CompiledArtifact: + """Compile a verified source allowlist, never reading or executing target files.""" + if type(source_map) is not bool: + raise TypeError('source_map must be bool') + graph = _Modules(sources, entrypoint) + output, required, module_records = [], {'input'}, [] + modules = sorted(graph.trees) + for index, name in enumerate(modules): + source = graph.sources[name] + filename = f'module-{index:03}.mjs' + files, helpers, exports = _Emitter(graph.lower(name)).render( + source.data.decode('utf-8'), filename, source_map, shared_runtime=True) + output.extend(files) + required.update(helpers) + module_records.append({'module': name, 'file': filename, 'is_package': source.is_package, + 'exports': exports, 'source': source.filename, 'sha256': source.sha256}) + helpers = helper_closure(required) + bindings = [binding for helper in helpers for binding in HELPER_BINDINGS[helper]] + runtime = ["export function createRuntime(write) {", " if(typeof write!=='function')throw new TypeError('write must be a function');"] + for helper in helpers: + runtime.extend(' ' + line for line in helper_source(helper).splitlines()) + runtime.extend([' return Object.freeze({' + ','.join(bindings) + '});', '}']) + output.append(OutputFile('runtime.mjs', ('\n'.join(runtime) + '\n').encode(), 'compatibility-helper')) + imports = ["import {createRuntime} from './runtime.mjs';"] + imports.extend(f"import {{initialize as module{index}}} from './{record['file']}';" for index, record in enumerate(module_records)) + registry = '[' + ','.join('[' + _json(record['module']) + ',module' + str(index) + ']' for index, record in enumerate(module_records)) + ']' + packages = _json([record['module'] for record in module_records if record['is_package']]) + location = _json({'filename': graph.sources[entrypoint].filename, 'line': 1, 'column': 1, 'end_line': 1, 'end_column': 1}) + entry = '\n'.join(imports) + '\n' + f''' +export function createModule({{write = text => console.log(text.slice(0,-1))}} = {{}}) {{ + const runtime=createRuntime(write), factories=new Map({registry}), packages=new Set({packages}), cache=new Map(), loading=new Set(); + function importModule(name) {{ + if(cache.has(name))return cache.get(name); + const parent=name.includes('.')?name.slice(0,name.lastIndexOf('.')):null; + if(parent && packages.has(parent) && !loading.has(parent))importModule(parent); + if(cache.has(name))return cache.get(name); + if(loading.has(name))throw new Error('Source-import cycle is outside this profile.'); + if(!factories.has(name))throw new Error('Undeclared compiled source module.'); + loading.add(name); + const value=factories.get(name)(runtime,importModule); + loading.delete(name);cache.set(name,value);return value; + }} + const module=importModule({_json(entrypoint)}); + return Object.freeze(Object.fromEntries(Object.entries(module).map(([name,value])=>[name,runtime.$expose(value,{location})]))); +}} +''' + output.append(OutputFile('app.mjs', entry.encode('utf-8'), 'executable-module')) + compiler_version = version('Python-JavaScript') + output.append(OutputFile('NOTICES.txt', ('Generated by Python-JavaScript ' + compiler_version + + '\nhttps://github.com/outside-labs/Python-JavaScript\nOriginal compiler helpers; no third-party runtime is embedded.\nThis provenance notice grants no additional license rights.\n').encode(), 'notice')) + if not isinstance(notices, tuple) or any(not isinstance(item, OutputFile) or item.role != 'notice' + or not item.path.startswith('LICENSES/') or not _portable(item.path) + or type(item.data) is not bytes or not item.data.strip() for item in notices): + raise ValueError('notices must contain nonempty immutable notice files under LICENSES/.') + output.extend(notices) + if len({item.path.casefold() for item in output}) != len(output): + raise ValueError('Artifact paths must be unique and portable.') + if any(left.path.casefold().startswith(right.path.casefold() + '/') for left in output for right in output): + raise ValueError('Artifact file/directory paths collide.') + files = tuple(sorted(output, key=lambda item: item.path)) + manifest = {'kind': 'python-javascript-compiled', 'schema_version': 1, + 'compiler': {'name': 'Python-JavaScript', 'version': compiler_version, 'abi': 'python-browser-modules-esm/1'}, + 'language_profile': 'python-browser-modules/1', 'target': 'ecmascript-2022-module', + 'unicode_version': unicodedata.unidata_version, 'entrypoint': 'app.mjs', 'entrypoint_export': 'createModule', + 'application_exports': next(record['exports'] for record in module_records if record['module'] == entrypoint), + 'modules': module_records, 'helpers': list(helpers), + 'sources': [{'path': graph.sources[name].filename, 'sha256': graph.sources[name].sha256, 'size': len(graph.sources[name].data)} for name in modules], + 'imports': [{'from': 'app.mjs', 'to': name} for name in ['runtime.mjs', *[record['file'] for record in module_records]]], + 'source_imports': [{'from': name, 'to': dependency} for name in modules for dependency in sorted(graph.dependencies[name])], + 'notices': sorted(item.path for item in files if item.role == 'notice'), + 'files': [{'path': item.path, 'size': len(item.data), 'sha256': hashlib.sha256(item.data).hexdigest(), 'role': item.role} for item in files]} + return CompiledArtifact(files, (_json(manifest) + '\n').encode('utf-8')) diff --git a/tests/test_compiler_modules.py b/tests/test_compiler_modules.py new file mode 100644 index 0000000..6fa797f --- /dev/null +++ b/tests/test_compiler_modules.py @@ -0,0 +1,162 @@ +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from JavaScript.compiler import CompileError +from JavaScript.emitter import OutputFile, compile_source +from JavaScript.modules import SourceModule, compile_modules +from test_compiler_emitter import run_node + + +def source(name, text, *, package=False): + return SourceModule.from_text(name, text, is_package=package) + + +class CompilerModuleTests(unittest.TestCase): + def assert_parity(self, sources, entrypoint='app'): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for item in sources: + path = root / item.filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(item.data) + oracle = subprocess.run([sys.executable, '-I', '-X', 'utf8', '-c', + 'import sys,importlib;sys.path.insert(0,sys.argv[1]);importlib.import_module(sys.argv[2])', + temporary, entrypoint], capture_output=True, text=True, encoding='utf-8', timeout=5) + artifact = compile_modules(sources, entrypoint=entrypoint) + result = run_node(artifact) + self.assertEqual(result['stdout'], oracle.stdout.replace('\r\n', '\n')) + self.assertEqual(result['error']['name'] if result['error'] else None, + oracle.stderr.splitlines()[-1].split(':', 1)[0] if oracle.returncode else None) + return result + + def test_text_metrics_relative_imports_and_package_reexports(self): + # The supported helper bytes are the reduced source form, without the + # unused reverse utility whose slice syntax remains outside this profile. + corpus = json.loads((Path(__file__).parent / 'fixtures/compiler_contract.json').read_text(encoding='utf-8')) + reference = next(item['source'] for item in corpus['cases'] if item['id'] == 'text-metrics-single-module') + metrics = reference[reference.index('SEPARATORS ='):reference.index('\nprint(metrics(')] + metrics = metrics.replace('character_count(text)', 'count_characters(text)') + sources = (source('app', 'from text_metrics import metrics as selected\n' + 'print(selected(""))\nprint(selected("A😀 é\\nline two"))'), + source('text_metrics', 'from .metrics import metrics', package=True), + source('text_metrics.metrics', 'from .helpers import character_count as count_characters\n' + metrics), + source('text_metrics.helpers', 'def character_count(text):\n return len(text)')) + self.assert_parity(sources) + bad = (source('app', 'from text_metrics import metrics\nmetrics(1)'), *sources[1:]) + result = self.assert_parity(bad) + self.assertEqual(result['error']['location']['filename'], 'text_metrics/metrics.py') + self.assertEqual(result['error']['message'], 'text must be str') + + def test_initialization_order_cached_imports_and_module_entry(self): + sources = (source('app', 'print("app-before")\nfrom pkg.worker import work\n' + 'from pkg.worker import work as again\nprint(work(), again())'), + source('pkg', 'print("package-before")\nfrom .worker import work\nprint("package-after")', package=True), + source('pkg.worker', 'print("worker")\ndef work():\n return 7')) + result = self.assert_parity(sources) + self.assertEqual(result['stdout'], 'app-before\npackage-before\nworker\npackage-after\n7 7\n') + # Loading a child as the entry still executes its initializer once even + # when that initializer itself imports the child. + self.assert_parity(sources, 'pkg.worker') + + def test_shared_values_defaults_aliases_and_separate_contexts(self): + sources = (source('app', 'from helper import echo, echo as alias, default\n' + 'def selected():\n first = default()\n second = default()\n' + ' return first is second, echo(first) is first, isinstance(echo("😀"), str), len(echo((1, 2)))\n' + 'print(echo is alias, selected())'), + source('helper', 'def echo(value):\n return value\ndef default(value=(1, "😀")):\n return value')) + self.assert_parity(sources) + result = run_node(compile_modules(sources, entrypoint='app'), + 'const one=createModule({write: text=>stdout+=text});' + 'const two=createModule({write: text=>stdout+=text});' + 'stdout+=String(one.echo===one.alias)+":"+String(one.echo!==two.echo)+":"+one.echo(9007199254740993n).toString();') + self.assertIsNone(result['error']) + self.assertTrue(result['stdout'].endswith('true:true:9007199254740993')) + + def test_manifest_closure_provenance_maps_and_notices(self): + sources = (source('app', 'from helper import size\nprint(size("A😀"))'), + source('helper', 'def size(value):\n return len(value)'), + source('unused', 'raise RuntimeError("never parse this unsupported source")')) + notice = OutputFile('LICENSES/upstream.txt', b'Original notice\n', 'notice') + artifact = compile_modules(sources, entrypoint='app', notices=(notice,)) + self.assertEqual(artifact, compile_modules(tuple(reversed(sources)), entrypoint='app', notices=(notice,))) + manifest = artifact.manifest + self.assertEqual(manifest['compiler']['abi'], 'python-browser-modules-esm/1') + self.assertEqual([item['module'] for item in manifest['modules']], ['app', 'helper']) + self.assertEqual(manifest['source_imports'], [{'from': 'app', 'to': 'helper'}]) + self.assertEqual(manifest['notices'], ['LICENSES/upstream.txt', 'NOTICES.txt']) + self.assertIn(notice, artifact.files) + by_path = {item.path: item for item in artifact.files} + for record in manifest['files']: + self.assertEqual(record['sha256'], hashlib.sha256(by_path[record['path']].data).hexdigest()) + self.assertEqual(record['size'], len(by_path[record['path']].data)) + for record in manifest['imports']: + self.assertIn(record['from'], by_path) + self.assertIn(record['to'], by_path) + for module in manifest['modules']: + mapping = json.loads(by_path[module['file'] + '.map'].data) + original = next(item for item in sources if item.name == module['module']) + self.assertEqual(mapping['sources'], [original.filename]) + self.assertEqual(mapping['sourcesContent'], [original.data.decode()]) + self.assertEqual(module['sha256'], original.sha256) + no_maps = compile_modules(sources, entrypoint='app', source_map=False) + self.assertNotIn('source-map', {item.role for item in no_maps.files}) + self.assertNotIn('javascript', manifest['helpers']) + self.assertNotIn('iterate', manifest['helpers']) + + def test_import_and_initialization_boundaries_fail_without_execution(self): + cases = [ + (source('app', 'import helper'), source('helper', 'pass')), + (source('app', 'from helper import *'), source('helper', 'pass')), + (source('app', 'from missing import selected'),), + (source('app', 'from helper import absent'), source('helper', 'pass')), + (source('app', 'from helper import selected'), source('helper', 'from app import selected')), + (source('app', 'if True:\n from helper import selected'), source('helper', 'def selected():\n pass')), + (source('app', 'def selected():\n from helper import x'), source('helper', 'x=1')), + (source('app', 'from pkg.helper import x'), source('pkg.helper', 'x=1')), + (source('app', 'from helper import x'), source('helper', 'class x:\n pass')), + (source('app', 'from helper import Array'), source('helper', 'from JavaScript import Array')), + (source('app', 'from helper import x\nx=1'), source('helper', 'x=1')), + (source('app', 'from pkg import x'), source('pkg', 'values=[]\nx=1', package=True)), + (source('app', 'from helper import x'), source('helper', 'x=1\ndef unused(text):\n return text[::-1]')), + (source('app', 'from ...helper import x'),), + (source('app', '__import__("helper")'),), + ] + for sources in cases: + with self.subTest(source=sources[0].data), self.assertRaises(CompileError) as caught: + compile_modules(sources, entrypoint='app') + self.assertIn(caught.exception.diagnostic.code, {'PJS100', 'PJS110'}) + with self.assertRaises(CompileError): + compile_source('from helper import selected') + result = self.assert_parity((source('app', 'print(later())\nfrom helper import later'), + source('helper', 'def later():\n return 1'))) + self.assertEqual(result['error']['location']['line'], 1) + + def test_verified_input_limits_and_portable_output_paths(self): + item = source('app', 'pass') + with self.assertRaises(ValueError): + SourceModule('app', b'pass', '0' * 64) + for name in ['sys', 'JavaScript', 'os.local', 'x-y', '__main__', 'class']: + with self.subTest(name=name), self.assertRaises(ValueError): + source(name, 'pass') + for filename in ['../app.py', '/app.py', 'CON.py', 'app.py\n', 'x\\app.py']: + with self.subTest(filename=filename), self.assertRaises(ValueError): + SourceModule.from_text('app', 'pass', filename=filename) + for sources in [(), (item,) * 2, tuple(source('app' + str(i), 'pass') for i in range(33))]: + with self.assertRaises(ValueError): + compile_modules(sources, entrypoint='app') + with self.assertRaises(TypeError): + compile_modules([item], entrypoint='app') + for notice in [OutputFile('LICENSES/../app.mjs', b'x', 'notice'), + OutputFile('LICENSES/CON.txt', b'x', 'notice'), + OutputFile('LICENSES/x', b'', 'notice'), + OutputFile('LICENSES/x', b'x', 'executable-module')]: + with self.subTest(notice=notice), self.assertRaises(ValueError): + compile_modules((item,), entrypoint='app', notices=(notice,)) + with self.assertRaises(ValueError): + compile_modules((item,), entrypoint='app', notices=( + OutputFile('LICENSES/x', b'x', 'notice'), OutputFile('LICENSES/x/y', b'y', 'notice'))) From c79b172b7106f41dbfd40d757e7371495a2d6f34 Mon Sep 17 00:00:00 2001 From: ellainng <299958882+ellainng@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:33:59 -0700 Subject: [PATCH 2/2] feat(ASYNC-01): lower coroutines and Promise awaits --- IMPLEMENTATION_PLAN.md | 6 ++ README.md | 4 + docs/compiler-async.md | 101 ++++++++++++++++++ docs/compiler-language.md | 5 +- src/JavaScript/_compiler_runtime.py | 101 ++++++++++++++++++ src/JavaScript/compiler.py | 69 ++++++++++--- src/JavaScript/emitter.py | 46 +++++++-- src/JavaScript/modules.py | 34 ++++--- tests/test_compiler_async.py | 152 ++++++++++++++++++++++++++++ 9 files changed, 478 insertions(+), 40 deletions(-) create mode 100644 docs/compiler-async.md create mode 100644 tests/test_compiler_async.py diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 4e4c719..64fd095 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -450,3 +450,9 @@ rejected. Per-module maps, source hashes, static output closure and unchanged notices are retained. Published 0.1.0 artifacts remain unchanged. GitHub Issues and Project fields record live delivery status; the forward table records the original adopted dependency order. ASYNC-01 and BROWSER-01 follow this foundation. + +The explicit [async profile](docs/compiler-async.md) implements ASYNC-01 +coroutines, Promise conversion/chaining, synchronous callbacks, source-located +rejections and AbortSignal context cancellation. JavaScript microtask timing and +CPython scheduling differences are documented. Fetch/DOM/worker transport remains +BROWSER-01; no new async runtime dependency or published artifact is introduced. diff --git a/README.md b/README.md index c048f20..e70a32a 100644 --- a/README.md +++ b/README.md @@ -108,3 +108,7 @@ network requests. The development [compiler emitter](docs/compiler-emitter.md) produces deterministic ES modules for the approved single-module Python profile. This API is not part of the published 0.1.0 distribution. + +The explicit [async compiler profile](docs/compiler-async.md) adds lazy coroutines, +awaited Promise values, callbacks and context cancellation. Browser host adapters +remain separately tracked. diff --git a/docs/compiler-async.md b/docs/compiler-async.md new file mode 100644 index 0000000..4471b7f --- /dev/null +++ b/docs/compiler-async.md @@ -0,0 +1,101 @@ +# Compiled async profile + +The development APIs `parse_source`, `compile_source` and `compile_modules` +accept `async_enabled=True`. Without it, async syntax remains rejected. Published +0.1.0 artifacts predate these compiler APIs. Single-module artifacts identify +`python-browser-async/1` and ABI `python-browser-async-esm/1`; local-module artifacts +identify `python-browser-modules-async/1` and ABI +`python-browser-modules-async-esm/1`. These are explicit extensions of the +[source language](compiler-language.md) and [module contract](compiler-modules.md). + +```python +from JavaScript import Promise + +async def main(): + value = await Promise.resolve(9007199254740993) + print(value + 1) + return value +``` + +Compile this source with the async flag, import its generated `createModule`, +then use `await createModule().main()` in JavaScript. An exported async function +returns a native Promise, including for invalid arguments and cancellation. +Python integers, strings and tuples keep their adopted runtime representation; +no implicit Number conversion is added. Synchronous exports returning an adopted +Promise also expose a native Promise. Other synchronous exports remain synchronous. + +## Coroutine and Promise behavior + +Top-level plain `async def` functions support the existing positional parameters, +immutable defaults, ordinary control flow and return values. Calling one inside +compiled Python creates a lazy, single-use coroutine. Its body begins when +awaited. Arguments/defaults are bound when called. Awaiting the same coroutine +twice raises RuntimeError; a Promise can be awaited repeatedly. Await in defaults, +module initialization or a synchronous function is rejected. Async generators, +async iteration/context managers, asyncio task APIs, gather and top-level await +are deferred. Unawaited coroutines have no execution side effect; no finalizer +warning is promised. + +The approved async JavaScript registry adds `Promise.resolve(value)`, +`Promise.reject(TypeError(message))` and `promise.then(on_fulfilled, on_rejected)`. +Registry imports/aliases resolve by binding identity. A handler is a supported +plain synchronous Python function or None; omitted/None handlers forward outcomes. +Each handler receives one argument and is called once. A returned adopted Promise +is chained; a raised adopted exception rejects the chain without retrying the +handler. Async callbacks and arbitrary thenables are rejected. Promise constructors, +catch/finally, all/race and general native Promise inputs remain deferred. +Coroutines must be awaited before passing them as arguments; async functions +cannot return an unawaited coroutine or Promise as their final value. + +Source locations survive async rejection, imported calls and callbacks. Explicit +builtin TypeError and existing runtime errors retain their Python names and +messages. Unexpected host rejections become a bounded RuntimeError rather than +exposing host internals. Synchronous compiled calls retain depth 128; at most 128 +async coroutine bodies may be active in one context. This bound also limits +concurrent active bodies and is not CPython's recursion limit. + +## Scheduling and cancellation + +An exported async invocation starts on a JavaScript microtask. Internal coroutine +creation remains lazy; native async lowering and Promise continuations preserve +source evaluation order, short circuiting, and callback single invocation. +Awaited boolean/chained-comparison expressions may add a continuation even on a +short-circuited path. Awaiting an already fulfilled Promise also crosses a +microtask checkpoint. CPython's companion Promise uses thread-backed callbacks +and its asyncio bridge may resume an already completed awaitable without the +same checkpoint. Timing relative to unrelated tasks or callbacks is therefore +not a cross-target guarantee. No implicit event loop or Python scheduler is +embedded in the generated artifact. + +`createModule({write, signal})` accepts an optional AbortSignal for that context. +Generated statements, public calls, callbacks and await boundaries check it. +Cancellation rejects pending exported operations with name AbortError and code +CANCELLED at a recorded source location. Await listeners are removed on success, +failure and cancellation. Completed output/other side effects remain committed. +The signal applies to the whole application context; create a fresh context to +restart. Signal reasons are not copied into application errors. + +AbortSignal delivery cannot interrupt busy synchronous execution on its own. +In a worker, the trusted main thread must retain worker termination as the stop +path. The [host protocol](https://github.com/outside-labs/pocket-python/blob/main/docs/browser-host-protocol.md) +uses session/request IDs, serialized tagged values, exactly one terminal result +and CANCELLED/TIMEOUT replies. A future transport adapter must wrap a pending +host result as an adopted Promise, forward cancellation to its request, discard +stale replies and release handles. Raw Promise/Event/DOM objects do not cross +that transport. HOST-01 specifies that adapter; this task does not implement it. + +## Fetch boundary + +CPython code continues to use the published `fetch_api.fetch` implementation +re-exported by `JavaScript`, including its thread-backed Promise and HTTP types. +Those implementations are not copied into compiled artifacts. `fetch`, Request, +Response, streaming bodies and response JSON methods remain rejected by this +compiler profile. BROWSER-01 adds the separately granted `fetch/1` text operation, +lowering to browser-native Fetch through the trusted host with AbortSignal, +origin/CORS, URL, credential, redirect, size and deadline rules. Its first text +metrics UI requires no network access. Streaming, async iterables and standalone +Promise package extraction remain separate extensions. + +Local fixtures compare awaited values, laziness, defaults, order and callback +errors with CPython and execute the generated modules in Node. Abort and listener +cleanup tests use a local AbortController; no external network is needed. diff --git a/docs/compiler-language.md b/docs/compiler-language.md index cbe5759..950aed5 100644 --- a/docs/compiler-language.md +++ b/docs/compiler-language.md @@ -93,8 +93,9 @@ provides initial positive and rejected/deferred cases; parsing known sources for the contract does not prove a compiler already exists. PARSE-02 implements the front end and EMIT-01 must execute emitted output against that corpus. -MOD-01 adds explicit local-module identity and initialization; ASYNC-01 adds -supported await/Promise/cancellation behavior. BROWSER-01 integrates main-thread +The local-module profile implements explicit module identity and initialization; +the [async profile](compiler-async.md) adds supported await/Promise/cancellation +behavior when explicitly enabled. BROWSER-01 integrates main-thread DOM/events and native Fetch through the versioned host protocol; QA-03 verifies CPython/Node/browser parity and source-map errors. Origin/CORS and worker message constraints remain explicit. diff --git a/src/JavaScript/_compiler_runtime.py b/src/JavaScript/_compiler_runtime.py index 9a500c2..f6d593a 100644 --- a/src/JavaScript/_compiler_runtime.py +++ b/src/JavaScript/_compiler_runtime.py @@ -216,6 +216,105 @@ class $JSArray { constructor(values) { this.values = values; } } if (value instanceof $JSArray && name === 'length') return value.values.length; $fail('TypeError', 'Attribute is outside the approved JavaScript registry.', location); } +'''), + 'async': (('call', 'javascript'), r''' +if (signal !== null && (typeof signal !== 'object' || typeof signal.aborted !== 'boolean' || + typeof signal.addEventListener !== 'function' || typeof signal.removeEventListener !== 'function')) { + throw new TypeError('signal must be an AbortSignal or null'); +} +function $cancelled(location) { + const error = new $PyError('AbortError', 'Compiled operation cancelled.', location); + error.code = 'CANCELLED'; return error; +} +function $checkpoint(location) { if (signal?.aborted) throw $cancelled(location); } +class $Coroutine { + constructor(start) { this.start = start; this.started = false; } +} +class $PromiseValue { + constructor(promise) { this.promise = promise; promise.catch(() => {}); Object.freeze(this); } +} +const $asyncFunctions = new WeakSet(); +let $activeAsync = 0; +function $cfn(name, count, defaults, body) { + const fn = $fn(name, count, defaults, (args, location) => new $Coroutine(() => body(args, location))); + $asyncFunctions.add(fn); return fn; +} +function $wait(promise, location) { + $checkpoint(location); + return new Promise((resolve, reject) => { + let finished = false; + const cleanup = () => signal?.removeEventListener('abort', abort); + const abort = () => { if (!finished) { finished = true; cleanup(); reject($cancelled(location)); } }; + signal?.addEventListener('abort', abort, {once: true}); + if (signal?.aborted) { abort(); return; } + promise.then(value => { + if (finished) return; finished = true; cleanup(); + try { $checkpoint(location); resolve(value); } catch (error) { reject(error); } + }, error => { + if (finished) return; finished = true; cleanup(); + reject(error instanceof $PyError ? error : new $PyError('RuntimeError', 'Unhandled Promise rejection.', location)); + }); + }); +} +function $await(value, location) { + $checkpoint(location); + if (value instanceof $PromiseValue) return $wait(value.promise, location); + if (!(value instanceof $Coroutine)) $fail('TypeError', 'await requires an adopted coroutine or Promise.', location); + if (value.started) $fail('RuntimeError', 'cannot reuse already awaited coroutine', location); + value.started = true; + if ($activeAsync >= 128) $fail('RecursionError', 'Maximum active async calls (128) exceeded.', location); + $activeAsync++; + let promise; + try { promise = Promise.resolve(value.start()); } catch (error) { promise = Promise.reject(error); } + const pending = promise.finally(() => { $activeAsync--; }); + pending.catch(() => {}); + return $wait(pending, location); +} +const $PromiseRegistry = Object.freeze({ + resolve: $fn('Promise.resolve', 1, [], (args, location) => { + if (args[0] instanceof $Coroutine) $fail('TypeError', 'Await a coroutine before Promise conversion.', location); + return args[0] instanceof $PromiseValue ? args[0] : new $PromiseValue(Promise.resolve(args[0])); + }), + reject: $fn('Promise.reject', 1, [], (args, location) => { + if (!(args[0] instanceof $PyError)) $fail('TypeError', 'Promise.reject requires an adopted exception.', location); + return new $PromiseValue(Promise.reject(args[0])); + }) +}); +function $asyncAttribute(value, name, location) { + if (value === $JS && name === 'Promise') return $PromiseRegistry; + if (value === $PromiseRegistry && (name === 'resolve' || name === 'reject')) return value[name]; + if (value instanceof $PromiseValue && name === 'then') return $fn('Promise.then', null, [], (args, where) => { + if (args.length > 2 || args.some(fn => fn !== null && (!(fn instanceof $Callable) || $asyncFunctions.has(fn)))) { + $fail('TypeError', 'then requires up to two synchronous Python callbacks or None.', where); + } + const callback = fn => fn == null ? undefined : result => { + $checkpoint(where); + const returned = $call(fn, [result], where); + if (returned instanceof $Coroutine) $fail('TypeError', 'Async Promise callbacks are outside this profile.', where); + return returned instanceof $PromiseValue ? returned.promise : returned; + }; + return new $PromiseValue(value.promise.then(callback(args[0]), callback(args[1]))); + }); + $fail('TypeError', 'Attribute is outside the adopted async registry.', location); +} +'''), + 'async-expose': (('async', 'input'), r''' +const $asyncExposed = new WeakMap(); +function $asyncExpose(value, location) { + if (!(value instanceof $Callable)) return value; + if (!$asyncExposed.has(value)) { + const invoke = args => { + $checkpoint(location); + return $call(value, args.map(argument => $input(argument, location)), location); + }; + $asyncExposed.set(value, $asyncFunctions.has(value) ? + (...args) => Promise.resolve().then(() => $await(invoke(args), location)) : (...args) => { + const result = invoke(args); + return result instanceof $PromiseValue ? $await(result, location) : result; + }); + } + return $asyncExposed.get(value); +} '''), 'input': (('call', 'str', 'tuple'), r''' function $input(value, location, depth = 0, seen = new Set()) { @@ -248,6 +347,8 @@ class $JSArray { constructor(values) { this.values = values; } } 'types': ('$types', '$isinstance'), 'exception': ('$TypeError',), 'repr': ('$repr',), 'print': ('$print',), 'javascript': ('$JS', '$JArray', '$attribute'), 'input': ('$input', '$expose'), + 'async': ('$checkpoint', '$await', '$cfn', '$PromiseRegistry', '$asyncAttribute'), + 'async-expose': ('$asyncExpose',), } diff --git a/src/JavaScript/compiler.py b/src/JavaScript/compiler.py index 1173d5b..5a7ae9e 100644 --- a/src/JavaScript/compiler.py +++ b/src/JavaScript/compiler.py @@ -10,7 +10,8 @@ ValueType = Literal['statement', 'python-value', 'python-none', 'python-bool', 'python-int', 'python-str', 'python-tuple', 'python-exception', 'python-type', 'callable', 'javascript-type', 'javascript-array', - 'javascript-number', 'javascript-callable'] + 'javascript-number', 'javascript-callable', 'async-callable', + 'python-coroutine', 'python-promise'] @dataclass(frozen=True) @@ -116,7 +117,7 @@ def accepted(value): class _FrontEnd: - def __init__(self, source, filename, source_imports=None): + def __init__(self, source, filename, source_imports=None, *, async_enabled=False): self.filename = filename self.lines = source.splitlines() or [''] self.bindings = {} @@ -124,6 +125,8 @@ def __init__(self, source, filename, source_imports=None): self.locals = {} self.types = {} self.source_imports = source_imports or {} + self.async_enabled = async_enabled + self.async_scopes = set() try: self.tree = ast.parse(source, filename=filename) self.table = symtable.symtable(source, filename, 'exec') @@ -166,8 +169,11 @@ def _declare_module(self): continue names = [] kind, target = 'constant', None - if isinstance(node, ast.FunctionDef): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): names, kind = [node.name], 'function' + if isinstance(node, ast.AsyncFunctionDef): + self.types['module:' + node.name] = 'async-callable' + self.async_scopes.add('function:' + node.name) elif isinstance(node, ast.Assign): if len(node.targets) != 1 or not isinstance(node.targets[0], ast.Name) or not _immutable(node.value): self.fail('PJS100', 'Module bindings require single-name immutable literals.', node) @@ -180,9 +186,9 @@ def _declare_module(self): self.fail('PJS110', 'Only the approved JavaScript registry may be imported.', node) names, target = [node.names[0].asname or 'JavaScript'], 'JavaScript' else: - if node.level or node.module != 'JavaScript' or len(node.names) != 1 or node.names[0].name != 'Array': + if node.level or node.module != 'JavaScript' or len(node.names) != 1 or node.names[0].name not in ({'Array', 'Promise'} if self.async_enabled else {'Array'}): self.fail('PJS110', 'Only the approved JavaScript.Array binding may be imported.', node) - names, target = [node.names[0].asname or 'Array'], 'JavaScript.Array' + names, target = [node.names[0].asname or node.names[0].name], 'JavaScript.' + node.names[0].name for name in names: if name in self.root: self.fail('PJS100', 'Module rebinding is outside the immutable profile.', node) @@ -221,18 +227,34 @@ def expression(self, node, scope): kind = self.types.get(binding.id, 'python-value') target = binding.target if binding.kind == 'function': - kind, target = 'callable', binding.id + kind, target = self.types.get(binding.id, 'callable'), binding.id elif binding.kind == 'javascript': kind = 'javascript-type' elif binding.kind == 'builtin': kind = 'python-type' if binding.name in _TYPES else 'callable' elif binding.kind == 'source': - target = binding.id if kind == 'callable' else None + target = binding.id if kind in {'callable', 'async-callable'} else None return self.node(node, 'name', kind, binding=binding.id, target=target) + if isinstance(node, ast.Await): + if not self.async_enabled or scope not in self.async_scopes: + self.fail('PJS100', 'Await requires an adopted async function.', node) + value = self.expression(node.value, scope) + if value.value_type not in {'python-coroutine', 'python-promise', 'python-value'}: + self.fail('PJS120', 'Await requires an adopted coroutine or Promise.', node) + return self.node(node, 'await', 'python-value', value=value) if isinstance(node, ast.Tuple): return self.node(node, 'tuple', 'python-tuple', values=tuple(self.expression(value, scope) for value in node.elts)) if isinstance(node, ast.Attribute): value = self.expression(node.value, scope) + if self.async_enabled: + async_target = {('JavaScript', 'Promise'): 'JavaScript.Promise', + ('JavaScript.Promise', 'resolve'): 'JavaScript.Promise.resolve', + ('JavaScript.Promise', 'reject'): 'JavaScript.Promise.reject'}.get((value.field('target'), node.attr)) + if value.value_type == 'python-promise' and node.attr == 'then': + async_target = 'Promise.then' + if async_target: + return self.node(node, 'async-attribute', 'javascript-type' if async_target == 'JavaScript.Promise' else 'javascript-callable', + value=value, name=node.attr, target=async_target) target = {( 'JavaScript', 'Array'): 'JavaScript.Array', ('JavaScript.Array', 'of'): 'JavaScript.Array.of'}.get((value.field('target'), node.attr)) if target: @@ -250,15 +272,24 @@ def expression(self, node, scope): allowed = {'builtin:len': ('python-int', 1), 'builtin:print': ('python-none', None), 'builtin:isinstance': ('python-bool', 2), 'builtin:TypeError': ('python-exception', 1), 'JavaScript.Array.of': ('javascript-array', None)} + if self.async_enabled: + allowed.update({'JavaScript.Promise.resolve': ('python-promise', 1), + 'JavaScript.Promise.reject': ('python-promise', 1), + 'Promise.then': ('python-promise', None)}) if target in allowed: kind, count = allowed[target] if count is not None and len(node.args) != count: self.fail('PJS120', 'Unsupported argument count.', node) elif target in self.bindings and self.bindings[target].kind in {'function', 'source'}: - kind = 'python-value' + kind = 'python-coroutine' if function.value_type == 'async-callable' else 'python-value' else: self.fail('PJS120', 'Call does not resolve to an approved direct callable.', node) arguments = tuple(self.expression(argument, scope) for argument in node.args) + if any(argument.value_type == 'python-coroutine' for argument in arguments): + self.fail('PJS120', 'Coroutine arguments must be awaited first.', node) + if target == 'Promise.then' and (len(arguments) > 2 or any( + argument.value_type not in {'callable', 'python-none'} for argument in arguments)): + self.fail('PJS120', 'then accepts up to two plain synchronous Python callbacks or None.', node) if target == 'builtin:isinstance' and arguments[1].field('target') not in {'builtin:' + name for name in _TYPES}: self.fail('PJS120', 'isinstance requires an approved builtin type binding.', node) return self.node(node, 'call', kind, function=function, arguments=arguments, target=target) @@ -325,7 +356,10 @@ def statement(self, node, scope, loop): bindings=tuple((self.root[imported.name].id, imported.export) for imported in imports)) names = [alias.asname or alias.name for alias in node.names] return self.node(node, 'import', bindings=tuple(self.root[name].id for name in names)) - if isinstance(node, ast.FunctionDef): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + asynchronous = isinstance(node, ast.AsyncFunctionDef) + if asynchronous and not self.async_enabled: + self.fail('PJS100', 'Async functions require the explicit async profile.', node) if node not in self.tree.body or scope or node.decorator_list or node.type_params or node.returns or any(arg.annotation for arg in [*node.args.posonlyargs, *node.args.args]): self.fail('PJS100', 'Nested, decorated, generic or annotated functions are deferred.', node) if node.args.vararg or node.args.kwarg or node.args.kwonlyargs: @@ -339,7 +373,7 @@ def statement(self, node, scope, loop): self.types = original return self.node(node, 'function', binding=self.root[node.name].id, parameters=tuple(self.locals[function_scope][arg.arg].id for arg in [*node.args.posonlyargs, *node.args.args]), - defaults=defaults, body=body) + defaults=defaults, body=body, asynchronous=asynchronous) if isinstance(node, ast.Assign): if any(not isinstance(target, ast.Name) for target in node.targets): self.fail('PJS100', 'Only simple-name assignment is supported.', node) @@ -355,7 +389,11 @@ def statement(self, node, scope, loop): if isinstance(node, ast.Return): if scope is None: self.fail('PJS100', 'Return requires a function.', node) - return self.node(node, 'return', value=self.expression(node.value, scope) if node.value else None) + value = self.expression(node.value, scope) if node.value else None + if value and (value.value_type == 'python-coroutine' or + (scope in self.async_scopes and value.value_type == 'python-promise')): + self.fail('PJS120', 'Returning an unawaited coroutine or Promise is outside this profile.', node) + return self.node(node, 'return', value=value) if isinstance(node, ast.If): test = self.expression(node.test, scope) original = self.types.copy() @@ -393,7 +431,7 @@ def statement(self, node, scope, loop): self.fail('PJS100', 'Unsupported statement: ' + type(node).__name__, node) -def parse_source(source: str, *, filename: str = '') -> ModuleIR: +def parse_source(source: str, *, filename: str = '', async_enabled: bool = False) -> ModuleIR: """Validate source and return immutable typed IR without executing target code.""" if not isinstance(source, str) or not isinstance(filename, str): raise TypeError('source and filename must be strings') @@ -403,7 +441,10 @@ def parse_source(source: str, *, filename: str = '') -> ModuleIR: raise CompileError(Diagnostic('PJS100', 'Source must be valid UTF-8.', SourceSpan(filename, 1, 1, 1, 1), 'parse')) from None if len(encoded) > 1_048_576: raise CompileError(Diagnostic('PJS100', 'Source exceeds the 1 MiB profile limit.', SourceSpan(filename, 1, 1, 1, 1), 'parse')) - frontend = _FrontEnd(source, filename) + if type(async_enabled) is not bool: + raise TypeError('async_enabled must be bool') + frontend = _FrontEnd(source, filename, async_enabled=async_enabled) body = frontend.statements(frontend.tree.body) return ModuleIR(filename, hashlib.sha256(encoded).hexdigest(), - tuple(frontend.bindings[key] for key in sorted(frontend.bindings)), body) + tuple(frontend.bindings[key] for key in sorted(frontend.bindings)), body, + profile='python-browser-async/1' if async_enabled else 'python-browser/1') diff --git a/src/JavaScript/emitter.py b/src/JavaScript/emitter.py index 5f25aa2..629f7a1 100644 --- a/src/JavaScript/emitter.py +++ b/src/JavaScript/emitter.py @@ -45,6 +45,12 @@ def _vlq(value): return encoded +def _contains_await(node): + if not hasattr(node, 'fields'): + return any(_contains_await(item) for item in node) if isinstance(node, tuple) else False + return node.kind == 'await' or any(_contains_await(value) for _, value in node.fields) + + class _Emitter: def __init__(self, module): self.module = module @@ -93,6 +99,12 @@ def expression(self, node): return value self.use('binding') return f'$get({self.names[binding.id]},{_json(binding.name)},{str(binding.kind in {"local", "parameter"}).lower()},{location})' + if kind == 'await': + self.use('async') + return f'(await $await({self.expression(fields["value"])},{location}))' + if kind == 'async-attribute': + self.use('async') + return f'$asyncAttribute({self.expression(fields["value"])},{_json(fields["name"])},{location})' if kind == 'attribute': self.use('javascript') return f'$attribute({self.expression(fields["value"])},{_json(fields["name"])},{location})' @@ -118,7 +130,8 @@ def expression(self, node): if fields['operator'] == 'and': condition = '!' + condition pieces.extend([f'if({condition})return {name};', f'{name}={self.expression(value)};']) - return '(()=>{' + ''.join(pieces) + f'return {name};' + '})()' + code = '(()=>{' + ''.join(pieces) + f'return {name};' + '})()' + return '(await ' + code.replace('(()=>', '(async()=>', 1) + ')' if _contains_await(node) else code if kind == 'compare': self.use('compare') left, right = self.temporary(), self.temporary() @@ -126,11 +139,15 @@ def expression(self, node): for operator, value in zip(fields['operators'], fields['values'][1:]): pieces.extend([f'{right}={self.expression(value)};', f'if(!$compare({_json(operator)},{left},{right},{location}))return false;', f'{left}={right};']) - return '(()=>{' + ''.join(pieces) + 'return true;})()' + code = '(()=>{' + ''.join(pieces) + 'return true;})()' + return '(await ' + code.replace('(()=>', '(async()=>', 1) + ')' if _contains_await(node) else code raise AssertionError('Unrecognized validated expression: ' + kind) def statements(self, nodes, indent=1): for node in nodes: + if 'async/' in self.module.profile: + self.use('async') + self.add(f'$checkpoint({self.location(node)});', indent, node) self.statement(node, indent) def statement(self, node, indent): @@ -143,13 +160,20 @@ def statement(self, node, indent): elif kind == 'import': self.use('javascript') for identifier in fields['bindings']: - value = '$JS' if self.bindings[identifier].target == 'JavaScript' else '$JArray' + target = self.bindings[identifier].target + if target == 'JavaScript.Promise': + self.use('async') + value = {'JavaScript': '$JS', 'JavaScript.Array': '$JArray', 'JavaScript.Promise': '$PromiseRegistry'}[target] self.add(f'{self.names[identifier]}={value};', indent, node) elif kind == 'function': self.use('call') binding = self.bindings[fields['binding']] defaults = '[' + ','.join(map(self.expression, fields['defaults'])) + ']' - self.add(f'{self.names[binding.id]}=$fn({_json(binding.name)},{len(fields["parameters"])},{defaults},($args)=>{{', indent, node) + asynchronous = fields.get('asynchronous', False) + if asynchronous: + self.use('async') + factory, modifier = ('$cfn', 'async ') if asynchronous else ('$fn', '') + self.add(f'{self.names[binding.id]}={factory}({_json(binding.name)},{len(fields["parameters"])},{defaults},{modifier}($args)=>{{', indent, node) locals_ = [item for item in self.module.bindings if item.scope == 'function:' + binding.name] if locals_: self.use('binding') @@ -201,9 +225,10 @@ def render(self, source, module_name, source_map, *, shared_runtime=False): for binding in exports: value = self.names[binding.id] if binding.kind == 'function' and not shared_runtime: - self.use('input') + self.use('async-expose' if 'async/' in self.module.profile else 'input') definition = next(node for node in self.module.body if node.kind == 'function' and node.field('binding') == binding.id) - value = f'$expose({value},{self.location(definition)})' + expose = '$asyncExpose' if 'async/' in self.module.profile else '$expose' + value = f'{expose}({value},{self.location(definition)})' items.append(f'[{_json(binding.name)}]:{value}') self.add('return Object.freeze({' + ','.join(items) + '});') helpers = helper_closure(self.required) @@ -213,7 +238,8 @@ def render(self, source, module_name, source_map, *, shared_runtime=False): if names: lines.append((' const {' + ','.join(names) + '}=runtime;', None)) else: - lines = [('export function createModule({write = text => console.log(text.slice(0,-1))} = {}) {', None), + option = ', signal = null' if 'async/' in self.module.profile else '' + lines = [('export function createModule({write = text => console.log(text.slice(0,-1))' + option + '} = {}) {', None), (" if(typeof write !== 'function')throw new TypeError('write must be a function');", None)] for helper in helpers: lines.extend((' ' + line, None) for line in helper_source(helper).splitlines()) @@ -239,13 +265,13 @@ def render(self, source, module_name, source_map, *, shared_runtime=False): def compile_source(source: str, *, filename: str = 'app.py', module_name: str = 'app.mjs', - source_map: bool = True) -> CompiledArtifact: + source_map: bool = True, async_enabled: bool = False) -> CompiledArtifact: """Compile one approved module to immutable artifacts; do not execute or publish it.""" if not isinstance(module_name, str) or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_-]*\.mjs', module_name): raise ValueError('module_name must be a portable .mjs basename') if type(source_map) is not bool: raise TypeError('source_map must be bool') - module = parse_source(source, filename=filename) + module = parse_source(source, filename=filename, async_enabled=async_enabled) output, helpers, exports = _Emitter(module).render(source, module_name, source_map) compiler_version = version('Python-JavaScript') notice = ('Generated by Python-JavaScript ' + compiler_version + '\n' @@ -255,7 +281,7 @@ def compile_source(source: str, *, filename: str = 'app.py', module_name: str = output.append(OutputFile('NOTICES.txt', notice.encode('utf-8'), 'notice')) files = tuple(sorted(output, key=lambda item: item.path)) manifest = {'kind': 'python-javascript-compiled', 'schema_version': 1, - 'compiler': {'name': 'Python-JavaScript', 'version': compiler_version, 'abi': 'python-browser-esm/1'}, + 'compiler': {'name': 'Python-JavaScript', 'version': compiler_version, 'abi': 'python-browser-async-esm/1' if async_enabled else 'python-browser-esm/1'}, 'language_profile': module.profile, 'target': 'ecmascript-2022-module', 'unicode_version': unicodedata.unidata_version, 'entrypoint': module_name, 'entrypoint_export': 'createModule', 'application_exports': exports, diff --git a/src/JavaScript/modules.py b/src/JavaScript/modules.py index d3853b9..2dbbe71 100644 --- a/src/JavaScript/modules.py +++ b/src/JavaScript/modules.py @@ -60,7 +60,8 @@ def from_text(cls, name, text, *, is_package=False, filename=None): class _Modules: - def __init__(self, sources, entrypoint): + def __init__(self, sources, entrypoint, async_enabled): + self.async_enabled = async_enabled if not isinstance(sources, tuple) or any(not isinstance(source, SourceModule) for source in sources): raise TypeError('sources must be a tuple of verified SourceModule records') if not 1 <= len(sources) <= 32 or sum(len(source.data) for source in sources) > 8_388_608: @@ -114,7 +115,7 @@ def __init__(self, sources, entrypoint): self.dependencies[name].add(base) pending.append(base) for node in tree.body: - if isinstance(node, ast.FunctionDef): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): self.declarations[name][node.name] = ('function', node) elif isinstance(node, ast.Assign): if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): @@ -162,7 +163,7 @@ def export_type(self, name, exported, requester, node): self.fail(requester, 'Export is outside the declared Python namespace: ' + name + ':' + exported, node) kind, value = declaration if kind == 'function': - return 'callable' + return 'async-callable' if isinstance(value, ast.AsyncFunctionDef) else 'callable' if kind == 'constant': if not _immutable(value.value): self.fail(name, 'Module bindings require immutable literals.', value, 'PJS100') @@ -179,20 +180,22 @@ def lower(self, name): SourceImport(alias.asname or alias.name, base, alias.name, self.export_type(base, alias.name, name, node)) for alias in node.names) for node, base in self.imports[name]} - frontend = _FrontEnd(source.data.decode('utf-8'), source.filename, imports) + frontend = _FrontEnd(source.data.decode('utf-8'), source.filename, imports, async_enabled=self.async_enabled) body = frontend.statements(frontend.tree.body) return ModuleIR(source.filename, source.sha256, tuple(frontend.bindings[key] for key in sorted(frontend.bindings)), body, - profile='python-browser-modules/1') + profile='python-browser-modules-async/1' if self.async_enabled else 'python-browser-modules/1') def compile_modules(sources: tuple[SourceModule, ...], *, entrypoint: str, - source_map: bool = True, notices: tuple[OutputFile, ...] = ()) -> CompiledArtifact: + source_map: bool = True, async_enabled: bool = False, notices: tuple[OutputFile, ...] = ()) -> CompiledArtifact: """Compile a verified source allowlist, never reading or executing target files.""" if type(source_map) is not bool: raise TypeError('source_map must be bool') - graph = _Modules(sources, entrypoint) - output, required, module_records = [], {'input'}, [] + if type(async_enabled) is not bool: + raise TypeError('async_enabled must be bool') + graph = _Modules(sources, entrypoint, async_enabled) + output, required, module_records = [], {'async-expose' if async_enabled else 'input'}, [] modules = sorted(graph.trees) for index, name in enumerate(modules): source = graph.sources[name] @@ -205,7 +208,7 @@ def compile_modules(sources: tuple[SourceModule, ...], *, entrypoint: str, 'exports': exports, 'source': source.filename, 'sha256': source.sha256}) helpers = helper_closure(required) bindings = [binding for helper in helpers for binding in HELPER_BINDINGS[helper]] - runtime = ["export function createRuntime(write) {", " if(typeof write!=='function')throw new TypeError('write must be a function');"] + runtime = ["export function createRuntime(write" + (", signal = null" if async_enabled else "") + ") {", " if(typeof write!=='function')throw new TypeError('write must be a function');"] for helper in helpers: runtime.extend(' ' + line for line in helper_source(helper).splitlines()) runtime.extend([' return Object.freeze({' + ','.join(bindings) + '});', '}']) @@ -215,9 +218,12 @@ def compile_modules(sources: tuple[SourceModule, ...], *, entrypoint: str, registry = '[' + ','.join('[' + _json(record['module']) + ',module' + str(index) + ']' for index, record in enumerate(module_records)) + ']' packages = _json([record['module'] for record in module_records if record['is_package']]) location = _json({'filename': graph.sources[entrypoint].filename, 'line': 1, 'column': 1, 'end_line': 1, 'end_column': 1}) + expose = '$asyncExpose' if async_enabled else '$expose' + signal_option = ', signal = null' if async_enabled else '' + signal_argument = ',signal' if async_enabled else '' entry = '\n'.join(imports) + '\n' + f''' -export function createModule({{write = text => console.log(text.slice(0,-1))}} = {{}}) {{ - const runtime=createRuntime(write), factories=new Map({registry}), packages=new Set({packages}), cache=new Map(), loading=new Set(); +export function createModule({{write = text => console.log(text.slice(0,-1)){signal_option}}} = {{}}) {{ + const runtime=createRuntime(write{signal_argument}), factories=new Map({registry}), packages=new Set({packages}), cache=new Map(), loading=new Set(); function importModule(name) {{ if(cache.has(name))return cache.get(name); const parent=name.includes('.')?name.slice(0,name.lastIndexOf('.')):null; @@ -230,7 +236,7 @@ def compile_modules(sources: tuple[SourceModule, ...], *, entrypoint: str, loading.delete(name);cache.set(name,value);return value; }} const module=importModule({_json(entrypoint)}); - return Object.freeze(Object.fromEntries(Object.entries(module).map(([name,value])=>[name,runtime.$expose(value,{location})]))); + return Object.freeze(Object.fromEntries(Object.entries(module).map(([name,value])=>[name,runtime.{expose}(value,{location})]))); }} ''' output.append(OutputFile('app.mjs', entry.encode('utf-8'), 'executable-module')) @@ -248,8 +254,8 @@ def compile_modules(sources: tuple[SourceModule, ...], *, entrypoint: str, raise ValueError('Artifact file/directory paths collide.') files = tuple(sorted(output, key=lambda item: item.path)) manifest = {'kind': 'python-javascript-compiled', 'schema_version': 1, - 'compiler': {'name': 'Python-JavaScript', 'version': compiler_version, 'abi': 'python-browser-modules-esm/1'}, - 'language_profile': 'python-browser-modules/1', 'target': 'ecmascript-2022-module', + 'compiler': {'name': 'Python-JavaScript', 'version': compiler_version, 'abi': 'python-browser-modules-async-esm/1' if async_enabled else 'python-browser-modules-esm/1'}, + 'language_profile': 'python-browser-modules-async/1' if async_enabled else 'python-browser-modules/1', 'target': 'ecmascript-2022-module', 'unicode_version': unicodedata.unidata_version, 'entrypoint': 'app.mjs', 'entrypoint_export': 'createModule', 'application_exports': next(record['exports'] for record in module_records if record['module'] == entrypoint), 'modules': module_records, 'helpers': list(helpers), diff --git a/tests/test_compiler_async.py b/tests/test_compiler_async.py new file mode 100644 index 0000000..8129976 --- /dev/null +++ b/tests/test_compiler_async.py @@ -0,0 +1,152 @@ +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from JavaScript.compiler import CompileError, parse_source +from JavaScript.emitter import compile_source +from JavaScript.modules import SourceModule, compile_modules + + +def run_async(artifact, invocation='await app.main();', options='write: text=>stdout+=text'): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for file in artifact.files: + (root / file.path).write_bytes(file.data) + (root / 'run.mjs').write_text( + "import {createModule} from './app.mjs';let stdout='',error=null;\n" + "const controller=new AbortController();let listeners=0;" + "const signal={get aborted(){return controller.signal.aborted}," + "addEventListener(...args){listeners++;controller.signal.addEventListener(...args)}," + "removeEventListener(...args){listeners--;controller.signal.removeEventListener(...args)}};\n" + "try { const app=createModule({" + options + "});" + invocation + " }" + "catch(caught){error={name:caught.name,message:caught.message,code:caught.code,location:caught.location};}" + "await new Promise(resolve=>setImmediate(resolve));" + "console.log(JSON.stringify({stdout,error,listeners}));", encoding='utf-8') + process = subprocess.run(['node', '--unhandled-rejections=strict', str(root / 'run.mjs')], + check=True, capture_output=True, text=True, encoding='utf-8', timeout=10) + return json.loads(process.stdout) + + +class CompilerAsyncTests(unittest.TestCase): + def assert_parity(self, source): + oracle = subprocess.run([sys.executable, '-I', '-X', 'utf8', '-c', source + '\nimport asyncio\nasyncio.run(main())'], + capture_output=True, text=True, encoding='utf-8', timeout=10) + result = run_async(compile_source(source, filename='async_case.py', async_enabled=True)) + self.assertEqual(result['stdout'], oracle.stdout.replace('\r\n', '\n')) + self.assertEqual(result['error']['name'] if result['error'] else None, + oracle.stderr.splitlines()[-1].split(':', 1)[0] if oracle.returncode else None) + return result + + def test_lazy_coroutines_defaults_order_and_reuse(self): + self.assert_parity('''async def selected(value=(1, "😀")): + print("inside", value) + return value +async def main(): + pending = selected() + print("before") + first = await pending + print("after", first) + second = await selected() + print(first is second) +''') + result = self.assert_parity('''async def selected(): + return 1 +async def main(): + pending = selected() + print(await pending) + await pending +''') + self.assertEqual(result['error']['message'], 'cannot reuse already awaited coroutine') + + def test_promise_conversion_chained_callbacks_and_errors_once(self): + source = '''from JavaScript import Promise as P +def add(value): + print("callback", value) + return P.resolve(value + 1) +def recover(error): + print("recovered") + return 8 +async def main(): + print(await P.resolve(9007199254740993).then(add)) + print(await P.reject(TypeError("bad")).then(None, recover)) +''' + self.assert_parity(source) + result = self.assert_parity('''from JavaScript import Promise +def fail(value): + print("once") + raise TypeError("callback failed") +async def main(): + print(await Promise.resolve(1).then(fail)) +''') + self.assertEqual(result['error']['message'], 'callback failed') + self.assertEqual(result['error']['location']['line'], 4) + + def test_await_in_chained_comparisons_short_circuit_and_loops(self): + self.assert_parity('''import JavaScript as JS +async def mark(value): + print(value) + return await JS.Promise.resolve(value) +async def main(): + print(await mark(1) < await mark(2) < await mark(3)) + print(await mark(3) < await mark(2) < await mark(1)) + print(False and await mark("never")) + print(True or await mark("never")) + for value in (1, 2): + print(await mark(value)) +''') + + def test_cross_module_await_preserves_values_and_provenance(self): + sources = (SourceModule.from_text('app', 'from helper import selected as main'), + SourceModule.from_text('helper', 'from JavaScript import Promise\nasync def selected():\n print(await Promise.resolve(("😀", 9007199254740993)))')) + artifact = compile_modules(sources, entrypoint='app', async_enabled=True) + result = run_async(artifact) + self.assertIsNone(result['error']) + self.assertEqual(result['stdout'], "('😀', 9007199254740993)\n") + self.assertEqual(artifact.manifest['language_profile'], 'python-browser-modules-async/1') + self.assertIn('async-expose', artifact.manifest['helpers']) + artifact = compile_source('async def main():\n return 1 // 0', async_enabled=True, filename='original.py') + error = run_async(artifact)['error'] + self.assertEqual(error['name'], 'ZeroDivisionError') + self.assertEqual(error['location']['filename'], 'original.py') + self.assertEqual(error['location']['line'], 2) + + def test_cancellation_checkpoints_release_listeners_and_reject_exports(self): + artifact = compile_source('from JavaScript import Promise\nasync def main():\n print("begin")\n' + ' value = await Promise.resolve(1)\n print("after", value)', async_enabled=True) + for invocation in ['controller.abort();await app.main();', + 'const pending=app.main();queueMicrotask(()=>controller.abort());await pending;']: + result = run_async(artifact, invocation, 'write: text=>stdout+=text,signal') + self.assertEqual(result['error']['code'], 'CANCELLED') + self.assertNotIn('after', result['stdout']) + self.assertEqual(result['listeners'], 0) + # An output sink can cancel synchronously while a coroutine is starting. + result = run_async(artifact, options='write: text=>{stdout+=text;controller.abort()},signal') + self.assertEqual(result['error']['code'], 'CANCELLED') + self.assertEqual(result['stdout'], 'begin\n') + self.assertEqual(result['listeners'], 0) + result = run_async(artifact, options='write: text=>stdout+=text,signal') + self.assertIsNone(result['error']) + self.assertEqual(result['listeners'], 0) + + def test_async_call_bound_and_rejected_forms(self): + artifact = compile_source('async def main():\n return await main()', async_enabled=True) + self.assertEqual(run_async(artifact)['error']['name'], 'RecursionError') + sources = ['async def main():\n return await 1', + 'from JavaScript import Promise\nasync def callback(value):\n return value\n' + 'async def main():\n return await Promise.resolve(1).then(callback)', + 'from JavaScript import Promise\nasync def main():\n return Promise.resolve(1)', + 'async def main():\n return main()', + 'async def main():\n async for value in ():\n pass', + 'async def main():\n async with thing:\n pass', + 'from JavaScript import fetch\nasync def main():\n return await fetch("url")', + 'from JavaScript import Promise\nPromise.all(())'] + for source in sources: + with self.subTest(source=source), self.assertRaises(CompileError): + compile_source(source, async_enabled=True) + with self.assertRaises(CompileError): + parse_source('async def main():\n return 1') + with self.assertRaises(TypeError): + parse_source('pass', async_enabled=1)