diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 73ec1ed..7731846 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -434,3 +434,8 @@ and publishes no package. - [Fetch standard](https://fetch.spec.whatwg.org/) - [Python AST](https://docs.python.org/3/library/ast.html) - [PyPI trusted publishing](https://docs.pypi.org/trusted-publishers/) + +The development checkout now provides the [ES module emitter](docs/compiler-emitter.md) +for the approved single-module subset, with selected helpers, source maps and a +versioned artifact manifest. MOD-01 and browser profile integration follow their +explicit dependencies. diff --git a/README.md b/README.md index b21f0f0..c74b7d0 100644 --- a/README.md +++ b/README.md @@ -104,3 +104,7 @@ Node.js 24 runs the native ECMAScript differential fixture. That test is skipped when Node is absent locally; CI and release builds require it. Runtime code needs only Python and the two declared companions. Tests make no external network requests. + +The development [compiler emitter](docs/compiler-emitter.md) produces deterministic +ES modules for the approved single-module Python profile. This API is not part +of the published 0.1.0 distribution. diff --git a/docs/compiler-emitter.md b/docs/compiler-emitter.md new file mode 100644 index 0000000..a2a995d --- /dev/null +++ b/docs/compiler-emitter.md @@ -0,0 +1,91 @@ +# Experimental ES module emitter + +The development checkout exposes `JavaScript.emitter.compile_source`. It +validates source through the python-browser/1 front end, lowers the immutable IR +and returns a CompiledArtifact without executing input or writing files. The +published 0.1.0 package predates this compiler implementation. + +```python +from JavaScript.emitter import compile_source + +artifact = compile_source( + "def count(text):\n return len(text)", + filename="counter.py", +) +print(artifact.manifest["application_exports"]) # ['count'] +``` + +`files` is a tuple of immutable OutputFile(path, data, role) values. The caller +can inspect or pass those bytes to the future pocket-python application builder. +The manifest bytes are separate from their own file/hash closure and can be +saved as PYTHON-JAVASCRIPT.json. Every recorded file has exact size, SHA-256 and +role; imports, notices and selected helper dependencies are explicit. Current +single-module output has no static imports. The compiler ABI is +python-browser-esm/1, independently of package and application-profile versions. + +The emitted entrypoint is app.mjs by default, with a named `createModule` export: + +```javascript +import {createModule} from './app.mjs'; +const app = createModule({write: text => console.log(text)}); +console.log(app.count('A😀')); // 2n +``` + +Each factory call initializes a fresh module in Python source order. Top-level +print/call statements execute at that time. Functions are assigned when their +definition executes; immutable literal defaults are created once per definition, +including tuple identity across calls. Function-local unbound values and global +use before definition raise source-located Python errors. Imports/aliases resolve +through the validated JavaScript registry; only Array.of/length are adopted. +The default output sink uses console.log with the print newline removed. + +External function arguments accept BigInt, bool, null and native JavaScript +strings. Number is rejected to avoid ambiguous rounded integers. An explicit +`{type:'str', codepoints:[...]}` preserves Python surrogate/code-point distinctions; +`{type:'tuple', items:[...]}` constructs a Python tuple. Cycles, arbitrary host +objects and input depth over 64 fail explicitly. Native string input iterates +Unicode code points at the boundary. Exported strings/tuples retain their frozen +str/codepoints or tuple/items wrappers; they are not JavaScript arrays. The host +protocol adapter will serialize these values separately. + +Python integers use BigInt, including signed floor division/modulo and numeric +bool behavior. Strings retain code points, negative indexing, membership and +lexicographic ordering. Tuples retain truthiness, iteration, equality/order and +alias identity. Chained comparisons and boolean operators evaluate each operand +once, in order, with short-circuiting and original operand results. + +Print formats supported values using Python spelling and tuple/string repr. +The compiler generates a compact nonprintable-range table from its CPython +Unicode database and records that database version, so a newer JavaScript +Unicode database cannot silently change repr. This is build-time data, not +runtime Python delegation. Raw surrogate text output requires a separately +adopted sink and fails explicitly; escaped surrogate tuple repr is supported. +Integer/string interning identity remains outside the profile. Dynamic values +outside an adopted operation fail with a source-located TypeError rather than +coercing to native JavaScript behavior. + +Compiled calls use an explicit depth limit of 128 and report RecursionError +before depending on host stack exhaustion. This is a profile resource limit, +not CPython's configurable recursion limit. Arbitrary compiled applications are +not instruction/memory sandboxed; the independent VM has its own budget model. +Unsupported syntax/imports fail compilation before an artifact is returned. +Ordinary error classes are preserved; argument-count wording is profile-specific. + +Source maps use version 3 and original logical filename, lines/columns and +sourcesContent. Runtime errors also carry original location records. Pass +source_map=False to omit map bytes and their sourceMappingURL together. Use a +logical filename when artifacts are intended for distribution; maps intentionally +include the selected source. module_name must be a portable .mjs basename. + +Only selected helpers and their declared dependency closure enter the output. +NOTICES.txt records original helper provenance; no third-party interpreter or +runtime is embedded. The repository has no declared code license; this notice +does not choose or grant one. Redistribution licensing remains a release decision. + +Run `python -m unittest discover -s tests -p 'test_compiler*.py' -v` with the +installed development checkout and Node 24.19.0. Tests compare CPython and emitted +Node output/errors/evaluation order, the single-module text_metrics application, +precision/Unicode/tuple/default behavior and an independent Node source-map +consumer. Local modules, async browser capabilities and the final browser parity +gate remain MOD-01, ASYNC-01/BROWSER-01 and QA-03. This compiler executes only +ahead-of-time approved source; it does not parse new source in the browser. diff --git a/docs/compiler-language.md b/docs/compiler-language.md index 91f42d6..1c09c6d 100644 --- a/docs/compiler-language.md +++ b/docs/compiler-language.md @@ -1,10 +1,9 @@ # Compiler source language contract PARSE-01 adopts `python-browser/1` as the initial language profile, using Python -3.14 grammar and semantics for ordinary Python constructs. This contract is -future compiler work; the released package still provides Python-side -JavaScript compatibility objects. The reference corpus checks Python behavior, -not successful translation. +3.14 grammar and semantics for ordinary Python constructs. The development checkout implements its front end and +[ES module emitter](compiler-emitter.md). The released 0.1.0 package predates +these APIs and provides Python-side JavaScript compatibility objects. ## First application and accepted source diff --git a/pyproject.toml b/pyproject.toml index 8d7461b..7bd129b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ source-include = [ "tests/fixtures/conformance.mjs", "tests/fixtures/compiler_contract.json", "docs/compatibility.md", + "docs/compiler*.md", "scripts/install_distribution.py", "CHANGELOG.md", "RELEASING.md", diff --git a/src/JavaScript/_compiler_runtime.py b/src/JavaScript/_compiler_runtime.py new file mode 100644 index 0000000..1d4dbb9 --- /dev/null +++ b/src/JavaScript/_compiler_runtime.py @@ -0,0 +1,271 @@ +"""Original, dependency-selected ECMAScript helpers for python-browser/1.""" + +from functools import lru_cache +import json + + +HELPERS = { + 'error': ((), r''' +class $PyError extends Error { + constructor(name, message, location) { super(message); this.name = name; this.location = location; } +} +function $fail(name, message, location) { throw new $PyError(name, message, location); } +'''), + 'binding': (('error',), r''' +const $UNBOUND = Symbol('unbound'); +function $get(value, name, local, location) { + if (value === $UNBOUND) $fail(local ? 'UnboundLocalError' : 'NameError', + local ? `cannot access local variable '${name}' where it is not associated with a value` : `name '${name}' is not defined`, location); + return value; +} +'''), + 'str': ((), r''' +class $Str { + constructor(codepoints) { this.type = 'str'; this.codepoints = Object.freeze(codepoints); Object.freeze(this); } +} +function $str(points) { return new $Str(points); } +function $text(value) { return value.codepoints.map(point => String.fromCodePoint(point)).join(''); } +'''), + 'tuple': ((), r''' +class $Tuple { + constructor(items) { this.type = 'tuple'; this.items = Object.freeze(items); Object.freeze(this); } +} +const $EMPTY = new $Tuple([]); +function $tuple(items) { return items.length ? new $Tuple(items) : $EMPTY; } +'''), + 'numeric': (('error',), r''' +function $number(value, location) { + if (typeof value === 'bigint') return value; + if (typeof value === 'boolean') return value ? 1n : 0n; + $fail('TypeError', 'Operation requires an adopted Python integer.', location); +} +'''), + 'truth': (('error', 'str', 'tuple'), r''' +function $truth(value, location) { + if (value === null || typeof value === 'boolean' || typeof value === 'bigint') return Boolean(value); + if (value instanceof $Str) return value.codepoints.length !== 0; + if (value instanceof $Tuple) return value.items.length !== 0; + $fail('TypeError', 'Truth testing this value is outside python-browser/1.', location); +} +'''), + 'binary': (('numeric', 'str'), r''' +function $binary(operator, left, right, location) { + if (operator === '+' && left instanceof $Str && right instanceof $Str) return $str([...left.codepoints, ...right.codepoints]); + left = $number(left, location); right = $number(right, location); + if (operator === '+') return left + right; + if (operator === '-') return left - right; + if (operator === '*') return left * right; + if (right === 0n) $fail('ZeroDivisionError', operator === '//' ? 'division by zero' : 'integer modulo by zero', location); + let quotient = left / right; + if (left % right !== 0n && (left < 0n) !== (right < 0n)) quotient -= 1n; + return operator === '//' ? quotient : left - quotient * right; +} +'''), + 'unary': (('numeric', 'truth'), r''' +function $unary(operator, value, location) { + if (operator === 'not') return !$truth(value, location); + value = $number(value, location); return operator === '-' ? -value : value; +} +'''), + 'equal': (('numeric', 'str', 'tuple'), r''' +function $equal(left, right, location) { + if (left === right) return true; + if (['bigint', 'boolean'].includes(typeof left) && ['bigint', 'boolean'].includes(typeof right)) return $number(left, location) === $number(right, location); + if (left instanceof $Str && right instanceof $Str) return left.codepoints.length === right.codepoints.length && left.codepoints.every((point, index) => point === right.codepoints[index]); + if (left instanceof $Tuple && right instanceof $Tuple) return left.items.length === right.items.length && left.items.every((value, index) => $equal(value, right.items[index], location)); + return false; +} +function $order(left, right, location) { + if (['bigint', 'boolean'].includes(typeof left) && ['bigint', 'boolean'].includes(typeof right)) { + left = $number(left, location); right = $number(right, location); return left < right ? -1 : left > right ? 1 : 0; + } + const strings = left instanceof $Str && right instanceof $Str; + const tuples = left instanceof $Tuple && right instanceof $Tuple; + if (!strings && !tuples) $fail('TypeError', 'Ordering requires matching adopted sequence or numeric values.', location); + const a = strings ? left.codepoints : left.items, b = strings ? right.codepoints : right.items; + for (let index = 0; index < Math.min(a.length, b.length); index++) { + if (strings ? a[index] !== b[index] : !$equal(a[index], b[index], location)) return strings ? (a[index] < b[index] ? -1 : 1) : $order(a[index], b[index], location); + } + return a.length < b.length ? -1 : a.length > b.length ? 1 : 0; +} +'''), + 'compare': (('equal',), r''' +function $compare(operator, left, right, location) { + if (operator === '==' || operator === '!=') return $equal(left, right, location) === (operator === '=='); + if (operator === 'is' || operator === 'is not') { + if (left !== null && right !== null && typeof left !== 'boolean' && typeof right !== 'boolean' && + (typeof left === 'bigint' || typeof right === 'bigint' || left instanceof $Str || right instanceof $Str || typeof left === 'number' || typeof right === 'number')) { + $fail('TypeError', 'Integer/string interning identity is outside python-browser/1.', location); + } + return (left === right) === (operator === 'is'); + } + if (operator === 'in' || operator === 'not in') { + let found = false; + if (right instanceof $Tuple) found = right.items.some(value => $equal(left, value, location)); + else if (right instanceof $Str && left instanceof $Str) { + outer: for (let start = 0; start <= right.codepoints.length - left.codepoints.length; start++) { + for (let offset = 0; offset < left.codepoints.length; offset++) if (left.codepoints[offset] !== right.codepoints[start + offset]) continue outer; + found = true; break; + } + } else $fail('TypeError', 'Membership requires adopted string/tuple values.', location); + return found === (operator === 'in'); + } + const order = $order(left, right, location); + return operator === '<' ? order < 0 : operator === '<=' ? order <= 0 : operator === '>' ? order > 0 : order >= 0; +} +'''), + 'index': (('numeric', 'str', 'tuple'), r''' +function $index(value, index, location) { + const string = value instanceof $Str; + if (!string && !(value instanceof $Tuple)) $fail('TypeError', 'Indexing requires a Python str or tuple.', location); + const items = string ? value.codepoints : value.items; + index = $number(index, location); if (index < 0n) index += BigInt(items.length); + if (index < 0n || index >= BigInt(items.length)) $fail('IndexError', string ? 'string index out of range' : 'tuple index out of range', location); + return string ? $str([items[Number(index)]]) : items[Number(index)]; +} +'''), + 'iterate': (('error', 'str', 'tuple'), r''' +function $iterate(value, location) { + if (value instanceof $Str) return value.codepoints.map(point => $str([point])); + if (value instanceof $Tuple) return value.items; + $fail('TypeError', 'Iteration requires a Python str or tuple.', location); +} +'''), + 'call': (('error',), r''' +class $Callable { + constructor(name, count, defaults, body) { Object.assign(this, {name, count, defaults, body}); Object.freeze(this); } +} +let $depth = 0; +function $fn(name, count, defaults, body) { return new $Callable(name, count, defaults, body); } +function $call(fn, args, location) { + if (!(fn instanceof $Callable)) $fail('TypeError', 'Value is not an adopted callable.', location); + if (fn.count !== null) { + if (args.length < fn.count - fn.defaults.length || args.length > fn.count) $fail('TypeError', `${fn.name}() received an unsupported argument count`, location); + for (let index = args.length; index < fn.count; index++) args.push(fn.defaults[index - (fn.count - fn.defaults.length)]); + } + if ($depth >= 128) $fail('RecursionError', 'Maximum compiled call depth (128) exceeded.', location); + $depth++; + try { return fn.body(args, location); } finally { $depth--; } +} +'''), + 'len': (('call', 'str', 'tuple'), r''' +const $len = $fn('len', 1, [], (args, location) => { + const value = args[0]; + if (value instanceof $Str) return BigInt(value.codepoints.length); + if (value instanceof $Tuple) return BigInt(value.items.length); + $fail('TypeError', 'len() requires an adopted Python str or tuple.', location); +}); +'''), + 'types': (('call', 'str', 'tuple'), r''' +const $types = Object.freeze({str: Symbol('str'), int: Symbol('int'), bool: Symbol('bool'), tuple: Symbol('tuple')}); +const $isinstance = $fn('isinstance', 2, [], (args, location) => { + const [value, type] = args; + if (type === $types.str) return value instanceof $Str; + if (type === $types.tuple) return value instanceof $Tuple; + if (type === $types.int) return typeof value === 'bigint' || typeof value === 'boolean'; + if (type === $types.bool) return typeof value === 'boolean'; + $fail('TypeError', 'isinstance() requires an adopted builtin type.', location); +}); +'''), + 'exception': (('call', 'str'), r''' +const $TypeError = $fn('TypeError', 1, [], (args, location) => { + if (!(args[0] instanceof $Str)) $fail('TypeError', 'TypeError() requires a str message in this profile.', location); + return new $PyError('TypeError', $text(args[0]), location); +}); +'''), + 'repr': (('error', 'str', 'tuple'), r''' +const $nonprintable = __NONPRINTABLE__; +function $printable(point) { + let low = 0, high = $nonprintable.length; + while (low < high) { const mid = (low + high) >> 1; if ($nonprintable[mid][1] < point) low = mid + 1; else high = mid; } + return low === $nonprintable.length || point < $nonprintable[low][0]; +} +function $repr(value, location, quoteString = false) { + if (value === null) return 'None'; + if (typeof value === 'boolean') return value ? 'True' : 'False'; + if (typeof value === 'bigint' || typeof value === 'number') return value.toString(); + if (value instanceof $Str) { + if (!quoteString) { + if (value.codepoints.some(point => point >= 0xd800 && point <= 0xdfff)) $fail('TypeError', 'Surrogate text output requires a separately adopted sink.', location); + return $text(value); + } + const quote = !value.codepoints.includes(39) || value.codepoints.includes(34) ? "'" : '"'; + return quote + value.codepoints.map(point => { + if (point === 92 || point === quote.codePointAt(0)) return '\\' + String.fromCodePoint(point); + if (point === 9) return '\\t'; if (point === 10) return '\\n'; if (point === 13) return '\\r'; + if (!$printable(point)) return (point < 256 ? '\\x' : point < 65536 ? '\\u' : '\\U') + point.toString(16).padStart(point < 256 ? 2 : point < 65536 ? 4 : 8, '0'); + return String.fromCodePoint(point); + }).join('') + quote; + } + if (value instanceof $Tuple) return '(' + value.items.map(item => $repr(item, location, true)).join(', ') + (value.items.length === 1 ? ',' : '') + ')'; + if (value instanceof $PyError) return value.message; + $fail('TypeError', 'Printing this value is outside python-browser/1.', location); +} +'''), + 'print': (('call', 'repr'), r''' +const $print = $fn('print', null, [], (args, location) => { write(args.map(value => $repr(value, location)).join(' ') + '\n'); return null; }); +'''), + 'javascript': (('call',), r''' +class $JSArray { constructor(values) { this.values = values; } } +const $of = $fn('Array.of', null, [], args => new $JSArray(args)); +const $JArray = Object.freeze({of: $of}); +const $JS = Object.freeze({Array: $JArray}); +function $attribute(value, name, location) { + if (value === $JS && name === 'Array') return $JArray; + if (value === $JArray && name === 'of') return $of; + if (value instanceof $JSArray && name === 'length') return value.values.length; + $fail('TypeError', 'Attribute is outside the approved JavaScript registry.', location); +} +'''), + 'input': (('call', 'str', 'tuple'), r''' +function $input(value, location, depth = 0, seen = new Set()) { + if (depth > 64) $fail('TypeError', 'Input nesting exceeds 64.', location); + if (value === null || ['boolean', 'bigint'].includes(typeof value) || value instanceof $Str || value instanceof $Tuple) return value; + if (typeof value === 'string') return $str(Array.from(value, character => character.codePointAt(0))); + if (typeof value !== 'object' || seen.has(value)) $fail('TypeError', 'Input requires an adopted Python value.', location); + if (value.type === 'str' && Array.isArray(value.codepoints) && value.codepoints.length <= 1048576 && Array.from(value.codepoints).every(point => Number.isInteger(point) && point >= 0 && point <= 0x10ffff)) return $str([...value.codepoints]); + if (value.type === 'tuple' && Array.isArray(value.items)) { + seen.add(value); const items = Array.from(value.items, item => $input(item, location, depth + 1, seen)); seen.delete(value); return $tuple(items); + } + $fail('TypeError', 'Use BigInt, string, bool, null or explicit str/tuple input; JavaScript Number/Array is not Python int/tuple.', location); +} +function $expose(value, location) { + return value instanceof $Callable ? (...args) => $call(value, args.map(argument => $input(argument, location)), location) : value; +} +'''), +} + + +@lru_cache(maxsize=1) +def _nonprintable_ranges(): + ranges, start = [], None + for point in range(0x110000): + if not chr(point).isprintable(): + if start is None: + start = point + elif start is not None: + ranges.append([start, point - 1]) + start = None + if start is not None: + ranges.append([start, 0x10ffff]) + return json.dumps(ranges, separators=(',', ':')) + + +def helper_closure(required): + ordered = [] + def visit(name): + if name in ordered: + return + for dependency in HELPERS[name][0]: + visit(dependency) + ordered.append(name) + for name in sorted(required): + visit(name) + return tuple(ordered) + + +def helper_source(name): + source = HELPERS[name][1].strip() + if name == 'repr': + source = source.replace('__NONPRINTABLE__', _nonprintable_ranges()) + return source diff --git a/src/JavaScript/emitter.py b/src/JavaScript/emitter.py new file mode 100644 index 0000000..db59905 --- /dev/null +++ b/src/JavaScript/emitter.py @@ -0,0 +1,254 @@ +"""Deterministic ES modules for the validated python-browser/1 source profile.""" + +from dataclasses import dataclass +import hashlib +from importlib.metadata import version +import json +import re +import unicodedata + +from .compiler import parse_source +from ._compiler_runtime import helper_closure, helper_source + + +def _json(value): + return json.dumps(value, ensure_ascii=True, separators=(',', ':')) + + +@dataclass(frozen=True) +class OutputFile: + path: str + data: bytes + role: str + + +@dataclass(frozen=True) +class CompiledArtifact: + files: tuple[OutputFile, ...] + manifest_bytes: bytes + + @property + def manifest(self): + """Return a fresh snapshot; callers cannot mutate artifact metadata.""" + return json.loads(self.manifest_bytes) + + +def _vlq(value): + alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + value = (-value << 1 | 1) if value < 0 else value << 1 + encoded = '' + while True: + digit = value & 31 + value >>= 5 + encoded += alphabet[digit | (32 if value else 0)] + if not value: + return encoded + + +class _Emitter: + def __init__(self, module): + self.module = module + self.bindings = {binding.id: binding for binding in module.bindings} + self.names = {binding.id: '$v' + str(index) for index, binding in enumerate(module.bindings)} + self.required = set() + self.lines = [] + self.counter = 0 + + def use(self, helper): + self.required.add(helper) + + def temporary(self): + self.counter += 1 + return '$t' + str(self.counter) + + def location(self, node): + return _json(vars(node.span)) + + def add(self, text, indent=1, node=None): + self.lines.append((' ' * indent + text, node.span if node else None)) + + def expression(self, node): + fields, location = dict(node.fields), self.location(node) + kind = node.kind + if kind == 'constant': + if node.value_type == 'python-int': + return fields['value'] + 'n' + if node.value_type == 'python-str': + self.use('str') + return '$str(' + _json([ord(character) for character in fields['value']]) + ')' + return _json(fields['value']) + if kind == 'tuple': + self.use('tuple') + return '$tuple([' + ','.join(map(self.expression, fields['values'])) + '])' + if kind == 'name': + binding = self.bindings[fields['binding']] + if binding.kind == 'builtin': + name = binding.name + if name in {'str', 'int', 'bool', 'tuple'}: + self.use('types') + return '$types.' + name + helper, value = {'len': ('len', '$len'), 'print': ('print', '$print'), + 'isinstance': ('types', '$isinstance'), 'TypeError': ('exception', '$TypeError')}[name] + self.use(helper) + return value + self.use('binding') + return f'$get({self.names[binding.id]},{_json(binding.name)},{str(binding.kind in {"local", "parameter"}).lower()},{location})' + if kind == 'attribute': + self.use('javascript') + return f'$attribute({self.expression(fields["value"])},{_json(fields["name"])},{location})' + if kind == 'call': + self.use('call') + return f'$call({self.expression(fields["function"])},[{",".join(map(self.expression, fields["arguments"]))}],{location})' + if kind in {'binary', 'unary', 'index'}: + self.use(kind) + if kind == 'binary': + values = [_json(fields['operator']), self.expression(fields['left']), self.expression(fields['right'])] + elif kind == 'unary': + values = [_json(fields['operator']), self.expression(fields['value'])] + else: + values = [self.expression(fields['value']), self.expression(fields['index'])] + return '$' + kind + '(' + ','.join([*values, location]) + ')' + if kind == 'boolean': + self.use('truth') + name = self.temporary() + values = fields['values'] + pieces = [f'let {name}={self.expression(values[0])};'] + for value in values[1:]: + condition = f'$truth({name},{location})' + if fields['operator'] == 'and': + condition = '!' + condition + pieces.extend([f'if({condition})return {name};', f'{name}={self.expression(value)};']) + return '(()=>{' + ''.join(pieces) + f'return {name};' + '})()' + if kind == 'compare': + self.use('compare') + left, right = self.temporary(), self.temporary() + pieces = [f'let {left}={self.expression(fields["values"][0])};let {right};'] + for operator, value in zip(fields['operators'], fields['values'][1:]): + pieces.extend([f'{right}={self.expression(value)};', + f'if(!$compare({_json(operator)},{left},{right},{location}))return false;', f'{left}={right};']) + return '(()=>{' + ''.join(pieces) + 'return true;})()' + raise AssertionError('Unrecognized validated expression: ' + kind) + + def statements(self, nodes, indent=1): + for node in nodes: + self.statement(node, indent) + + def statement(self, node, indent): + fields, kind = dict(node.fields), node.kind + if kind == 'import': + self.use('javascript') + for identifier in fields['bindings']: + value = '$JS' if self.bindings[identifier].target == 'JavaScript' else '$JArray' + self.add(f'{self.names[identifier]}={value};', indent, node) + elif kind == 'function': + self.use('call') + binding = self.bindings[fields['binding']] + defaults = '[' + ','.join(map(self.expression, fields['defaults'])) + ']' + self.add(f'{self.names[binding.id]}=$fn({_json(binding.name)},{len(fields["parameters"])},{defaults},($args)=>{{', indent, node) + locals_ = [item for item in self.module.bindings if item.scope == 'function:' + binding.name] + if locals_: + self.use('binding') + self.add('let ' + ','.join(self.names[item.id] + '=$UNBOUND' for item in locals_) + ';', indent + 1) + for index, parameter in enumerate(fields['parameters']): + self.add(f'{self.names[parameter]}=$args[{index}];', indent + 1) + self.statements(fields['body'], indent + 1) + self.add('return null;', indent + 1) + self.add('});', indent) + elif kind == 'assign': + temporary = self.temporary() + self.add(f'const {temporary}={self.expression(fields["value"])};', indent, node) + for target in fields['targets']: + self.add(f'{self.names[target]}={temporary};', indent, node) + elif kind in {'expression', 'return', 'raise'}: + value = self.expression(fields['value']) if fields['value'] else 'null' + prefix = {'expression': '', 'return': 'return ', 'raise': 'throw '}[kind] + self.add(prefix + value + ';', indent, node) + elif kind == 'if': + self.use('truth') + self.add(f'if($truth({self.expression(fields["test"])},{self.location(node)})){{', indent, node) + self.statements(fields['body'], indent + 1) + if fields['otherwise']: + self.add('}else{', indent) + self.statements(fields['otherwise'], indent + 1) + self.add('}', indent) + elif kind == 'for': + self.use('iterate') + temporary = self.temporary() + self.add(f'for(const {temporary} of $iterate({self.expression(fields["value"])},{self.location(node)})){{', indent, node) + self.add(f'{self.names[fields["target"]]}={temporary};', indent + 1) + self.statements(fields['body'], indent + 1) + self.add('}', indent) + elif kind in {'break', 'continue'}: + self.add(kind + ';', indent, node) + elif kind == 'pass': + self.add(';', indent, node) + else: + raise AssertionError('Unrecognized validated statement: ' + kind) + + def render(self, source, module_name, source_map): + roots = [binding for binding in self.module.bindings if binding.scope == 'module'] + if roots: + self.use('binding') + self.add('let ' + ','.join(self.names[binding.id] + '=$UNBOUND' for binding in roots) + ';') + self.statements(self.module.body) + exports = [binding for binding in roots if binding.kind != 'javascript'] + items = [] + for binding in exports: + value = self.names[binding.id] + if binding.kind == 'function': + self.use('input') + definition = next(node for node in self.module.body if node.kind == 'function' and node.field('binding') == binding.id) + value = f'$expose({value},{self.location(definition)})' + items.append(f'[{_json(binding.name)}]:{value}') + self.add('return Object.freeze({' + ','.join(items) + '});') + helpers = helper_closure(self.required) + lines = [('export function createModule({write = text => console.log(text.slice(0,-1))} = {}) {', None), + (" if(typeof write !== 'function')throw new TypeError('write must be a function');", None)] + for helper in helpers: + lines.extend((' ' + line, None) for line in helper_source(helper).splitlines()) + lines.extend(self.lines) + lines.append(('}', None)) + mappings, previous_line, previous_column = [], 0, 0 + for text, span in lines: + if span: + column = len(text) - len(text.lstrip()) + mappings.append(''.join(_vlq(value) for value in [column, 0, span.line - 1 - previous_line, span.column - 1 - previous_column])) + previous_line, previous_column = span.line - 1, span.column - 1 + else: + mappings.append('') + code = '\n'.join(text for text, _ in lines) + '\n' + output = [OutputFile(module_name, code.encode('utf-8'), 'executable-module')] + if source_map: + code += '//# sourceMappingURL=' + module_name + '.map\n' + output[0] = OutputFile(module_name, code.encode('utf-8'), 'executable-module') + data = {'version': 3, 'file': module_name, 'sources': [self.module.filename], + 'sourcesContent': [source], 'names': [], 'mappings': ';'.join(mappings)} + output.append(OutputFile(module_name + '.map', (_json(data) + '\n').encode('utf-8'), 'source-map')) + return output, helpers, [binding.name for binding in exports] + + +def compile_source(source: str, *, filename: str = 'app.py', module_name: str = 'app.mjs', + source_map: bool = True) -> CompiledArtifact: + """Compile one approved module to immutable artifacts; do not execute or publish it.""" + if not isinstance(module_name, str) or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_-]*\.mjs', module_name): + raise ValueError('module_name must be a portable .mjs basename') + if type(source_map) is not bool: + raise TypeError('source_map must be bool') + module = parse_source(source, filename=filename) + output, helpers, exports = _Emitter(module).render(source, module_name, source_map) + compiler_version = version('Python-JavaScript') + notice = ('Generated by Python-JavaScript ' + compiler_version + '\n' + 'https://github.com/outside-labs/Python-JavaScript\n' + 'Original compiler helper implementation; no third-party runtime is embedded.\n' + 'This provenance notice grants no additional license rights.\n') + output.append(OutputFile('NOTICES.txt', notice.encode('utf-8'), 'notice')) + files = tuple(sorted(output, key=lambda item: item.path)) + manifest = {'kind': 'python-javascript-compiled', 'schema_version': 1, + 'compiler': {'name': 'Python-JavaScript', 'version': compiler_version, 'abi': 'python-browser-esm/1'}, + 'language_profile': module.profile, 'target': 'ecmascript-2022-module', + 'unicode_version': unicodedata.unidata_version, 'entrypoint': module_name, + 'entrypoint_export': 'createModule', 'application_exports': exports, + 'sources': [{'path': filename, 'sha256': module.source_sha256, 'size': len(source.encode('utf-8'))}], + 'files': [{'path': item.path, 'size': len(item.data), 'sha256': hashlib.sha256(item.data).hexdigest(), 'role': item.role} for item in files], + 'imports': [], 'helpers': list(helpers), 'notices': ['NOTICES.txt']} + return CompiledArtifact(files, (_json(manifest) + '\n').encode('utf-8')) diff --git a/tests/fixtures/compiler_contract.json b/tests/fixtures/compiler_contract.json index 01491c4..133e1e4 100644 --- a/tests/fixtures/compiler_contract.json +++ b/tests/fixtures/compiler_contract.json @@ -37,6 +37,11 @@ "id": "alias", "source": "from JavaScript import Array as A\nprint(A.of(1, 2).length)", "stdout": "2\n" + }, + { + "id": "text-metrics-single-module", + "source": "def character_count(text):\n return len(text)\n\n\n\nSEPARATORS = \" \\t\\n\\r\\v\\f\"\n\n\ndef metrics(text):\n if not isinstance(text, str):\n raise TypeError(\"text must be str\")\n characters = character_count(text)\n words = 0\n lines = 0\n inside_word = False\n if text:\n lines = 1\n for character in text:\n if character == \"\\n\":\n lines = lines + 1\n if character in SEPARATORS:\n inside_word = False\n elif not inside_word:\n words = words + 1\n inside_word = True\n return characters, words, lines\n\nprint(metrics(\"\"))\nprint(metrics(\"A😀 é\\nline two\"))\n", + "stdout": "(0, 0, 0)\n(14, 4, 2)\n" } ], "boundaries": [ diff --git a/tests/test_compiler_emitter.py b/tests/test_compiler_emitter.py new file mode 100644 index 0000000..f1e9ae9 --- /dev/null +++ b/tests/test_compiler_emitter.py @@ -0,0 +1,163 @@ +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from JavaScript.compiler import CompileError +from JavaScript.emitter import compile_source + + +CORPUS = Path(__file__).parent / 'fixtures' / 'compiler_contract.json' + + +def run_node(artifact, invocation='createModule({write: text => stdout += text});'): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for file in artifact.files: + (root / file.path).write_bytes(file.data) + runner = ("import {createModule} from './app.mjs'; let stdout=''; let error=null;\n" + "try { " + invocation + " } catch (caught) { error={name:caught.name,message:caught.message,location:caught.location}; }\n" + "console.log(JSON.stringify({stdout,error}));") + (root / 'run.mjs').write_text(runner, encoding='utf-8') + process = subprocess.run(['node', str(root / 'run.mjs')], check=True, + capture_output=True, text=True, encoding='utf-8', timeout=10) + return json.loads(process.stdout) + + +class CompilerEmitterTests(unittest.TestCase): + def assert_parity(self, source): + oracle = subprocess.run([sys.executable, '-I', '-X', 'utf8', '-c', source], + capture_output=True, text=True, encoding='utf-8', timeout=5) + result = run_node(compile_source(source, filename='case.py')) + self.assertEqual(result['stdout'], oracle.stdout.replace('\r\n', '\n')) + if oracle.returncode: + self.assertIsNotNone(result['error']) + self.assertEqual(result['error']['name'], oracle.stderr.splitlines()[-1].split(':', 1)[0]) + self.assertEqual(result['error']['location']['filename'], 'case.py') + else: + self.assertIsNone(result['error'], result['error']) + return result + + def test_corpus_executes_as_emitted_modules(self): + corpus = json.loads(CORPUS.read_text(encoding='utf-8')) + for case in corpus['cases']: + with self.subTest(case=case['id']): + result = self.assert_parity(case['source']) + self.assertEqual(result['stdout'], case['stdout']) + for case in corpus['boundaries']: + with self.subTest(boundary=case['id']), self.assertRaises(CompileError): + compile_source(case['source']) + + def test_single_evaluation_short_circuit_order_and_loop_control(self): + self.assert_parity('''def mark(value): + print(value) + return value +def selected(): + total = 0 + for value in (1, 2, 3, 4): + if value == 2: + continue + if value == 4: + break + total = total + value + return total +print(mark(1) < mark(2) < mark(3)) +print(mark(3) < mark(2) < mark(1)) +print(mark('') and mark('never')) +print(mark(()) or mark('fallback')) +print(selected())''') + + def test_scalar_tuple_unicode_and_registry_semantics(self): + self.assert_parity('''import JavaScript as JS +def selected(): + original = (1,) + alias = original + return original is alias, () == (), not (), True + 2, '😀'[-1], '😀' in 'A😀', 2 in (1, 2) +print(JS.Array.of(1, 2).length) +print(selected()) +print(('a\\nb', "a'b", '\\x08', '\\u00a0', '\\ud800')) +print(('\\U0001fae9', '\\u0301')) +print(len('\\ud83d\\ude00'), len('😀'), '\\ud83d\\ude00' == '😀') +print(-7 // 3, 7 // -3, -7 % 3, 7 % -3, (1, 2) < (1, 3))''') + + def test_defaults_preserve_definition_time_identity_and_recursion(self): + self.assert_parity('''def same(value=(1, 2)): + return value +def selected(): + first = same() + second = same() + return first is second +def factorial(value): + if value: + return value * factorial(value - 1) + return 1 +print(selected(), factorial(6))''') + result = run_node(compile_source('def forever():\n return forever()\nforever()')) + self.assertEqual(result['error']['name'], 'RecursionError') + + def test_runtime_errors_keep_source_and_output_before_failure(self): + sources = [ + 'print("before")\n1 // 0', 'print("before")\n(1,)[5]', + 'def selected(value):\n if value:\n item = 1\n return item\nselected(False)', + 'print(later())\ndef later():\n return 1', + 'def selected(value):\n return value + 1\nselected()', + 'def selected(value):\n if not isinstance(value, str):\n raise TypeError("text must be str")\n return len(value)\nselected(1)', + 'A.of(1)\nfrom JavaScript import Array as A', + ] + for source in sources: + with self.subTest(source=source): + self.assert_parity(source) + error = run_node(compile_source('def selected():\n return 1 // 0\nselected()', filename='original.py'))['error'] + self.assertEqual(error['location']['filename'], 'original.py') + self.assertEqual(error['location']['line'], 2) + self.assertEqual(error['location']['column'], 12) + + def test_exports_enforce_inputs_and_independent_module_state(self): + artifact = compile_source('def size(text):\n return len(text)\ndef echo(value):\n return value') + result = run_node(artifact, "const app=createModule({write: text => stdout+=text}); stdout=app.size('A😀').toString();") + self.assertEqual(result['stdout'], '2') + result = run_node(artifact, "const app=createModule(); app.echo(9007199254740992);") + self.assertEqual(result['error']['name'], 'TypeError') + result = run_node(artifact, "const app=createModule(); const value={type:'tuple',items:[]}; value.items.push(value); app.echo(value);") + self.assertEqual(result['error']['name'], 'TypeError') + + def test_manifest_determinism_closure_hashes_and_optional_maps(self): + source = 'def selected(value):\n return value + 1' + artifact = compile_source(source) + self.assertEqual(artifact, compile_source(source)) + manifest = artifact.manifest + self.assertEqual(manifest['kind'], 'python-javascript-compiled') + self.assertEqual(manifest['imports'], []) + self.assertIn('binary', manifest['helpers']) + self.assertNotIn('javascript', manifest['helpers']) + self.assertNotIn('print', manifest['helpers']) + self.assertEqual(manifest['application_exports'], ['selected']) + for record, file in zip(manifest['files'], artifact.files): + self.assertEqual(record['path'], file.path) + self.assertEqual(record['size'], len(file.data)) + self.assertEqual(record['sha256'], hashlib.sha256(file.data).hexdigest()) + source_map = json.loads(next(file.data for file in artifact.files if file.role == 'source-map')) + self.assertEqual(source_map['sourcesContent'], [source]) + self.assertEqual(source_map['version'], 3) + self.assertTrue(any(source_map['mappings'].split(';'))) + code = next(file.data.decode() for file in artifact.files if file.role == 'executable-module') + generated_line = next(index for index, line in enumerate(code.splitlines()) if 'return $binary' in line) + consumer = "import {SourceMap} from 'node:module'; const input=JSON.parse(process.argv[1]); console.log(JSON.stringify(new SourceMap(input.map).findEntry(input.line,4)));" + result = subprocess.run(['node', '--input-type=module', '-e', consumer, + json.dumps({'map': source_map, 'line': generated_line})], + check=True, capture_output=True, text=True, timeout=5) + mapping = json.loads(result.stdout) + self.assertEqual(mapping['originalLine'], 1) + self.assertEqual(mapping['originalColumn'], 4) + self.assertEqual(mapping['originalSource'], 'app.py') + manifest['files'][0]['sha256'] = 'changed' + self.assertNotEqual(artifact.manifest['files'][0]['sha256'], 'changed') + no_map = compile_source(source, source_map=False) + self.assertNotIn('sourceMappingURL', next(file.data.decode() for file in no_map.files if file.role == 'executable-module')) + self.assertNotIn('source-map', {file.role for file in no_map.files}) + for invalid in ['../app.mjs', 'app.js', 'app.mjs\nexecute', 'é.mjs']: + with self.assertRaises(ValueError): + compile_source('pass', module_name=invalid)