Support RNC-packed N64 code and verify original-cartridge exports

README.md

@@ -319,3 +319,19 @@ The code that came from decomp.wtf was released under the MIT license by Sam Wol
 [LICENSE-decomp.wtf](LICENSE-decomp.wtf).
 
 <p align="center"><sub>Not related to Nintendo, Sony or Microsoft. Platform names are trademarks of their owners.</sub></p>
+
+### RNC-packed N64 programs
+
+The N64 importer recognizes RNC method-1 blocks, verifies both CRC16 checksums, and checks the decoded bytes for
+code before adding them to the analysis image. Method 2 and encrypted/locked streams are not supported. A small
+initial function count can still mean that the game uses another packing format or an undiscovered loader.
+
+For RNC-expanded exports, `baserom.z64` is the expanded analysis image and `baserom.cartridge.z64` is the original
+normalized cartridge. Keep both private. The build first checks the linked analysis image, then reconstructs the
+packed blocks using literals from the newly linked output and the original stream's coding decisions. It rejects
+changes incompatible with those references and checks the complete cartridge against the original. This template
+operation supports matching builds; it is not a general compressor for mods. Other expanded packing formats fail
+export with an explanation until they have a verified cartridge reconstruction.
+
+An identical rebuild can still contain assembly and binary fallbacks. Check the reported number of functions
+selected as C separately; an identical cartridge does not mean that every function has been decompiled.

src/decompnet/consoles/n64/code.py

@@ -214,14 +214,29 @@ def unpack_code(rom: bytes) -> list[tuple[int, bytes]]:
             if len(out) == size and _looks_like_code(out):
                 found.append((at, out))
         at = rom.find(b"Yaz0", at + 4)
+    from . import rnc
+
+    at = rom.find(b"RNC\x01")
+    while at >= 0:
+        size = int.from_bytes(rom[at + 4:at + 8], "big")
+        packed = int.from_bytes(rom[at + 8:at + 12], "big")
+        if 0x1000 <= size <= rnc.MAX_OUTPUT and at + rnc.HEADER_BYTES + packed <= len(rom):
+            try:
+                out = rnc.unpack(rom[at:at + rnc.HEADER_BYTES + packed])
+            except rnc.RncError:
+                pass
+            else:
+                if _looks_like_code(out):
+                    found.append((at, out))
+        at = rom.find(b"RNC\x01", at + 4)
     return sorted(found)
 
 
 # ---------------------------------------------------------------------------------------------- into splat's config
 
 def add_segments(config: Path, workdir: Path, image: str, log=print) -> str:
     """An N64 game's code outside the main segment (n64code.py), added to splat's config as code segments of their
-    own. Compressed code (Rare's, Yaz0) is unpacked first and appended to a working copy of the ROM, which splat then
+    own. Compressed code (Rare's, Yaz0, RNC method 1) is unpacked first and appended to a working copy of the ROM, which splat then
     reads instead (and an export rebuilds). Returns the image splat reads."""
     import hashlib
 

src/decompnet/consoles/n64/rnc.py

@@ -0,0 +1,186 @@
+"""RNC method 1 containers: bounded Huffman/LZ unpacking of code held in a cartridge image.
+
+Control bits are little-endian words interleaved with literal bytes. The reader must discard prefetched literal
+bytes after each run, keeping only the unused control bits. Both container checksums are checked before code is used.
+"""
+
+from __future__ import annotations
+
+import functools
+import struct
+from dataclasses import dataclass
+
+HEADER_BYTES = 18
+MAX_OUTPUT = 4 << 20
+MAX_CODE_BITS = 16
+
+
+class RncError(ValueError):
+    pass
+
+
+@functools.cache
+def _crc_table() -> tuple[int, ...]:
+    table = []
+    for value in range(256):
+        for _ in range(8):
+            value = (value >> 1) ^ (0xA001 if value & 1 else 0)
+        table.append(value)
+    return tuple(table)
+
+
+def crc16(data: bytes) -> int:
+    value = 0
+    table = _crc_table()
+    for byte in data:
+        value = (value >> 8) ^ table[(value ^ byte) & 0xFF]
+    return value
+
+
+@dataclass
+class Literal:
+    packed: int
+    output: int
+    size: int
+
+
+class _Bits:
+    def __init__(self, payload: bytes):
+        self.data = payload + b"\0\0"
+        self.end = len(payload)
+        self.pos = 0
+        self.buffer = self.word(0)
+        self.left = 0
+
+    def word(self, pos: int) -> int:
+        if pos + 2 > len(self.data):
+            raise RncError("truncated RNC control bits")
+        return self.data[pos] | self.data[pos + 1] << 8
+
+    def take(self, count: int) -> int:
+        if not 0 <= count <= 30:
+            raise RncError("invalid RNC bit count")
+        if count > 16:
+            return self.take(16) | self.take(count - 16) << 16
+        if not count:
+            return 0
+        value = self.buffer & ((1 << count) - 1)
+        if count > self.left:
+            self.buffer >>= self.left
+            self.pos += 2
+            self.buffer |= self.word(self.pos) << 16
+            count -= self.left
+            self.left = 16
+        self.buffer >>= count
+        self.left -= count
+        return value
+
+    def literal(self, count: int) -> bytes:
+        if count < 0 or self.pos + count > self.end:
+            raise RncError("truncated RNC literal run")
+        result = self.data[self.pos:self.pos + count]
+        self.pos += count
+        self.buffer &= (1 << self.left) - 1
+        self.buffer |= self.word(self.pos) << self.left
+        return result
+
+
+def _table(bits: _Bits) -> list[tuple[int, int, int, int]]:
+    lengths = [bits.take(4) for _ in range(bits.take(5))]
+    entries = []
+    code = 0
+    for width in range(1, MAX_CODE_BITS + 1):
+        for symbol, length in enumerate(lengths):
+            if length != width:
+                continue
+            if code >= 1 << width:
+                raise RncError("oversubscribed RNC Huffman tree")
+            reversed_code = int(f"{code:0{width}b}"[::-1], 2)
+            entries.append(((1 << width) - 1, reversed_code, width, symbol))
+            code += 1
+        code <<= 1
+    return entries
+
+
+def _value(bits: _Bits, table: list[tuple[int, int, int, int]]) -> int:
+    for mask, code, width, symbol in table:
+        if bits.buffer & mask == code:
+            bits.take(width)
+            return symbol if symbol < 2 else (1 << (symbol - 1)) | bits.take(symbol - 1)
+    raise RncError("RNC stream has no matching Huffman code")
+
+
+def unpack(data: bytes, *, max_output: int = MAX_OUTPUT, literals: list[Literal] | None = None) -> bytes:
+    """Unpack a checksummed method-1 block. Optional literal locations support a verified compression template."""
+    if len(data) < HEADER_BYTES or data[:4] != b"RNC\x01":
+        raise RncError("not an RNC method-1 container")
+    size, packed, unpacked_crc, packed_crc = struct.unpack_from(">IIHH", data, 4)
+    if not 0 < size <= max_output or packed < 2 or HEADER_BYTES + packed > len(data):
+        raise RncError("invalid RNC sizes")
+    payload = data[HEADER_BYTES:HEADER_BYTES + packed]
+    if crc16(payload) != packed_crc:
+        raise RncError("RNC packed checksum differs")
+    bits = _Bits(payload)
+    if bits.take(2):
+        raise RncError("encrypted or locked RNC streams are unsupported")
+    out = bytearray()
+    chunks = 0
+    while len(out) < size:
+        raw, distance, length = _table(bits), _table(bits), _table(bits)
+        count = bits.take(16)
+        if not count:
+            raise RncError("empty RNC chunk")
+        for index in range(count):
+            n = _value(bits, raw)
+            if len(out) + n > size:
+                raise RncError("RNC literal run exceeds declared output")
+            if n:
+                if literals is not None:
+                    literals.append(Literal(HEADER_BYTES + bits.pos, len(out), n))
+                out += bits.literal(n)
+            if index + 1 < count:
+                offset, n = _value(bits, distance) + 1, _value(bits, length) + 2
+                if offset > len(out) or len(out) + n > size:
+                    raise RncError("RNC back-reference is outside its output")
+                for _ in range(n):
+                    out.append(out[-offset])  # a run can overlap its own already-decoded bytes
+        chunks += 1
+        if chunks > 255:
+            raise RncError("too many RNC chunks")
+    if crc16(out) != unpacked_crc:
+        raise RncError("RNC unpacked checksum differs")
+    return bytes(out)
+
+
+def repack_like(output: bytes, template: bytes) -> bytes:
+    """Reuse a valid stream's coding decisions, deriving its literal bytes from newly built output.
+
+    This is a matching-build operation, not a general compressor. Edits incompatible with the template's LZ
+    references are rejected by decoding the result and comparing every output byte.
+    """
+    locations: list[Literal] = []
+    original = unpack(template, literals=locations)
+    if len(output) != len(original):
+        raise RncError("RNC template requires the original output size")
+    packed = int.from_bytes(template[8:12], "big")
+    result = bytearray(template[:HEADER_BYTES + packed])
+    for span in locations:
+        result[span.packed:span.packed + span.size] = output[span.output:span.output + span.size]
+    struct.pack_into(">HH", result, 12, crc16(output), crc16(result[HEADER_BYTES:]))
+    if unpack(result) != output:
+        raise RncError("new output is incompatible with RNC template references")
+    return bytes(result)
+
+
+def rebuild_cartridge(analysis: bytes, original: bytes, blocks: list[dict]) -> bytes:
+    """Replace packed code with literals derived from a linked analysis image; preserve rebuilt prefix bytes."""
+    if len(analysis) < len(original):
+        raise RncError("analysis image is shorter than the cartridge")
+    result = bytearray(analysis[:len(original)])
+    for block in blocks:
+        offset, start, size = block['rom'], block['analysis'], block['size']
+        if not 0 <= offset <= len(original) - HEADER_BYTES or not len(original) <= start <= len(analysis) - size:
+            raise RncError("compressed-block mapping is outside its images")
+        packed = repack_like(analysis[start:start + size], original[offset:])
+        result[offset:offset + len(packed)] = packed
+    return bytes(result)

src/decompnet/rom/export.py

@@ -229,6 +229,8 @@ def export_rom(net, slug: str, out: Path, console, log=print) -> dict:
     for seg in segments:
         kind = seg.get("type")
         name = re.sub(r"\W", "_", str(seg.get("name") or f"{seg['start']:X}"))
+        if name[0].isdigit():
+            name = "seg_" + name
         if kind == "header" or (kind == "hasm" and "vram" not in seg):
             layout.append({"name": name, "kind": "raw", "start": seg["start"], "objects": [f"build/asm/{seg.get('name', 'header')}.o"]})
         elif kind == "bin":
@@ -281,6 +283,25 @@ def export_rom(net, slug: str, out: Path, console, log=print) -> dict:
         "assembly": objects, "segments": layout, "functions": fn_json,
         "symbols": {k: v for k, v in sorted(absolute.items(), key=lambda kv: kv[1])},
     }
+    # An expanded analysis image is not the cartridge. Rebuild RNC literals from linked code before reporting
+    # an original-ROM match; both the expanded image and the final cartridge must pass their comparisons.
+    original_name = cfg["rom"].get("program")
+    if console.id == "n64" and original_name and original_name != image_name:
+        from ..consoles.n64.code import unpack_code
+
+        original = (work / original_name).read_bytes()
+        blocks, cursor = [], len(original)
+        for offset, payload in unpack_code(original):
+            cursor += -cursor % 0x10 + 0x2000
+            if original[offset:offset + 4] != b"RNC\x01" or image[cursor:cursor + len(payload)] != payload:
+                raise ExportError("compressed analysis export needs a verified reconstruction for every block")
+            blocks.append({"rom": offset, "analysis": cursor, "size": len(payload)})
+            cursor += len(payload)
+        if not blocks:
+            raise ExportError("expanded analysis image has no verified compressed-block reconstruction")
+        cartridge = f"baserom.cartridge{ext}"
+        (out / cartridge).write_bytes(original)
+        build["cartridge"] = {"image": cartridge, "sha1": hashlib.sha1(original).hexdigest(), "blocks": blocks}
     (out / "build.json").write_text(json.dumps(build, indent=1), encoding="utf-8")
     shutil.copyfile(Path(__file__).with_name("export_build.py"), out / "build.py")
     units_json = []
@@ -294,6 +315,18 @@ def export_rom(net, slug: str, out: Path, console, log=print) -> dict:
                                       encoding="utf-8")
     matched = sum(1 for f in functions if f["kind"] == "c")
     _write_docs(out, proj["name"], console.name, base, matched, len(functions), as_fallback)
+    if cartridge := build.get("cartridge"):
+        readme = out / "README.md"
+        text = readme.read_text(encoding="utf-8")
+        text = text.replace(f"1. Put your own copy of the game here as `{base}` (it is in `.gitignore`; never commit it).",
+                            f"1. Keep both generated private inputs: `{base}` (expanded analysis image) and "
+                            f"`{cartridge['image']}` (original normalized cartridge). Reimport and export your ROM "
+                            "to regenerate them; the original cartridge alone is not the expanded analysis input.")
+        text += ("\nThe build first compares the expanded linked image, then reconstructs RNC method-1 blocks "
+                 "from that linked output using the original compression template. It checks the final cartridge "
+                 "against its fixed original checksum. Incompatible edits require a general compressor.\n")
+        readme.write_text(text, encoding="utf-8")
+
     # everything made from the game itself stays local: build.py regenerates it from your copy
     (out / ".gitignore").write_text("build/\nbaserom*\nasm/\nassets/\n*.ld\n*_auto.txt\n.splat/\n*.d\n",
                                     encoding="utf-8")

src/decompnet/rom/export_build.py

@@ -258,6 +258,24 @@ def linker_script(provides: dict[str, str], gaps: list[tuple[int, str]]) -> str:
     if B.get("gp"):
         lines.append(f"_gp = 0x{B['gp']:08X};")
     lines += [f"{name} = 0x{addr:08X};" for name, addr in B["symbols"].items()]
+    # The splitter can leave references into raw data or across local labels in separate function objects.
+    # Only generated address-bearing names can supply their own absolute address. PROVIDE keeps real definitions
+    # authoritative; arbitrary unresolved names still fail at link time and the whole-image comparison stays fixed.
+    from elftools.elf.elffile import ELFFile
+
+    auto_addresses = {}
+    for obj in set(provides.values()):
+        with open(ROOT / obj, "rb") as stream:
+            symtab = ELFFile(stream).get_section_by_name(".symtab")
+            for symbol in symtab.iter_symbols() if symtab else ():
+                if symbol["st_shndx"] != "SHN_UNDEF":
+                    continue
+                name = symbol.name
+                address = AUTO_NAME.match(name) or re.fullmatch(r"\.L([0-9A-Fa-f]{8})", name)
+                if address:
+                    auto_addresses[name] = int(address.group(1), 16)
+    lines += [f'PROVIDE("{name}" = 0x{address:08X});' for name, address in sorted(auto_addresses.items())]
+
     # jump tables in the data point at labels inside functions (.L001006F8); with one object per function those
     # labels are local to another object, so give them their addresses (which their names spell out)
     labels = set()
@@ -292,7 +310,19 @@ def build() -> bool:
     out, base = (ROOT / B["output"]).read_bytes(), (ROOT / B["image"]).read_bytes()
     c_count = sum(1 for fn in B["functions"] if c_source(fn))
     if out == base:
-        print(f"identical: {B['output']} matches {B['image']} byte for byte "
+        if cartridge := B.get("cartridge"):
+            from decompnet.consoles.n64.rnc import rebuild_cartridge
+
+            original = (ROOT / cartridge["image"]).read_bytes()
+            if hashlib.sha1(original).hexdigest() != cartridge["sha1"]:
+                raise ValueError("original cartridge checksum differs from the fixed export target")
+            out = rebuild_cartridge(out, original, cartridge["blocks"])
+            (ROOT / B["output"]).write_bytes(out)
+            if out != original:
+                print("differs: reconstructed cartridge does not match the original")
+                return False
+            print(f"cartridge verified: {cartridge['sha1']}")
+        print(f"identical: {B['output']} matches {B.get('cartridge', {}).get('image', B['image'])} byte for byte "
               f"({c_count} of {len(B['functions'])} functions in C)")
         return True
     at = next((i for i in range(min(len(out), len(base))) if out[i] != base[i]), min(len(out), len(base)))