path: root/tools/new_extract_assets.py

#!/usr/bin/env python3

from io import BufferedReader
import os
import sys
import math
import json
import shutil
import struct
import tempfile
import subprocess
import traceback

# Maps asset names to temporary files containing the raw asset data
# Only really useful for asset groups that use shared palettes
# Otherwise, its a little overkill
raw_asset_files = dict()
assets_requested = set()

file_open = []

def get_asset_raw(baserom, asset):
    global raw_asset_files
    global assets_requested

    asset_name = asset["name"]
    # If available, return a saved file handle to the asset
    if asset_name in raw_asset_files:
        if raw_asset_files[asset_name].closed:
            raw_asset_files[asset_name] = open(raw_asset_files[asset_name].name, "rb")
        raw_asset_files[asset_name].seek(0)
        return raw_asset_files[asset_name]

    # Otherwise, extract it
    raw_file = extract_asset(baserom, asset)

    # If the asset has been requested before, save the file handle
    if asset_name in assets_requested:
        raw_asset_files[asset_name] = raw_file
    # Otherwise, just mark the asset as being requested once before
    else:
        assets_requested.add(asset_name)

    return raw_file

def extract_asset(baserom:BufferedReader, asset):
    rom_offset = int(asset["rom_offset"], 16)
    if baserom.closed:
        baserom = open(baserom.name, "rb")
    baserom.seek(rom_offset)
    asset_magic = baserom.read(4)

    if asset_magic == b"MIO0":
        asset_block = extract_mio0_block(baserom, asset)
    elif asset_magic == b"TKMK":
        asset_block = extract_tkmk_block(baserom, asset)
    else: # Raw asset, no compression
        # This is silly
        asset_block = baserom
        asset_block.seek(rom_offset)

    return asset_from_block(asset_block, asset)

def asset_from_block(asset_block, asset):
    if asset["type"] in { "ia1" }:
        asset_size = ((asset["width"] * asset["height"]) + 7) // 8
    elif asset["type"] in { "ci4", "ia4", "i4" }:
        asset_size = ((asset["width"] * asset["height"]) + 1) // 2
    elif asset["type"] in { "ci8", "ia8", "i8" }:
        asset_size = asset["width"] * asset["height"]
    elif asset["type"] in { "ia16", "rgba16" }:
        asset_size = asset["width"] * asset["height"] * 2
    elif asset["type"] in { "rgba32" }:
        asset_size = asset["width"] * asset["height"] * 4
    elif asset["type"] in { "bin" }:
        asset_size = asset["size"]
    else:
        print("TODO: raise an exception here")
        return None

    # If not specified, assume the asset begins at the start of the block
    block_offset = int(asset.get("block_offset", "0x0"), 16)
    # The SEEK_CUR here is incredibly silly, but its done to make the
    # similarly silly raw block handling work.
    # For MIO0 and TKMK this should make no difference
    asset_block.seek(block_offset, os.SEEK_CUR)
    asset_data = asset_block.read(asset_size)
    asset_block.close()
    asset_file = tempfile.NamedTemporaryFile(mode="wb", prefix="raw_asset_", delete=False)
    file_open.append(asset_file.name)
    asset_file.write(asset_data)
    asset_file.flush()

    return asset_file

# This dict maps ROM offsets to file-like objects. This way we don't have to keep
# decompressing/collecting the same block of MIO0 data from the baserom
mio0_blocks = dict()
mio0s_requested = set()

def extract_mio0_block(baserom, asset):
    global mio0_blocks
    global mio0s_requested

    rom_offset = asset["rom_offset"]
    if rom_offset in mio0_blocks:
        if mio0_blocks[rom_offset].closed:
            mio0_blocks[rom_offset] = open(mio0_blocks[rom_offset].name, "rb")
        mio0_blocks[rom_offset].seek(0)
        return mio0_blocks[rom_offset]

    mio0_file = tempfile.NamedTemporaryFile(prefix="mio0_block_",delete=False)
    file_open.append(mio0_file.name)
    mio0_file.close()
    subprocess.run(
        [
            "./tools/mio0",
            "-d",
            "-o", rom_offset,
            baserom.name,
            mio0_file.name,
        ],
        check=True
    )
    mio0_file = open(mio0_file.name, "rb")
    if rom_offset in mio0s_requested:
        mio0_blocks[rom_offset] = mio0_file
    else:
        mio0s_requested.add(rom_offset)

    return mio0_file

def extract_tkmk_block(baserom, asset):
    rom_offset = asset["rom_offset"]
    alpha = asset.get("alpha", "0x01")
    tkmk_block = tempfile.NamedTemporaryFile(mode="rb", prefix="tkmk_block_", delete=False)
    file_open.append(tkmk_block.name)
    subprocess.run(
        [
            "./tools/tkmk00",
            "-d",
            "-o", rom_offset,
            "-a", alpha,
            baserom.name,
            tkmk_block.name,
        ],
        check=True,
    )

    return tkmk_block

def export_stitched_palette(baserom:BufferedReader, asset, asset_list):
    palette_raw = tempfile.NamedTemporaryFile(mode="wb", prefix="stitched_palette", delete=False)
    file_open.append(palette_raw.name)

    # For stitched palettes, expect the "tlut" key to be a list of strings
    for tlut_name in asset["tlut"]:
        tlut = asset_list[tlut_name]
        tlut_file = get_asset_raw(baserom, tlut)

        with open(tlut_file.name, "rb") as f:
            shutil.copyfileobj(f, palette_raw)
    palette_raw.flush()

    output_dir = os.path.join(asset["output_dir"], "stitched_palettes")
    stitched_file_name = os.path.join(output_dir, f'{asset["name"]}_stitched_palette.png')

    os.makedirs(output_dir, exist_ok=True)

    cmd = [
        "./tools/n64graphics",
        "-e", palette_raw.name,
        "-g", stitched_file_name,
        # While hard coding stuff like this is usually bad, its justifiable here
        "-f", "rgba16",
        "-w", "16",
        "-h", "16",
    ]
    subprocess.run(cmd, check=True)

    return palette_raw

def export_wheel_mask(asset_file, asset):
    output_dir = os.path.join(asset["output_dir"], "wheel_masks")
    wheel_mask_file_name = os.path.join(output_dir, f'{asset["name"]}_wheel_mask.raw')

    os.makedirs(output_dir, exist_ok=True)

    with open(asset_file.name, "rb") as f, open(wheel_mask_file_name, mode="wb") as wheel_mask:
        for index in f.read():
            # This is super-duper specific to MK64
            if index < 0xC0:
                wheel_mask.write((0).to_bytes(1, "big"))
            else:
                wheel_mask.write((1).to_bytes(1, "big"))

def export_image(baserom, asset, asset_list):
    asset_file = get_asset_raw(baserom, asset)

    asset_png_filename = os.path.join(asset["output_dir"], f'{asset["name"]}.png')

    os.makedirs(asset["output_dir"], exist_ok=True)

    cmd = [
        "./tools/n64graphics",
        "-e", asset_file.name,
        "-g", asset_png_filename,
        "-f", asset["type"],
        "-w", str(asset["width"]),
        "-h", str(asset["height"]),
    ]

    if asset["type"] in ("ci4", "ci8"):
        # This is hightly specific to MK64:
        # Some TLUT images have a "stitched palette". This means that their palette is actually
        # two smaller palettes stitched together.
        if "stitched_palette" in asset.get("meta", set()):
            palette_file = export_stitched_palette(baserom, asset, asset_list)

            # If the image uses a "stitched palette" we need to create a "wheel mask" file too
            export_wheel_mask(asset_file, asset)

        # The palette is "normal", expect the "tlut" key to be a single string
        else:
            tlut = asset_list[asset["tlut"]]
            palette_file = get_asset_raw(baserom, tlut)

        cmd.extend([
            "-c", "rgba16", # I don't think anything other than rgba16 is even valid for N64 stuff...
            "-p", palette_file.name
        ])
        palette_file.close()

    asset_file.close()
    subprocess.run(cmd, check=True)

def export_bin(baserom, asset):
    asset_filename = os.path.join(asset["output_dir"], f'{asset["name"]}.{asset["type"]}')
    os.makedirs(asset["output_dir"], exist_ok=True)

    with open(asset_filename, "wb") as asset_file:
        baserom.seek(int(asset["rom_offset"], 16))
        asset_data = baserom.read(int(asset["size"], 16))
        asset_file.write(asset_data)

# TODO: use a proper argument parser
baserom_name = sys.argv[1]
# really, this should be a list of json files, that way we can just do $< in the Makefile
assest_json_file = sys.argv[2]

image_types = { "rgba16", "rgba32", "ci4", "ci8", "i4", "i8", "ia4", "ia8", "ia16", "ia1" }
# Types that extracted as-is from the ROM. No decompression or converting, just rip the bytes out of the ROM
# Hopefully some day in the future we'll be able to properly extract m64 types (music of one variety or another)
raw_types = { "bin", "m64" }

try:
    with open(baserom_name, "rb") as baserom, open(assest_json_file, "r") as json_file:
        asset_list = json.load(json_file)

        for asset_name, asset in asset_list.items():
            # Kind of silly, but makes assets easier to work with
            asset["name"] = asset_name
            # All output directories are relative to the directory the json file resides in
            asset["output_dir"] = os.path.join(os.path.dirname(os.path.realpath(assest_json_file)), asset["output_dir"])

        for asset_name, asset in asset_list.items():
            if asset["type"] in image_types:
                export_image(baserom, asset, asset_list)
            elif asset["type"] in raw_types:
                export_bin(baserom, asset)
            else:
                # Should we raise here?
                print(f"Unexpected asset type {0}", asset["type"])
    for f in file_open:
        os.remove(f)
except Exception as e:
    print(traceback.format_exc())
    exit(1)