Ports over OoT's version config (#1750)

* Port over version_config from OoT

* extract_text

* remove need for calling by python modules

* Remove extract audio comment

* Newline

* pyyaml

7 files changed, 193 insertions, 68 deletions

diff --git a/tools/buildtools/decompress_baserom.py b/tools/decompress_baserom.py
index 34d017e76..ee91d08eb 100755
--- a/tools/buildtools/decompress_baserom.py
+++ b/tools/decompress_baserom.py
@@ -16,7 +16,8 @@ import crunch64
 import ipl3checksum
 import zlib
 
-import dmadata
+from buildtools import dmadata
+from version import version_config
 
 
 def decompress_zlib(data: bytes) -> bytes:
@@ -174,7 +175,9 @@ def main():
 
     uncompressed_path = baserom_dir / "baserom-decompressed.z64"
 
-    dmadata_start = int((baserom_dir / "dmadata_start.txt").read_text(), 16)
+    config = version_config.load_version_config(version)
+
+    dmadata_start = config.dmadata_start
     correct_str_hash = (baserom_dir / "checksum.md5").read_text().split()[0]
 
     if check_existing_rom(uncompressed_path, correct_str_hash):
diff --git a/tools/audio_extraction.py b/tools/extract_audio.py
index 9479ed157..0b6b3339a 100644
--- a/tools/audio_extraction.py
+++ b/tools/extract_audio.py
@@ -7,6 +7,8 @@
 
 import argparse
 
+from version import version_config
+
 from audio.extraction.audio_extract import extract_audio_for_version, GameVersionInfo
 from audio.extraction.disassemble_sequence import MMLVersion, SequenceTableSpec, SqSection
 
@@ -20,12 +22,13 @@ if __name__ == '__main__':
 
     version = args.version
 
-    # TODO move these to config yaml
-    soundfont_table_code_offset, seq_font_table_code_offset, seq_table_code_offset, sample_bank_table_code_offset = {
-        "n64-jp-1.0"     : (0xC97130-0xB5F000, 0xC973C0-0xB5F000, 0xC975D0-0xB5F000, 0xC97DE0-0xB5F000),
-        "n64-us"         : (0xC776C0-0xB3C000, 0xC77960-0xB3C000, 0xC77B70-0xB3C000, 0xC78380-0xB3C000),
-        "n64-eu-1.1-dbg" : (0xE0F7E0-0xC95000, 0xE0FA80-0xC95000, 0xE0FC90-0xC95000, 0xE104A0-0xC95000),
-    }[version]
+    config = version_config.load_version_config(version)
+
+    code_vram = config.dmadata_segments["code"].vram
+    soundfont_table_code_offset = config.variables["gSoundFontTable"] - code_vram
+    seq_font_table_code_offset = config.variables["gSequenceFontTable"] - code_vram
+    seq_table_code_offset = config.variables["gSequenceTable"] - code_vram
+    sample_bank_table_code_offset = config.variables["gSampleBankTable"] - code_vram
 
     # List any sequences that are "handwritten", we don't extract these by
     # default as we want these checked in for documentation.
diff --git a/tools/buildtools/extract_baserom.py b/tools/extract_baserom.py
index 57062a05f..3e009e976 100755
--- a/tools/buildtools/extract_baserom.py
+++ b/tools/extract_baserom.py
@@ -9,7 +9,9 @@ import argparse
 from pathlib import Path
 import sys
 
-import dmadata
+from buildtools import dmadata
+from version import version_config
+
 
 
 def main():
@@ -25,26 +27,28 @@ def main():
         help="Output directory for segments",
     )
     parser.add_argument(
+        "-v",
+        "--version",
+        help="version to process",
+        default="n64-us",
+    )
+    parser.add_argument(
         "--dmadata-start",
         type=lambda s: int(s, 16),
-        required=True,
         help=(
-            "The dmadata location in the rom, as a hexadecimal offset (e.g. 0x12f70)"
+            "Override dmadata location for non-matching ROMs, as a hexadecimal offset (e.g. 0x12F70)"
         ),
     )
-    parser.add_argument(
-        "--dmadata-names",
-        type=Path,
-        required=True,
-        help="Path to file containing segment names",
-    )
 
     args = parser.parse_args()
 
     rom_data = memoryview(args.rom.read_bytes())
 
-    dma_names = args.dmadata_names.read_text().splitlines()
-    dma_entries = dmadata.read_dmadata(rom_data, args.dmadata_start)
+    config = version_config.load_version_config(args.version)
+    dmadata_start = args.dmadata_start or config.dmadata_start
+    dma_names = config.dmadata_segments.keys()
+
+    dma_entries = dmadata.read_dmadata(rom_data, dmadata_start)
     if len(dma_names) != len(dma_entries):
         print(
             f"Error: expected {len(dma_names)} DMA entries but found {len(dma_entries)} in ROM",
diff --git a/tools/extract_text.py b/tools/extract_text.py
new file mode 100644
index 000000000..d2d5ea247
--- /dev/null
+++ b/tools/extract_text.py
@@ -0,0 +1,38 @@
+import argparse
+from pathlib import Path
+
+from version import version_config
+from text import msgdis
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract text from the baserom into .h files")
+    parser.add_argument(
+        "baserom_segments_dir",
+        type=Path,
+        help="Directory of uncompressed ROM segments",
+    )
+    parser.add_argument(
+        "output_dir",
+        type=Path,
+        help="Output directory to place files in",
+    )
+    parser.add_argument(
+        "-v",
+        "--version",
+        help="version to process",
+        default="n64-us",
+    )
+    args = parser.parse_args()
+
+    baserom_segments_dir : Path = args.baserom_segments_dir
+    version : str = args.version
+    output_dir : Path = args.output_dir
+
+    config = version_config.load_version_config(version)
+
+    version_info = msgdis.GameVersionInfo(config.dmadata_segments["code"].vram, config.variables)
+
+    msgdis.extract(version_info, baserom_segments_dir, output_dir)
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/buildtools/extract_yars.py b/tools/extract_yars.py
index fac68a98f..7f6b3363a 100755
--- a/tools/buildtools/extract_yars.py
+++ b/tools/extract_yars.py
@@ -21,6 +21,7 @@ import dataclasses
 from pathlib import Path
 import struct
 
+from version import version_config
 
 PRINT_XML = False
 
@@ -121,15 +122,13 @@ def main():
     global PRINT_XML
     PRINT_XML = args.xml
 
-    archivesCsvPath = Path(f"tools/filelists/{version}/archives.csv")
+    config = version_config.load_version_config(version)
 
-    with archivesCsvPath.open() as f:
-        for line in f:
-            archiveName = line.strip().split(",")[1]
-            archivePath = baseromSegmentsDir / archiveName
+    for archiveName in config.archives:
+        archivePath = baseromSegmentsDir / archiveName
 
-            extractedPath = Path(str(archivePath) + ".unarchive")
-            extractArchive(archivePath, extractedPath)
+        extractedPath = Path(str(archivePath) + ".unarchive")
+        extractArchive(archivePath, extractedPath)
 
 
 if __name__ == "__main__":
diff --git a/tools/text/msgdis.py b/tools/text/msgdis.py
index f11f9afbd..eedbe53af 100755
--- a/tools/text/msgdis.py
+++ b/tools/text/msgdis.py
@@ -3,10 +3,20 @@
 #   message_data_static disassembler/decompiler
 #
 
-import argparse, re, struct
+import struct
 from pathlib import Path
 from typing import Callable, Dict, List, Optional, Tuple, TypeVar
 
+from dataclasses import dataclass
+
+@dataclass
+class GameVersionInfo:
+    # Vram location of the code segment
+    code_vram : int
+    # Variables dict from config
+    variables : Dict[str, int]
+
+
 T = TypeVar("T")
 
 item_ids = {
@@ -3054,9 +3064,8 @@ class MessageEntry:
 
         return out
 
-# TODO: Use version_config instead
 def collect_messages(message_tables : List[Optional[MessageTableDesc]], baserom_segments_dir : Path, 
-                     config : Dict[str,int], code_vram : int, code_bin : bytes):
+                     version_info : GameVersionInfo, code_vram : int, code_bin : bytes):
 
     messages : Dict[int,MessageEntry] = {}
 
@@ -3067,7 +3076,7 @@ def collect_messages(message_tables : List[Optional[MessageTableDesc]], baserom_
             continue
 
         baserom_seg = (baserom_segments_dir / desc.seg_name).read_bytes()
-        code_offset = config[desc.table_name] - code_vram
+        code_offset = version_info.variables[desc.table_name] - code_vram
 
         if desc.parent is None:
             # Complete table
@@ -3122,33 +3131,10 @@ def collect_messages(message_tables : List[Optional[MessageTableDesc]], baserom_
 
     return messages
 
-def main():
-    parser = argparse.ArgumentParser(description="Extract text from the baserom into .h files")
-    parser.add_argument(
-        "baserom_segments_dir",
-        type=Path,
-        help="Directory of uncompressed ROM segments",
-    )
-    parser.add_argument(
-        "output_dir",
-        type=Path,
-        help="Output directory to place files in",
-    )
-    parser.add_argument(
-        "-v",
-        "--version",
-        help="version to process",
-        default="n64-us",
-    )
-    args = parser.parse_args()
-
-    baserom_segments_dir : Path = args.baserom_segments_dir
-    output_dir : Path = args.output_dir
-
-    args.output_dir.mkdir(parents=True, exist_ok=True)
-
-    # TODO: use version config instead to get code vram
-    code_vram = 0x800A5AC0
+def extract(version_info: GameVersionInfo, baserom_segments_dir : Path, output_dir: Path):
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    code_vram = version_info.code_vram
 
     code_bin = (baserom_segments_dir / "code").read_bytes()
 
@@ -3158,17 +3144,11 @@ def main():
     message_tables : MessageTableDesc = [None for _ in range(1)] # EN
     message_table_staff : MessageTableDesc = None
 
-    message_tables[0]  = MessageTableDesc("sMessageTableNES",   "message_data_static", nes_decoder, None)
+    message_tables[0]  = MessageTableDesc("sMessageTableNES", "message_data_static", nes_decoder, None)
     message_table_staff = MessageTableDesc("sMessageTableCredits", "staff_message_data_static", credits_decoder, None)
 
-    # TODO: use version config instead
-    config = {
-        "sMessageTableNES": 0x801C6B98,
-        "sMessageTableCredits": 0x801CFB08,
-    }
-
-    messages = collect_messages(message_tables, baserom_segments_dir, config, code_vram, code_bin)
-    staff_messages = collect_messages([message_table_staff], baserom_segments_dir, config, code_vram, code_bin)
+    messages = collect_messages(message_tables, baserom_segments_dir, version_info, code_vram, code_bin)
+    staff_messages = collect_messages([message_table_staff], baserom_segments_dir, version_info, code_vram, code_bin)
 
     message_data = []
 
@@ -3184,5 +3164,3 @@ def main():
     (output_dir / "message_data.h").write_text(message_data)
     (output_dir / "message_data_staff.h").write_text(message_data_staff)
 
-if __name__ == "__main__":
-    main()
diff --git a/tools/version/version_config.py b/tools/version/version_config.py
new file mode 100644
index 000000000..761d0291c
--- /dev/null
+++ b/tools/version/version_config.py
@@ -0,0 +1,100 @@
+# Version-specific configuration for setup and assets extraction
+
+# SPDX-FileCopyrightText: © 2024 ZeldaRET
+# SPDX-License-Identifier: CC0-1.0
+
+from __future__ import annotations
+
+from collections import OrderedDict
+import csv
+import dataclasses
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+
+
+@dataclasses.dataclass
+class VersionConfig:
+    # Version name
+    version: str
+    # ROM offset to start of DMA table
+    dmadata_start: int
+    # DMA segment information, in ROM order
+    dmadata_segments: OrderedDict[str, SegmentInfo]
+    # ROM pieces that are copied directly into the build with .incbin
+    incbins: list[IncbinConfig]
+    # Addresses of important variables needed for asset extraction
+    variables: dict[str, int]
+    # Assets to extract
+    assets: list[AssetConfig]
+    # Archive names to extract
+    archives: list[str]
+
+
+@dataclasses.dataclass
+class SegmentInfo:
+    name: str
+    vram: Optional[int]
+
+
+@dataclasses.dataclass
+class IncbinConfig:
+    name: str
+    segment: str
+    vram: int
+    size: int
+
+
+@dataclasses.dataclass
+class AssetConfig:
+    name: str
+    xml_path: Path
+    start_offset: Optional[int]
+    end_offset: Optional[int]
+
+
+def load_dmadata_segments(version: str) -> OrderedDict[str, SegmentInfo]:
+    segments = OrderedDict()
+    with open(PROJECT_ROOT / f"baseroms/{version}/segments.csv", "r") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            name = row["Name"]
+            vram = int(row["VRAM start"], 16) if row["VRAM start"] else None
+            segments[name] = SegmentInfo(name, vram)
+    return segments
+
+
+def load_version_config(version: str) -> VersionConfig:
+    with open(PROJECT_ROOT / f"baseroms/{version}/config.yml", "r") as f:
+        config = yaml.load(f, Loader=yaml.Loader)
+
+    incbins = []
+    if config["incbins"] is not None:
+        for incbin in config["incbins"]:
+            incbins.append(
+                IncbinConfig(
+                    incbin["name"], incbin["segment"], incbin["vram"], incbin["size"]
+                )
+            )
+
+    assets = []
+    if config["assets"] is not None:
+        for asset in config["assets"]:
+            name = asset["name"]
+            xml_path = Path(asset["xml_path"])
+            start_offset = asset.get("start_offset", None)
+            end_offset = asset.get("end_offset", None)
+            assets.append(AssetConfig(name, xml_path, start_offset, end_offset))
+
+    return VersionConfig(
+        version=version,
+        dmadata_start=config["dmadata_start"],
+        dmadata_segments=load_dmadata_segments(version),
+        incbins=incbins,
+        variables=config["variables"],
+        assets=assets,
+        archives=config["archives"]
+    )