Update dtk-template, dtk, and objdiff

7 files changed, 207 insertions, 96 deletions

diff --git a/config/GZLE01/splits.txt b/config/GZLE01/splits.txt
index a1580e8e..46f3797c 100644
--- a/config/GZLE01/splits.txt
+++ b/config/GZLE01/splits.txt
@@ -141,9 +141,6 @@ m_Do/m_Do_machine_exception.cpp:
 	.rodata     start:0x803396F8 end:0x803398D8
 	.sbss       start:0x803F69B0 end:0x803F69B8
 
-dolphin/os/__start.c:
-	.init       start:0x80003100 end:0x800033A8
-
 c/c_damagereaction.cpp:
 	.text       start:0x8001BEDC end:0x800227A0
 	.ctors      start:0x803386A4 end:0x803386A8
@@ -1704,9 +1701,6 @@ d/d_wpot_water.cpp:
 	.sbss       start:0x803F7300 end:0x803F7308
 	.sdata2     start:0x803FBB00 end:0x803FBB20
 
-PowerPC_EABI_Support/Runtime/Src/__mem.c:
-	.init       start:0x800033A8 end:0x800034E0
-
 DynamicLink.cpp:
 	.text       start:0x80240098 end:0x80241278
 	.text       start:0x80241278 end:0x802412C4
@@ -4008,6 +4002,12 @@ TRK_MINNOW_DOLPHIN/Portable/notify.c:
 TRK_MINNOW_DOLPHIN/ppc/Generic/flush_cache.c:
 	.text       start:0x80335690 end:0x803356C8
 
+dolphin/os/__start.c:
+	.init       start:0x80003100 end:0x800033A8
+
+PowerPC_EABI_Support/Runtime/Src/__mem.c:
+	.init       start:0x800033A8 end:0x800034E0
+
 TRK_MINNOW_DOLPHIN/Portable/mem_TRK.c:
 	.init       start:0x800034E0 end:0x80003534
 	.text       start:0x803356C8 end:0x80335784
diff --git a/config/GZLJ01/splits.txt b/config/GZLJ01/splits.txt
index 9ff6e37b..aab43c2a 100644
--- a/config/GZLJ01/splits.txt
+++ b/config/GZLJ01/splits.txt
@@ -140,9 +140,6 @@ m_Do/m_Do_machine_exception.cpp:
 	.rodata     start:0x80337158 end:0x80337338
 	.sbss       start:0x803E9E90 end:0x803E9E98
 
-dolphin/os/__start.c:
-	.init       start:0x80003100 end:0x800033A8
-
 c/c_damagereaction.cpp:
 	.text       start:0x8001BB70 end:0x80022430
 	.ctors      start:0x803360A0 end:0x803360A4
@@ -1704,9 +1701,6 @@ d/d_wpot_water.cpp:
 	.sbss       start:0x803EA7A0 end:0x803EA7A8
 	.sdata2     start:0x803EEFD8 end:0x803EEFF8
 
-PowerPC_EABI_Support/Runtime/Src/__mem.c:
-	.init       start:0x800033A8 end:0x800034E0
-
 DynamicLink.cpp:
 	.text       start:0x8023D828 end:0x8023EA08
 	.text       start:0x8023EA08 end:0x8023EA54
@@ -4004,6 +3998,12 @@ TRK_MINNOW_DOLPHIN/Portable/notify.c:
 TRK_MINNOW_DOLPHIN/ppc/Generic/flush_cache.c:
 	.text       start:0x80333094 end:0x803330CC
 
+dolphin/os/__start.c:
+	.init       start:0x80003100 end:0x800033A8
+
+PowerPC_EABI_Support/Runtime/Src/__mem.c:
+	.init       start:0x800033A8 end:0x800034E0
+
 TRK_MINNOW_DOLPHIN/Portable/mem_TRK.c:
 	.init       start:0x800034E0 end:0x80003534
 	.text       start:0x803330CC end:0x80333188
diff --git a/config/GZLP01/splits.txt b/config/GZLP01/splits.txt
index 2ecc37b7..8153c1ee 100644
--- a/config/GZLP01/splits.txt
+++ b/config/GZLP01/splits.txt
@@ -141,9 +141,6 @@ m_Do/m_Do_machine_exception.cpp:
 	.rodata     start:0x8033EC90 end:0x8033EE70
 	.sbss       start:0x803FE1B0 end:0x803FE1B8
 
-dolphin/os/__start.c:
-	.init       start:0x80003100 end:0x800033A8
-
 c/c_damagereaction.cpp:
 	.text       start:0x8001C22C end:0x80022AF0
 	.ctors      start:0x8033DBE4 end:0x8033DBE8
@@ -1704,9 +1701,6 @@ d/d_wpot_water.cpp:
 	.sbss       start:0x803FEB20 end:0x803FEB28
 	.sdata2     start:0x80403350 end:0x80403370
 
-PowerPC_EABI_Support/Runtime/Src/__mem.c:
-	.init       start:0x800033A8 end:0x800034E0
-
 DynamicLink.cpp:
 	.text       start:0x80245238 end:0x80246418
 	.text       start:0x80246418 end:0x80246464
@@ -4008,6 +4002,12 @@ TRK_MINNOW_DOLPHIN/Portable/notify.c:
 TRK_MINNOW_DOLPHIN/ppc/Generic/flush_cache.c:
 	.text       start:0x8033ABE8 end:0x8033AC20
 
+dolphin/os/__start.c:
+	.init       start:0x80003100 end:0x800033A8
+
+PowerPC_EABI_Support/Runtime/Src/__mem.c:
+	.init       start:0x800033A8 end:0x800034E0
+
 TRK_MINNOW_DOLPHIN/Portable/mem_TRK.c:
 	.init       start:0x800034E0 end:0x80003534
 	.text       start:0x8033AC20 end:0x8033ACDC
diff --git a/configure.py b/configure.py
index f5049c0e..f8561fe4 100644
--- a/configure.py
+++ b/configure.py
@@ -151,8 +151,8 @@ if args.no_asm:
 # Tool versions
 config.binutils_tag = "2.42-1"
 config.compilers_tag = "20240706"
-config.dtk_tag = "v1.3.0"
-config.objdiff_tag = "v2.4.0"
+config.dtk_tag = "v1.4.0"
+config.objdiff_tag = "v2.7.1"
 config.sjiswrap_tag = "v1.2.0"
 config.wibo_tag = "0.6.11"
 
@@ -1799,6 +1799,23 @@ if config_path.exists():
             emit_build_rule(asset)
 
 
+# Optional callback to adjust link order. This can be used to add, remove, or reorder objects.
+# This is called once per module, with the module ID and the current link order.
+#
+# For example, this adds "dummy.c" to the end of the DOL link order if configured with --non-matching.
+# "dummy.c" *must* be configured as a Matching (or Equivalent) object in order to be linked.
+def link_order_callback(module_id: int, objects: List[str]) -> List[str]:
+    # Don't modify the link order for matching builds
+    if not config.non_matching:
+        return objects
+    if module_id == 0:  # DOL
+        return objects + ["dummy.c"]
+    return objects
+
+# Uncomment to enable the link order callback.
+# config.link_order_callback = link_order_callback
+
+
 # Optional extra categories for progress tracking
 config.progress_categories = [
     ProgressCategory("game", "TWW Game Code"),
diff --git a/tools/decompctx.py b/tools/decompctx.py
index f31ade12..f2f31dfa 100755
--- a/tools/decompctx.py
+++ b/tools/decompctx.py
@@ -18,63 +18,63 @@ from typing import List
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.abspath(os.path.join(script_dir, ".."))
 src_dir = os.path.join(root_dir, "src")
-include_dirs = [
-    os.path.join(root_dir, "include"),
-    os.path.join(root_dir, "src"),
-    os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Include"),
-    os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Include"),
-    os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Include"),
-    os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C++/MSL_Common/Include"),
-    os.path.join(root_dir, "src/PowerPC_EABI_Support/Runtime/Inc"),
-]
-
-include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]')
-guard_pattern = re.compile(r"^#ifndef\s+(.*)$")
+include_dirs: List[str] = []  # Set with -I flag
+
+include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]')
+guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$")
+once_pattern = re.compile(r"^#\s*pragma\s+once$")
 
 defines = set()
+deps = []
 
 
-def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str:
+def import_h_file(in_file: str, r_path: str) -> str:
     rel_path = os.path.join(root_dir, r_path, in_file)
     if os.path.exists(rel_path):
-        return import_c_file(rel_path, deps)
+        return import_c_file(rel_path)
     for include_dir in include_dirs:
         inc_path = os.path.join(include_dir, in_file)
         if os.path.exists(inc_path):
-            return import_c_file(inc_path, deps)
+            return import_c_file(inc_path)
     else:
         print("Failed to locate", in_file)
         return ""
 
 
-def import_c_file(in_file: str, deps: List[str]) -> str:
+def import_c_file(in_file: str) -> str:
     in_file = os.path.relpath(in_file, root_dir)
     deps.append(in_file)
     out_text = ""
 
     try:
         with open(in_file, encoding="utf-8") as file:
-            out_text += process_file(in_file, list(file), deps)
+            out_text += process_file(in_file, list(file))
     except Exception:
         with open(in_file) as file:
-            out_text += process_file(in_file, list(file), deps)
+            out_text += process_file(in_file, list(file))
     return out_text
 
 
-def process_file(in_file: str, lines: List[str], deps: List[str]) -> str:
+def process_file(in_file: str, lines: List[str]) -> str:
     out_text = ""
     for idx, line in enumerate(lines):
-        guard_match = guard_pattern.match(line.strip())
         if idx == 0:
+            guard_match = guard_pattern.match(line.strip())
             if guard_match:
                 if guard_match[1] in defines:
                     break
                 defines.add(guard_match[1])
+            else:
+                once_match = once_pattern.match(line.strip())
+                if once_match:
+                    if in_file in defines:
+                        break
+                    defines.add(in_file)
             print("Processing file", in_file)
         include_match = include_pattern.match(line.strip())
         if include_match and not include_match[1].endswith(".s"):
             out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n'
-            out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps)
+            out_text += import_h_file(include_match[1], os.path.dirname(in_file))
             out_text += f'/* end "{include_match[1]}" */\n'
         else:
             out_text += line
@@ -105,10 +105,19 @@ def main():
         "--depfile",
         help="""Dependency file""",
     )
+    parser.add_argument(
+        "-I",
+        "--include",
+        help="""Include directory""",
+        action="append",
+    )
     args = parser.parse_args()
 
-    deps = []
-    output = import_c_file(args.c_file, deps)
+    if args.include is None:
+        exit("No include directories specified")
+    global include_dirs
+    include_dirs = args.include
+    output = import_c_file(args.c_file)
 
     with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f:
         f.write(output)
diff --git a/tools/download_tool.py b/tools/download_tool.py
index f4512d01..7adbf0d7 100755
--- a/tools/download_tool.py
+++ b/tools/download_tool.py
@@ -91,6 +91,21 @@ TOOLS: Dict[str, Callable[[str], str]] = {
     "wibo": wibo_url,
 }
 
+def download(url, response, output) -> None:
+    if url.endswith(".zip"):
+        data = io.BytesIO(response.read())
+        with zipfile.ZipFile(data) as f:
+            f.extractall(output)
+        # Make all files executable
+        for root, _, files in os.walk(output):
+            for name in files:
+                os.chmod(os.path.join(root, name), 0o755)
+        output.touch(mode=0o755)  # Update dir modtime
+    else:
+        with open(output, "wb") as f:
+            shutil.copyfileobj(response, f)
+        st = os.stat(output)
+        os.chmod(output, st.st_mode | stat.S_IEXEC)
 
 def main() -> None:
     parser = argparse.ArgumentParser()
@@ -104,22 +119,21 @@ def main() -> None:
 
     print(f"Downloading {url} to {output}")
     req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
-    with urllib.request.urlopen(req) as response:
-        if url.endswith(".zip"):
-            data = io.BytesIO(response.read())
-            with zipfile.ZipFile(data) as f:
-                f.extractall(output)
-            # Make all files executable
-            for root, _, files in os.walk(output):
-                for name in files:
-                    os.chmod(os.path.join(root, name), 0o755)
-            output.touch(mode=0o755)  # Update dir modtime
-        else:
-            with open(output, "wb") as f:
-                shutil.copyfileobj(response, f)
-            st = os.stat(output)
-            os.chmod(output, st.st_mode | stat.S_IEXEC)
-
+    try:
+        with urllib.request.urlopen(req) as response:
+            download(url, response, output)
+    except urllib.error.URLError as e:
+        if str(e).find("CERTIFICATE_VERIFY_FAILED") == -1:
+            raise e
+        try:
+            import certifi
+            import ssl
+        except:
+            print("\"certifi\" module not found. Please install it using \"python -m pip install certifi\".")
+            return
+            
+        with urllib.request.urlopen(req, context=ssl.create_default_context(cafile=certifi.where())) as response:
+            download(url, response, output)
 
 if __name__ == "__main__":
     main()
diff --git a/tools/project.py b/tools/project.py
index f1539647..bb282674 100644
--- a/tools/project.py
+++ b/tools/project.py
@@ -17,7 +17,20 @@ import os
 import platform
 import sys
 from pathlib import Path
-from typing import IO, Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast
+from typing import (
+    Any,
+    Callable,
+    cast,
+    Dict,
+    IO,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    TypedDict,
+    Union,
+)
 
 from . import ninja_syntax
 from .ninja_syntax import serialize_path
@@ -176,7 +189,12 @@ class ProjectConfig:
             True  # Generate compile_commands.json for clangd
         )
         self.extra_clang_flags: List[str] = []  # Extra flags for clangd
-        self.scratch_preset_id: Optional[int] = None  # Default decomp.me preset ID for scratches
+        self.scratch_preset_id: Optional[int] = (
+            None  # Default decomp.me preset ID for scratches
+        )
+        self.link_order_callback: Optional[Callable[[int, List[str]], List[str]]] = (
+            None  # Callback to add/remove/reorder units within a module
+        )
 
         # Progress output, progress.json and report.json config
         self.progress = True  # Enable report.json generation and CLI progress output
@@ -292,10 +310,38 @@ def make_flags_str(flags: Optional[List[str]]) -> str:
     return " ".join(flags)
 
 
+# Unit configuration
+class BuildConfigUnit(TypedDict):
+    object: Optional[str]
+    name: str
+    autogenerated: bool
+
+
+# Module configuration
+class BuildConfigModule(TypedDict):
+    name: str
+    module_id: int
+    ldscript: str
+    entry: str
+    units: List[BuildConfigUnit]
+
+
+# Module link configuration
+class BuildConfigLink(TypedDict):
+    modules: List[str]
+
+
+# Build configuration generated by decomp-toolkit
+class BuildConfig(BuildConfigModule):
+    version: str
+    modules: List[BuildConfigModule]
+    links: List[BuildConfigLink]
+
+
 # Load decomp-toolkit generated config.json
 def load_build_config(
     config: ProjectConfig, build_config_path: Path
-) -> Optional[Dict[str, Any]]:
+) -> Optional[BuildConfig]:
     if not build_config_path.is_file():
         return None
 
@@ -303,7 +349,7 @@ def load_build_config(
         return tuple(map(int, (v.split("."))))
 
     f = open(build_config_path, "r", encoding="utf-8")
-    build_config: Dict[str, Any] = json.load(f)
+    build_config: BuildConfig = json.load(f)
     config_version = build_config.get("version")
     if config_version is None:
         print("Invalid config.json, regenerating...")
@@ -319,6 +365,24 @@ def load_build_config(
         return None
 
     f.close()
+
+    # Apply link order callback
+    if config.link_order_callback:
+        modules: List[BuildConfigModule] = [build_config, *build_config["modules"]]
+        for module in modules:
+            unit_names = list(map(lambda u: u["name"], module["units"]))
+            unit_names = config.link_order_callback(module["module_id"], unit_names)
+            units: List[BuildConfigUnit] = []
+            for unit_name in unit_names:
+                units.append(
+                    # Find existing unit or create a new one
+                    next(
+                        (u for u in module["units"] if u["name"] == unit_name),
+                        {"object": None, "name": unit_name, "autogenerated": False},
+                    )
+                )
+            module["units"] = units
+
     return build_config
 
 
@@ -336,7 +400,7 @@ def generate_build(config: ProjectConfig) -> None:
 def generate_build_ninja(
     config: ProjectConfig,
     objects: Dict[str, Object],
-    build_config: Optional[Dict[str, Any]],
+    build_config: Optional[BuildConfig],
 ) -> None:
     out = io.StringIO()
     n = ninja_syntax.Writer(out)
@@ -380,7 +444,7 @@ def generate_build_ninja(
     decompctx = config.tools_dir / "decompctx.py"
     n.rule(
         name="decompctx",
-        command=f"$python {decompctx} $in -o $out -d $out.d",
+        command=f"$python {decompctx} $in -o $out -d $out.d $includes",
         description="CTX $in",
         depfile="$out.d",
         deps="gcc",
@@ -697,9 +761,9 @@ def generate_build_ninja(
         return path.parent / (path.name + ".MAP")
 
     class LinkStep:
-        def __init__(self, config: Dict[str, Any]) -> None:
-            self.name: str = config["name"]
-            self.module_id: int = config["module_id"]
+        def __init__(self, config: BuildConfigModule) -> None:
+            self.name = config["name"]
+            self.module_id = config["module_id"]
             self.ldscript: Optional[Path] = Path(config["ldscript"])
             self.entry = config["entry"]
             self.inputs: List[str] = []
@@ -809,10 +873,8 @@ def generate_build_ninja(
                 else:
                     extra_cflags.insert(0, "-lang=c")
 
-            cflags_str = make_flags_str(cflags)
-            if len(extra_cflags) > 0:
-                extra_cflags_str = make_flags_str(extra_cflags)
-                cflags_str += " " + extra_cflags_str
+            all_cflags = cflags + extra_cflags
+            cflags_str = make_flags_str(all_cflags)
             used_compiler_versions.add(obj.options["mw_version"])
 
             # Add MWCC build rule
@@ -836,11 +898,21 @@ def generate_build_ninja(
 
             # Add ctx build rule
             if obj.ctx_path is not None:
+                include_dirs = []
+                for flag in all_cflags:
+                    if (
+                        flag.startswith("-i ")
+                        or flag.startswith("-I ")
+                        or flag.startswith("-I+")
+                    ):
+                        include_dirs.append(flag[3:])
+                includes = " ".join([f"-I {d}" for d in include_dirs])
                 n.build(
                     outputs=obj.ctx_path,
                     rule="decompctx",
                     inputs=src_path,
                     implicit=decompctx,
+                    variables={"includes": includes},
                 )
 
             # Add host build rule
@@ -897,13 +969,14 @@ def generate_build_ninja(
 
             return obj_path
 
-        def add_unit(build_obj, link_step: LinkStep):
+        def add_unit(build_obj: BuildConfigUnit, link_step: LinkStep):
             obj_path, obj_name = build_obj["object"], build_obj["name"]
             obj = objects.get(obj_name)
             if obj is None:
                 if config.warn_missing_config and not build_obj["autogenerated"]:
                     print(f"Missing configuration for {obj_name}")
-                link_step.add(obj_path)
+                if obj_path is not None:
+                    link_step.add(Path(obj_path))
                 return
 
             link_built_obj = obj.completed
@@ -932,12 +1005,7 @@ def generate_build_ninja(
                 link_step.add(built_obj_path)
             elif obj_path is not None:
                 # Use the original (extracted) object
-                link_step.add(obj_path)
-            else:
-                lib_name = obj.options["lib"]
-                sys.exit(
-                    f"Missing object for {obj_name}: {obj.src_path} {lib_name} {obj}"
-                )
+                link_step.add(Path(obj_path))
 
         # Add DOL link step
         link_step = LinkStep(build_config)
@@ -1258,7 +1326,7 @@ def generate_build_ninja(
 def generate_objdiff_config(
     config: ProjectConfig,
     objects: Dict[str, Object],
-    build_config: Optional[Dict[str, Any]],
+    build_config: Optional[BuildConfig],
 ) -> None:
     if build_config is None:
         return
@@ -1323,7 +1391,7 @@ def generate_objdiff_config(
     }
 
     def add_unit(
-        build_obj: Dict[str, Any], module_name: str, progress_categories: List[str]
+        build_obj: BuildConfigUnit, module_name: str, progress_categories: List[str]
     ) -> None:
         obj_path, obj_name = build_obj["object"], build_obj["name"]
         base_object = Path(obj_name).with_suffix("")
@@ -1358,9 +1426,21 @@ def generate_objdiff_config(
             unit_config["base_path"] = obj.src_obj_path
             unit_config["metadata"]["source_path"] = obj.src_path
 
-        cflags = obj.options["cflags"]
+        # Filter out include directories
+        def keep_flag(flag):
+            return (
+                not flag.startswith("-i ")
+                and not flag.startswith("-i-")
+                and not flag.startswith("-I ")
+                and not flag.startswith("-I+")
+                and not flag.startswith("-I-")
+            )
+
+        all_cflags = list(
+            filter(keep_flag, obj.options["cflags"] + obj.options["extra_cflags"])
+        )
         reverse_fn_order = False
-        for flag in cflags:
+        for flag in all_cflags:
             if not flag.startswith("-inline "):
                 continue
             for value in flag.split(" ")[1].split(","):
@@ -1369,20 +1449,11 @@ def generate_objdiff_config(
                 elif value == "nodeferred":
                     reverse_fn_order = False
 
-        # Filter out include directories
-        def keep_flag(flag):
-            return not flag.startswith("-i ") and not flag.startswith("-I ")
-
-        cflags = list(filter(keep_flag, cflags))
-
         compiler_version = COMPILER_MAP.get(obj.options["mw_version"])
         if compiler_version is None:
             print(f"Missing scratch compiler mapping for {obj.options['mw_version']}")
         else:
-            cflags_str = make_flags_str(cflags)
-            if len(obj.options["extra_cflags"]) > 0:
-                extra_cflags_str = make_flags_str(obj.options["extra_cflags"])
-                cflags_str += " " + extra_cflags_str
+            cflags_str = make_flags_str(all_cflags)
             unit_config["scratch"] = {
                 "platform": "gc_wii",
                 "compiler": compiler_version,
@@ -1468,7 +1539,7 @@ def generate_objdiff_config(
 def generate_compile_commands(
     config: ProjectConfig,
     objects: Dict[str, Object],
-    build_config: Optional[Dict[str, Any]],
+    build_config: Optional[BuildConfig],
 ) -> None:
     if build_config is None or not config.generate_compile_commands:
         return
@@ -1557,7 +1628,7 @@ def generate_compile_commands(
 
     clangd_config = []
 
-    def add_unit(build_obj: Dict[str, Any]) -> None:
+    def add_unit(build_obj: BuildConfigUnit) -> None:
         obj = objects.get(build_obj["name"])
         if obj is None:
             return