update dtk (#2221)

2 files changed, 61 insertions, 21 deletions

diff --git a/tools/decompctx.py b/tools/decompctx.py
index 491ae205df..ea6a46977c 100755
--- a/tools/decompctx.py
+++ b/tools/decompctx.py
@@ -29,7 +29,7 @@ include_dirs = [
     # Add additional include directories here
 ]
 
-include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]$')
+include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]')
 guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$")
 
 defines = set()
diff --git a/tools/project.py b/tools/project.py
index 1d9c79bedd..b4469fcd46 100644
--- a/tools/project.py
+++ b/tools/project.py
@@ -17,7 +17,7 @@ import os
 import platform
 import sys
 from pathlib import Path
-from typing import IO, Any, Dict, Iterable, List, Optional, Set, Tuple, cast
+from typing import IO, Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast
 
 from . import ninja_syntax
 from .ninja_syntax import serialize_path
@@ -41,8 +41,9 @@ class Object:
             "asflags": None,
             "asm_dir": None,
             "cflags": None,
-            "extra_asflags": None,
-            "extra_cflags": None,
+            "extra_asflags": [],
+            "extra_cflags": [],
+            "extra_clang_flags": [],
             "host": None,
             "lib": None,
             "mw_version": None,
@@ -81,6 +82,20 @@ class Object:
         set_default("shift_jis", config.shift_jis)
         set_default("src_dir", config.src_dir)
 
+        # Validate progress categories
+        def check_category(category: str):
+            if not any(category == c.id for c in config.progress_categories):
+                sys.exit(
+                    f"Progress category '{category}' missing from config.progress_categories"
+                )
+
+        progress_category = obj.options["progress_category"]
+        if isinstance(progress_category, list):
+            for category in progress_category:
+                check_category(category)
+        elif progress_category is not None:
+            check_category(progress_category)
+
         # Resolve paths
         build_dir = config.out_path()
         obj.src_path = Path(obj.options["src_dir"]) / obj.options["source"]
@@ -158,15 +173,19 @@ class ProjectConfig:
         self.generate_compile_commands: bool = (
             True  # Generate compile_commands.json for clangd
         )
+        self.extra_clang_flags: List[str] = []  # Extra flags for clangd
 
         # Progress output, progress.json and report.json config
-        self.progress = True  # Enable progress output
+        self.progress = True  # Enable report.json generation and CLI progress output
         self.progress_all: bool = True  # Include combined "all" category
         self.progress_modules: bool = True  # Include combined "modules" category
         self.progress_each_module: bool = (
             False  # Include individual modules, disable for large numbers of modules
         )
         self.progress_categories: List[ProgressCategory] = []  # Additional categories
+        self.print_progress_categories: Union[bool, List[str]] = (
+            True  # Print additional progress categories in the CLI progress output
+        )
 
         # Progress fancy printing
         self.progress_use_fancy: bool = False
@@ -770,14 +789,11 @@ def generate_build_ninja(
 
             # Add appropriate language flag if it doesn't exist already
             # Added directly to the source so it flows to other generation tasks
-            if not any(flag.startswith("-lang") for flag in cflags) and (
-                extra_cflags is None
-                or not any(flag.startswith("-lang") for flag in extra_cflags)
+            if not any(flag.startswith("-lang") for flag in cflags) and not any(
+                flag.startswith("-lang") for flag in extra_cflags
             ):
                 # Ensure extra_cflags is a unique instance,
                 # and insert into there to avoid modifying shared sets of flags
-                if extra_cflags is None:
-                    extra_cflags = []
                 extra_cflags = obj.options["extra_cflags"] = list(extra_cflags)
                 if file_is_cpp(src_path):
                     extra_cflags.insert(0, "-lang=c++")
@@ -785,7 +801,7 @@ def generate_build_ninja(
                     extra_cflags.insert(0, "-lang=c")
 
             cflags_str = make_flags_str(cflags)
-            if extra_cflags is not None:
+            if len(extra_cflags) > 0:
                 extra_cflags_str = make_flags_str(extra_cflags)
                 cflags_str += " " + extra_cflags_str
             used_compiler_versions.add(obj.options["mw_version"])
@@ -843,7 +859,7 @@ def generate_build_ninja(
             if obj.options["asflags"] is None:
                 sys.exit("ProjectConfig.asflags missing")
             asflags_str = make_flags_str(obj.options["asflags"])
-            if obj.options["extra_asflags"] is not None:
+            if len(obj.options["extra_asflags"]) > 0:
                 extra_asflags_str = make_flags_str(obj.options["extra_asflags"])
                 asflags_str += " " + extra_asflags_str
 
@@ -1348,7 +1364,7 @@ def generate_objdiff_config(
             print(f"Missing scratch compiler mapping for {obj.options['mw_version']}")
         else:
             cflags_str = make_flags_str(cflags)
-            if obj.options["extra_cflags"] is not None:
+            if len(obj.options["extra_cflags"]) > 0:
                 extra_cflags_str = make_flags_str(obj.options["extra_cflags"])
                 cflags_str += " " + extra_cflags_str
             unit_config["scratch"] = {
@@ -1451,7 +1467,10 @@ def generate_compile_commands(
         "-I-",
         "-i-",
     }
-    CFLAG_IGNORE_PREFIX: Tuple[str, ...] = tuple()
+    CFLAG_IGNORE_PREFIX: Tuple[str, ...] = (
+        # Recursive includes are not supported by modern compilers
+        "-ir ",
+    )
 
     # Flags to replace
     CFLAG_REPLACE: Dict[str, str] = {}
@@ -1488,12 +1507,28 @@ def generate_compile_commands(
         (
             "-lang",
             {
-                "c": ("--language=c", "--std=c89"),
+                "c": ("--language=c", "--std=c99"),
                 "c99": ("--language=c", "--std=c99"),
                 "c++": ("--language=c++", "--std=c++98"),
                 "cplus": ("--language=c++", "--std=c++98"),
             },
         ),
+        # Enum size
+        (
+            "-enum",
+            {
+                "min": ("-fshort-enums",),
+                "int": ("-fno-short-enums",),
+            },
+        ),
+        # Common BSS
+        (
+            "-common",
+            {
+                "off": ("-fno-common",),
+                "on": ("-fcommon",),
+            },
+        ),
     )
 
     # Flags to pass through
@@ -1584,8 +1619,9 @@ def generate_compile_commands(
                     continue
 
         append_cflags(obj.options["cflags"])
-        if isinstance(obj.options["extra_cflags"], list):
-            append_cflags(obj.options["extra_cflags"])
+        append_cflags(obj.options["extra_cflags"])
+        cflags.extend(config.extra_clang_flags)
+        cflags.extend(obj.options["extra_clang_flags"])
 
         unit_config = {
             "directory": Path.cwd(),
@@ -1644,7 +1680,7 @@ def calculate_progress(config: ProjectConfig) -> None:
                 data[key] = int(value)
 
     convert_numbers(report_data["measures"])
-    for category in report_data["categories"]:
+    for category in report_data.get("categories", []):
         convert_numbers(category["measures"])
 
     # Output to GitHub Actions job summary, if available
@@ -1686,8 +1722,12 @@ def calculate_progress(config: ProjectConfig) -> None:
         )
 
     print_category("All", report_data["measures"])
-    for category in report_data["categories"]:
-        print_category(category["name"], category["measures"])
+    for category in report_data.get("categories", []):
+        if config.print_progress_categories is True or (
+            isinstance(config.print_progress_categories, list)
+            and category["id"] in config.print_progress_categories
+        ):
+            print_category(category["name"], category["measures"])
 
     if config.progress_use_fancy:
         measures = report_data["measures"]
@@ -1740,7 +1780,7 @@ def calculate_progress(config: ProjectConfig) -> None:
     else:
         # Support for old behavior where "dol" was the main category
         add_category("dol", report_data["measures"])
-    for category in report_data["categories"]:
+    for category in report_data.get("categories", []):
         add_category(category["id"], category["measures"])
 
     with open(out_path / "progress.json", "w", encoding="utf-8") as w: