diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/decompctx.py | 55 | ||||
| -rwxr-xr-x | tools/download_tool.py | 46 | ||||
| -rw-r--r-- | tools/project.py | 145 |
3 files changed, 170 insertions, 76 deletions
diff --git a/tools/decompctx.py b/tools/decompctx.py index f31ade12..f2f31dfa 100755 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -18,63 +18,63 @@ from typing import List script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.abspath(os.path.join(script_dir, "..")) src_dir = os.path.join(root_dir, "src") -include_dirs = [ - os.path.join(root_dir, "include"), - os.path.join(root_dir, "src"), - os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Include"), - os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Include"), - os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Include"), - os.path.join(root_dir, "src/PowerPC_EABI_Support/MSL/MSL_C++/MSL_Common/Include"), - os.path.join(root_dir, "src/PowerPC_EABI_Support/Runtime/Inc"), -] - -include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]') -guard_pattern = re.compile(r"^#ifndef\s+(.*)$") +include_dirs: List[str] = [] # Set with -I flag + +include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]') +guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$") +once_pattern = re.compile(r"^#\s*pragma\s+once$") defines = set() +deps = [] -def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str: +def import_h_file(in_file: str, r_path: str) -> str: rel_path = os.path.join(root_dir, r_path, in_file) if os.path.exists(rel_path): - return import_c_file(rel_path, deps) + return import_c_file(rel_path) for include_dir in include_dirs: inc_path = os.path.join(include_dir, in_file) if os.path.exists(inc_path): - return import_c_file(inc_path, deps) + return import_c_file(inc_path) else: print("Failed to locate", in_file) return "" -def import_c_file(in_file: str, deps: List[str]) -> str: +def import_c_file(in_file: str) -> str: in_file = os.path.relpath(in_file, root_dir) deps.append(in_file) out_text = "" try: with open(in_file, encoding="utf-8") as file: - out_text += process_file(in_file, list(file), deps) + out_text += process_file(in_file, list(file)) except Exception: with open(in_file) as file: - out_text += process_file(in_file, list(file), deps) + out_text += process_file(in_file, list(file)) return out_text -def process_file(in_file: str, lines: List[str], deps: List[str]) -> str: +def process_file(in_file: str, lines: List[str]) -> str: out_text = "" for idx, line in enumerate(lines): - guard_match = guard_pattern.match(line.strip()) if idx == 0: + guard_match = guard_pattern.match(line.strip()) if guard_match: if guard_match[1] in defines: break defines.add(guard_match[1]) + else: + once_match = once_pattern.match(line.strip()) + if once_match: + if in_file in defines: + break + defines.add(in_file) print("Processing file", in_file) include_match = include_pattern.match(line.strip()) if include_match and not include_match[1].endswith(".s"): out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n' - out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps) + out_text += import_h_file(include_match[1], os.path.dirname(in_file)) out_text += f'/* end "{include_match[1]}" */\n' else: out_text += line @@ -105,10 +105,19 @@ def main(): "--depfile", help="""Dependency file""", ) + parser.add_argument( + "-I", + "--include", + help="""Include directory""", + action="append", + ) args = parser.parse_args() - deps = [] - output = import_c_file(args.c_file, deps) + if args.include is None: + exit("No include directories specified") + global include_dirs + include_dirs = args.include + output = import_c_file(args.c_file) with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f: f.write(output) diff --git a/tools/download_tool.py b/tools/download_tool.py index f4512d01..7adbf0d7 100755 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -91,6 +91,21 @@ TOOLS: Dict[str, Callable[[str], str]] = { "wibo": wibo_url, } +def download(url, response, output) -> None: + if url.endswith(".zip"): + data = io.BytesIO(response.read()) + with zipfile.ZipFile(data) as f: + f.extractall(output) + # Make all files executable + for root, _, files in os.walk(output): + for name in files: + os.chmod(os.path.join(root, name), 0o755) + output.touch(mode=0o755) # Update dir modtime + else: + with open(output, "wb") as f: + shutil.copyfileobj(response, f) + st = os.stat(output) + os.chmod(output, st.st_mode | stat.S_IEXEC) def main() -> None: parser = argparse.ArgumentParser() @@ -104,22 +119,21 @@ def main() -> None: print(f"Downloading {url} to {output}") req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) - with urllib.request.urlopen(req) as response: - if url.endswith(".zip"): - data = io.BytesIO(response.read()) - with zipfile.ZipFile(data) as f: - f.extractall(output) - # Make all files executable - for root, _, files in os.walk(output): - for name in files: - os.chmod(os.path.join(root, name), 0o755) - output.touch(mode=0o755) # Update dir modtime - else: - with open(output, "wb") as f: - shutil.copyfileobj(response, f) - st = os.stat(output) - os.chmod(output, st.st_mode | stat.S_IEXEC) - + try: + with urllib.request.urlopen(req) as response: + download(url, response, output) + except urllib.error.URLError as e: + if str(e).find("CERTIFICATE_VERIFY_FAILED") == -1: + raise e + try: + import certifi + import ssl + except: + print("\"certifi\" module not found. Please install it using \"python -m pip install certifi\".") + return + + with urllib.request.urlopen(req, context=ssl.create_default_context(cafile=certifi.where())) as response: + download(url, response, output) if __name__ == "__main__": main() diff --git a/tools/project.py b/tools/project.py index f1539647..bb282674 100644 --- a/tools/project.py +++ b/tools/project.py @@ -17,7 +17,20 @@ import os import platform import sys from pathlib import Path -from typing import IO, Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast +from typing import ( + Any, + Callable, + cast, + Dict, + IO, + Iterable, + List, + Optional, + Set, + Tuple, + TypedDict, + Union, +) from . import ninja_syntax from .ninja_syntax import serialize_path @@ -176,7 +189,12 @@ class ProjectConfig: True # Generate compile_commands.json for clangd ) self.extra_clang_flags: List[str] = [] # Extra flags for clangd - self.scratch_preset_id: Optional[int] = None # Default decomp.me preset ID for scratches + self.scratch_preset_id: Optional[int] = ( + None # Default decomp.me preset ID for scratches + ) + self.link_order_callback: Optional[Callable[[int, List[str]], List[str]]] = ( + None # Callback to add/remove/reorder units within a module + ) # Progress output, progress.json and report.json config self.progress = True # Enable report.json generation and CLI progress output @@ -292,10 +310,38 @@ def make_flags_str(flags: Optional[List[str]]) -> str: return " ".join(flags) +# Unit configuration +class BuildConfigUnit(TypedDict): + object: Optional[str] + name: str + autogenerated: bool + + +# Module configuration +class BuildConfigModule(TypedDict): + name: str + module_id: int + ldscript: str + entry: str + units: List[BuildConfigUnit] + + +# Module link configuration +class BuildConfigLink(TypedDict): + modules: List[str] + + +# Build configuration generated by decomp-toolkit +class BuildConfig(BuildConfigModule): + version: str + modules: List[BuildConfigModule] + links: List[BuildConfigLink] + + # Load decomp-toolkit generated config.json def load_build_config( config: ProjectConfig, build_config_path: Path -) -> Optional[Dict[str, Any]]: +) -> Optional[BuildConfig]: if not build_config_path.is_file(): return None @@ -303,7 +349,7 @@ def load_build_config( return tuple(map(int, (v.split(".")))) f = open(build_config_path, "r", encoding="utf-8") - build_config: Dict[str, Any] = json.load(f) + build_config: BuildConfig = json.load(f) config_version = build_config.get("version") if config_version is None: print("Invalid config.json, regenerating...") @@ -319,6 +365,24 @@ def load_build_config( return None f.close() + + # Apply link order callback + if config.link_order_callback: + modules: List[BuildConfigModule] = [build_config, *build_config["modules"]] + for module in modules: + unit_names = list(map(lambda u: u["name"], module["units"])) + unit_names = config.link_order_callback(module["module_id"], unit_names) + units: List[BuildConfigUnit] = [] + for unit_name in unit_names: + units.append( + # Find existing unit or create a new one + next( + (u for u in module["units"] if u["name"] == unit_name), + {"object": None, "name": unit_name, "autogenerated": False}, + ) + ) + module["units"] = units + return build_config @@ -336,7 +400,7 @@ def generate_build(config: ProjectConfig) -> None: def generate_build_ninja( config: ProjectConfig, objects: Dict[str, Object], - build_config: Optional[Dict[str, Any]], + build_config: Optional[BuildConfig], ) -> None: out = io.StringIO() n = ninja_syntax.Writer(out) @@ -380,7 +444,7 @@ def generate_build_ninja( decompctx = config.tools_dir / "decompctx.py" n.rule( name="decompctx", - command=f"$python {decompctx} $in -o $out -d $out.d", + command=f"$python {decompctx} $in -o $out -d $out.d $includes", description="CTX $in", depfile="$out.d", deps="gcc", @@ -697,9 +761,9 @@ def generate_build_ninja( return path.parent / (path.name + ".MAP") class LinkStep: - def __init__(self, config: Dict[str, Any]) -> None: - self.name: str = config["name"] - self.module_id: int = config["module_id"] + def __init__(self, config: BuildConfigModule) -> None: + self.name = config["name"] + self.module_id = config["module_id"] self.ldscript: Optional[Path] = Path(config["ldscript"]) self.entry = config["entry"] self.inputs: List[str] = [] @@ -809,10 +873,8 @@ def generate_build_ninja( else: extra_cflags.insert(0, "-lang=c") - cflags_str = make_flags_str(cflags) - if len(extra_cflags) > 0: - extra_cflags_str = make_flags_str(extra_cflags) - cflags_str += " " + extra_cflags_str + all_cflags = cflags + extra_cflags + cflags_str = make_flags_str(all_cflags) used_compiler_versions.add(obj.options["mw_version"]) # Add MWCC build rule @@ -836,11 +898,21 @@ def generate_build_ninja( # Add ctx build rule if obj.ctx_path is not None: + include_dirs = [] + for flag in all_cflags: + if ( + flag.startswith("-i ") + or flag.startswith("-I ") + or flag.startswith("-I+") + ): + include_dirs.append(flag[3:]) + includes = " ".join([f"-I {d}" for d in include_dirs]) n.build( outputs=obj.ctx_path, rule="decompctx", inputs=src_path, implicit=decompctx, + variables={"includes": includes}, ) # Add host build rule @@ -897,13 +969,14 @@ def generate_build_ninja( return obj_path - def add_unit(build_obj, link_step: LinkStep): + def add_unit(build_obj: BuildConfigUnit, link_step: LinkStep): obj_path, obj_name = build_obj["object"], build_obj["name"] obj = objects.get(obj_name) if obj is None: if config.warn_missing_config and not build_obj["autogenerated"]: print(f"Missing configuration for {obj_name}") - link_step.add(obj_path) + if obj_path is not None: + link_step.add(Path(obj_path)) return link_built_obj = obj.completed @@ -932,12 +1005,7 @@ def generate_build_ninja( link_step.add(built_obj_path) elif obj_path is not None: # Use the original (extracted) object - link_step.add(obj_path) - else: - lib_name = obj.options["lib"] - sys.exit( - f"Missing object for {obj_name}: {obj.src_path} {lib_name} {obj}" - ) + link_step.add(Path(obj_path)) # Add DOL link step link_step = LinkStep(build_config) @@ -1258,7 +1326,7 @@ def generate_build_ninja( def generate_objdiff_config( config: ProjectConfig, objects: Dict[str, Object], - build_config: Optional[Dict[str, Any]], + build_config: Optional[BuildConfig], ) -> None: if build_config is None: return @@ -1323,7 +1391,7 @@ def generate_objdiff_config( } def add_unit( - build_obj: Dict[str, Any], module_name: str, progress_categories: List[str] + build_obj: BuildConfigUnit, module_name: str, progress_categories: List[str] ) -> None: obj_path, obj_name = build_obj["object"], build_obj["name"] base_object = Path(obj_name).with_suffix("") @@ -1358,9 +1426,21 @@ def generate_objdiff_config( unit_config["base_path"] = obj.src_obj_path unit_config["metadata"]["source_path"] = obj.src_path - cflags = obj.options["cflags"] + # Filter out include directories + def keep_flag(flag): + return ( + not flag.startswith("-i ") + and not flag.startswith("-i-") + and not flag.startswith("-I ") + and not flag.startswith("-I+") + and not flag.startswith("-I-") + ) + + all_cflags = list( + filter(keep_flag, obj.options["cflags"] + obj.options["extra_cflags"]) + ) reverse_fn_order = False - for flag in cflags: + for flag in all_cflags: if not flag.startswith("-inline "): continue for value in flag.split(" ")[1].split(","): @@ -1369,20 +1449,11 @@ def generate_objdiff_config( elif value == "nodeferred": reverse_fn_order = False - # Filter out include directories - def keep_flag(flag): - return not flag.startswith("-i ") and not flag.startswith("-I ") - - cflags = list(filter(keep_flag, cflags)) - compiler_version = COMPILER_MAP.get(obj.options["mw_version"]) if compiler_version is None: print(f"Missing scratch compiler mapping for {obj.options['mw_version']}") else: - cflags_str = make_flags_str(cflags) - if len(obj.options["extra_cflags"]) > 0: - extra_cflags_str = make_flags_str(obj.options["extra_cflags"]) - cflags_str += " " + extra_cflags_str + cflags_str = make_flags_str(all_cflags) unit_config["scratch"] = { "platform": "gc_wii", "compiler": compiler_version, @@ -1468,7 +1539,7 @@ def generate_objdiff_config( def generate_compile_commands( config: ProjectConfig, objects: Dict[str, Object], - build_config: Optional[Dict[str, Any]], + build_config: Optional[BuildConfig], ) -> None: if build_config is None or not config.generate_compile_commands: return @@ -1557,7 +1628,7 @@ def generate_compile_commands( clangd_config = [] - def add_unit(build_obj: Dict[str, Any]) -> None: + def add_unit(build_obj: BuildConfigUnit) -> None: obj = objects.get(build_obj["name"]) if obj is None: return |