summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorelijah-thomas774 <elijahthomas774@gmail.com>2024-06-10 00:09:01 -0400
committerelijah-thomas774 <elijahthomas774@gmail.com>2024-06-10 00:09:01 -0400
commitd433cda9e2d70df71c8bff8275edc80ffd3f97b6 (patch)
tree0d25c001be8121d5e4b42c1c276a5b63beb0246c /tools
parent6fca1925f551c59e90a75a7e627687dac01e3664 (diff)
sync with dtk-template
Diffstat (limited to 'tools')
-rw-r--r--tools/decompctx.py101
-rw-r--r--tools/download_tool.py43
-rw-r--r--tools/ninja_syntax.py241
-rw-r--r--tools/project.py799
-rw-r--r--tools/transform_dep.py6
5 files changed, 758 insertions, 432 deletions
diff --git a/tools/decompctx.py b/tools/decompctx.py
index c42b1828..e86d5ef3 100644
--- a/tools/decompctx.py
+++ b/tools/decompctx.py
@@ -13,60 +13,74 @@
import argparse
import os
import re
+from typing import List
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.abspath(os.path.join(script_dir, ".."))
src_dir = os.path.join(root_dir, "src")
-include_dir = os.path.join(root_dir, "include")
+include_dirs = [
+ os.path.join(root_dir, "include"),
+ # Add additional include directories here
+]
include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$')
-guard_pattern = re.compile(r'^#ifndef\s+(.*)$')
+guard_pattern = re.compile(r"^#ifndef\s+(.*)$")
defines = set()
-def import_h_file(in_file: str, r_path: str) -> str:
+
+def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str:
rel_path = os.path.join(root_dir, r_path, in_file)
- inc_path = os.path.join(include_dir, in_file)
if os.path.exists(rel_path):
- return import_c_file(rel_path)
- elif os.path.exists(inc_path):
- return import_c_file(inc_path)
+ return import_c_file(rel_path, deps)
+ for include_dir in include_dirs:
+ inc_path = os.path.join(include_dir, in_file)
+ if os.path.exists(inc_path):
+ return import_c_file(inc_path, deps)
else:
- print("Failed to locate", in_file)
- exit(1)
+ print("Failed to locate", in_file)
+ return ""
+
-def import_c_file(in_file) -> str:
+def import_c_file(in_file: str, deps: List[str]) -> str:
in_file = os.path.relpath(in_file, root_dir)
- out_text = ''
+ deps.append(in_file)
+ out_text = ""
try:
- with open(in_file, encoding="utf-8") as file:
- out_text += process_file(in_file, list(file))
+ with open(in_file, encoding="utf-8") as file:
+ out_text += process_file(in_file, list(file), deps)
except Exception:
- with open(in_file) as file:
- out_text += process_file(in_file, list(file))
+ with open(in_file) as file:
+ out_text += process_file(in_file, list(file), deps)
return out_text
-def process_file(in_file: str, lines) -> str:
- out_text = ''
+
+def process_file(in_file: str, lines: List[str], deps: List[str]) -> str:
+ out_text = ""
for idx, line in enumerate(lines):
- guard_match = guard_pattern.match(line.strip())
- if idx == 0:
- if guard_match:
- if guard_match[1] in defines:
- break
- defines.add(guard_match[1])
- print("Processing file", in_file)
- include_match = include_pattern.match(line.strip())
- if include_match and not include_match[1].endswith(".s"):
- out_text += f"/* \"{in_file}\" line {idx} \"{include_match[1]}\" */\n"
- out_text += import_h_file(include_match[1], os.path.dirname(in_file))
- out_text += f"/* end \"{include_match[1]}\" */\n"
- else:
- out_text += line
+ guard_match = guard_pattern.match(line.strip())
+ if idx == 0:
+ if guard_match:
+ if guard_match[1] in defines:
+ break
+ defines.add(guard_match[1])
+ print("Processing file", in_file)
+ include_match = include_pattern.match(line.strip())
+ if include_match and not include_match[1].endswith(".s"):
+ out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n'
+ out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps)
+ out_text += f'/* end "{include_match[1]}" */\n'
+ else:
+ out_text += line
return out_text
+
+def sanitize_path(path: str) -> str:
+ return path.replace("\\", "/").replace(" ", "\\ ")
+
+
def main():
parser = argparse.ArgumentParser(
description="""Create a context file which can be used for decomp.me"""
@@ -75,13 +89,32 @@ def main():
"c_file",
help="""File from which to create context""",
)
+ parser.add_argument(
+ "-o",
+ "--output",
+ help="""Output file""",
+ default="ctx.c",
+ )
+ parser.add_argument(
+ "-d",
+ "--depfile",
+ help="""Dependency file""",
+ )
args = parser.parse_args()
- output = import_c_file(args.c_file)
+ deps = []
+ output = import_c_file(args.c_file, deps)
- with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f:
+ with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f:
f.write(output)
+ if args.depfile:
+ with open(os.path.join(root_dir, args.depfile), "w", encoding="utf-8") as f:
+ f.write(sanitize_path(args.output) + ":")
+ for dep in deps:
+ path = sanitize_path(dep)
+ f.write(f" \\\n\t{path}")
+
if __name__ == "__main__":
- main() \ No newline at end of file
+ main()
diff --git a/tools/download_tool.py b/tools/download_tool.py
index fef42d6e..7b386a4b 100644
--- a/tools/download_tool.py
+++ b/tools/download_tool.py
@@ -18,11 +18,29 @@ import shutil
import stat
import urllib.request
import zipfile
-
+from typing import Callable, Dict
from pathlib import Path
-def dtk_url(tag):
+def binutils_url(tag):
+ uname = platform.uname()
+ system = uname.system.lower()
+ arch = uname.machine.lower()
+ if system == "darwin":
+ system = "macos"
+ arch = "universal"
+ elif arch == "amd64":
+ arch = "x86_64"
+
+ repo = "https://github.com/encounter/gc-wii-binutils"
+ return f"{repo}/releases/download/{tag}/{system}-{arch}.zip"
+
+
+def compilers_url(tag: str) -> str:
+ return f"https://files.decomp.dev/compilers_{tag}.zip"
+
+
+def dtk_url(tag: str) -> str:
uname = platform.uname()
suffix = ""
system = uname.system.lower()
@@ -38,29 +56,26 @@ def dtk_url(tag):
return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}"
-def sjiswrap_url(tag):
+def sjiswrap_url(tag: str) -> str:
repo = "https://github.com/encounter/sjiswrap"
return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe"
-def wibo_url(tag):
+def wibo_url(tag: str) -> str:
repo = "https://github.com/decompals/wibo"
return f"{repo}/releases/download/{tag}/wibo"
-def compilers_url(tag):
- return f"https://files.decomp.dev/compilers_{tag}.zip"
-
-
-TOOLS = {
+TOOLS: Dict[str, Callable[[str], str]] = {
+ "binutils": binutils_url,
+ "compilers": compilers_url,
"dtk": dtk_url,
"sjiswrap": sjiswrap_url,
"wibo": wibo_url,
- "compilers": compilers_url,
}
-def main():
+def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("tool", help="Tool name")
parser.add_argument("output", type=Path, help="output file path")
@@ -77,7 +92,11 @@ def main():
data = io.BytesIO(response.read())
with zipfile.ZipFile(data) as f:
f.extractall(output)
- output.touch(mode=0o755)
+ # Make all files executable
+ for root, _, files in os.walk(output):
+ for name in files:
+ os.chmod(os.path.join(root, name), 0o755)
+ output.touch(mode=0o755) # Update dir modtime
else:
with open(output, "wb") as f:
shutil.copyfileobj(response, f)
diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py
index e73b21a4..7306ee1d 100644
--- a/tools/ninja_syntax.py
+++ b/tools/ninja_syntax.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python
-
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,82 +21,129 @@ use Python.
import re
import textwrap
+import os
+from io import StringIO
+from pathlib import Path
+from typing import Dict, List, Match, Optional, Tuple, Union
+
+NinjaPath = Union[str, Path]
+NinjaPaths = Union[
+ List[str],
+ List[Path],
+ List[NinjaPath],
+ List[Optional[str]],
+ List[Optional[Path]],
+ List[Optional[NinjaPath]],
+]
+NinjaPathOrPaths = Union[NinjaPath, NinjaPaths]
+
+
+def escape_path(word: str) -> str:
+ return word.replace("$ ", "$$ ").replace(" ", "$ ").replace(":", "$:")
-def escape_path(word):
- return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:')
class Writer(object):
- def __init__(self, output, width=78):
+ def __init__(self, output: StringIO, width: int = 78) -> None:
self.output = output
self.width = width
- def newline(self):
- self.output.write('\n')
-
- def comment(self, text):
- for line in textwrap.wrap(text, self.width - 2, break_long_words=False,
- break_on_hyphens=False):
- self.output.write('# ' + line + '\n')
-
- def variable(self, key, value, indent=0):
- if value is None:
- return
- if isinstance(value, list):
- value = ' '.join(filter(None, value)) # Filter out empty strings.
- self._line('%s = %s' % (key, value), indent)
-
- def pool(self, name, depth):
- self._line('pool %s' % name)
- self.variable('depth', depth, indent=1)
-
- def rule(self, name, command, description=None, depfile=None,
- generator=False, pool=None, restat=False, rspfile=None,
- rspfile_content=None, deps=None):
- self._line('rule %s' % name)
- self.variable('command', command, indent=1)
+ def newline(self) -> None:
+ self.output.write("\n")
+
+ def comment(self, text: str) -> None:
+ for line in textwrap.wrap(
+ text, self.width - 2, break_long_words=False, break_on_hyphens=False
+ ):
+ self.output.write("# " + line + "\n")
+
+ def variable(
+ self,
+ key: str,
+ value: Optional[NinjaPathOrPaths],
+ indent: int = 0,
+ ) -> None:
+ value = " ".join(serialize_paths(value))
+ self._line("%s = %s" % (key, value), indent)
+
+ def pool(self, name: str, depth: int) -> None:
+ self._line("pool %s" % name)
+ self.variable("depth", str(depth), indent=1)
+
+ def rule(
+ self,
+ name: str,
+ command: str,
+ description: Optional[str] = None,
+ depfile: Optional[NinjaPath] = None,
+ generator: bool = False,
+ pool: Optional[str] = None,
+ restat: bool = False,
+ rspfile: Optional[NinjaPath] = None,
+ rspfile_content: Optional[NinjaPath] = None,
+ deps: Optional[NinjaPathOrPaths] = None,
+ ) -> None:
+ self._line("rule %s" % name)
+ self.variable("command", command, indent=1)
if description:
- self.variable('description', description, indent=1)
+ self.variable("description", description, indent=1)
if depfile:
- self.variable('depfile', depfile, indent=1)
+ self.variable("depfile", depfile, indent=1)
if generator:
- self.variable('generator', '1', indent=1)
+ self.variable("generator", "1", indent=1)
if pool:
- self.variable('pool', pool, indent=1)
+ self.variable("pool", pool, indent=1)
if restat:
- self.variable('restat', '1', indent=1)
+ self.variable("restat", "1", indent=1)
if rspfile:
- self.variable('rspfile', rspfile, indent=1)
+ self.variable("rspfile", rspfile, indent=1)
if rspfile_content:
- self.variable('rspfile_content', rspfile_content, indent=1)
+ self.variable("rspfile_content", rspfile_content, indent=1)
if deps:
- self.variable('deps', deps, indent=1)
-
- def build(self, outputs, rule, inputs=None, implicit=None, order_only=None,
- variables=None, implicit_outputs=None, pool=None, dyndep=None):
- outputs = as_list(outputs)
+ self.variable("deps", deps, indent=1)
+
+ def build(
+ self,
+ outputs: NinjaPathOrPaths,
+ rule: str,
+ inputs: Optional[NinjaPathOrPaths] = None,
+ implicit: Optional[NinjaPathOrPaths] = None,
+ order_only: Optional[NinjaPathOrPaths] = None,
+ variables: Optional[
+ Union[
+ List[Tuple[str, Optional[NinjaPathOrPaths]]],
+ Dict[str, Optional[NinjaPathOrPaths]],
+ ]
+ ] = None,
+ implicit_outputs: Optional[NinjaPathOrPaths] = None,
+ pool: Optional[str] = None,
+ dyndep: Optional[NinjaPath] = None,
+ ) -> List[str]:
+ outputs = serialize_paths(outputs)
out_outputs = [escape_path(x) for x in outputs]
- all_inputs = [escape_path(x) for x in as_list(inputs)]
+ all_inputs = [escape_path(x) for x in serialize_paths(inputs)]
if implicit:
- implicit = [escape_path(x) for x in as_list(implicit)]
- all_inputs.append('|')
- all_inputs.extend(implicit)
+ implicit = [escape_path(x) for x in serialize_paths(implicit)]
+ all_inputs.append("|")
+ all_inputs.extend(map(str, implicit))
if order_only:
- order_only = [escape_path(x) for x in as_list(order_only)]
- all_inputs.append('||')
- all_inputs.extend(order_only)
+ order_only = [escape_path(x) for x in serialize_paths(order_only)]
+ all_inputs.append("||")
+ all_inputs.extend(map(str, order_only))
if implicit_outputs:
- implicit_outputs = [escape_path(x)
- for x in as_list(implicit_outputs)]
- out_outputs.append('|')
- out_outputs.extend(implicit_outputs)
-
- self._line('build %s: %s' % (' '.join(out_outputs),
- ' '.join([rule] + all_inputs)))
+ implicit_outputs = [
+ escape_path(x) for x in serialize_paths(implicit_outputs)
+ ]
+ out_outputs.append("|")
+ out_outputs.extend(map(str, implicit_outputs))
+
+ self._line(
+ "build %s: %s" % (" ".join(out_outputs), " ".join([rule] + all_inputs))
+ )
if pool is not None:
- self._line(' pool = %s' % pool)
+ self._line(" pool = %s" % pool)
if dyndep is not None:
- self._line(' dyndep = %s' % dyndep)
+ self._line(" dyndep = %s" % serialize_path(dyndep))
if variables:
if isinstance(variables, dict):
@@ -111,89 +156,99 @@ class Writer(object):
return outputs
- def include(self, path):
- self._line('include %s' % path)
+ def include(self, path: str) -> None:
+ self._line("include %s" % path)
- def subninja(self, path):
- self._line('subninja %s' % path)
+ def subninja(self, path: str) -> None:
+ self._line("subninja %s" % path)
- def default(self, paths):
- self._line('default %s' % ' '.join(as_list(paths)))
+ def default(self, paths: NinjaPathOrPaths) -> None:
+ self._line("default %s" % " ".join(serialize_paths(paths)))
- def _count_dollars_before_index(self, s, i):
+ def _count_dollars_before_index(self, s: str, i: int) -> int:
"""Returns the number of '$' characters right in front of s[i]."""
dollar_count = 0
dollar_index = i - 1
- while dollar_index > 0 and s[dollar_index] == '$':
+ while dollar_index > 0 and s[dollar_index] == "$":
dollar_count += 1
dollar_index -= 1
return dollar_count
- def _line(self, text, indent=0):
+ def _line(self, text: str, indent: int = 0) -> None:
"""Write 'text' word-wrapped at self.width characters."""
- leading_space = ' ' * indent
+ leading_space = " " * indent
while len(leading_space) + len(text) > self.width:
# The text is too wide; wrap if possible.
# Find the rightmost space that would obey our width constraint and
# that's not an escaped space.
- available_space = self.width - len(leading_space) - len(' $')
+ available_space = self.width - len(leading_space) - len(" $")
space = available_space
while True:
- space = text.rfind(' ', 0, space)
- if (space < 0 or
- self._count_dollars_before_index(text, space) % 2 == 0):
+ space = text.rfind(" ", 0, space)
+ if space < 0 or self._count_dollars_before_index(text, space) % 2 == 0:
break
if space < 0:
# No such space; just use the first unescaped space we can find.
space = available_space - 1
while True:
- space = text.find(' ', space + 1)
- if (space < 0 or
- self._count_dollars_before_index(text, space) % 2 == 0):
+ space = text.find(" ", space + 1)
+ if (
+ space < 0
+ or self._count_dollars_before_index(text, space) % 2 == 0
+ ):
break
if space < 0:
# Give up on breaking.
break
- self.output.write(leading_space + text[0:space] + ' $\n')
- text = text[space+1:]
+ self.output.write(leading_space + text[0:space] + " $\n")
+ text = text[space + 1 :]
# Subsequent lines are continuations, so indent them.
- leading_space = ' ' * (indent+2)
+ leading_space = " " * (indent + 2)
- self.output.write(leading_space + text + '\n')
+ self.output.write(leading_space + text + "\n")
- def close(self):
+ def close(self) -> None:
self.output.close()
-def as_list(input):
- if input is None:
- return []
+def serialize_path(input: Optional[NinjaPath]) -> str:
+ if not input:
+ return ""
+ if isinstance(input, Path):
+ return str(input).replace("/", os.sep)
+ else:
+ return str(input)
+
+
+def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]:
if isinstance(input, list):
- return input
- return [input]
+ return [serialize_path(path) for path in input if path]
+ return [serialize_path(input)] if input else []
-def escape(string):
+def escape(string: str) -> str:
"""Escape a string such that it can be embedded into a Ninja file without
further interpretation."""
- assert '\n' not in string, 'Ninja syntax does not allow newlines'
+ assert "\n" not in string, "Ninja syntax does not allow newlines"
# We only have one special metacharacter: '$'.
- return string.replace('$', '$$')
+ return string.replace("$", "$$")
-def expand(string, vars, local_vars={}):
+def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str:
"""Expand a string containing $vars as Ninja would.
Note: doesn't handle the full Ninja variable syntax, but it's enough
to make configure.py's use of it work.
"""
- def exp(m):
+
+ def exp(m: Match[str]) -> str:
var = m.group(1)
- if var == '$':
- return '$'
- return local_vars.get(var, vars.get(var, ''))
- return re.sub(r'\$(\$|\w*)', exp, string) \ No newline at end of file
+ if var == "$":
+ return "$"
+ return local_vars.get(var, vars.get(var, ""))
+
+ return re.sub(r"\$(\$|\w*)", exp, string)
diff --git a/tools/project.py b/tools/project.py
index 8ccfde1a..4e2263d0 100644
--- a/tools/project.py
+++ b/tools/project.py
@@ -12,13 +12,15 @@
import io
import json
+import math
import os
import platform
import sys
-import math
-
from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+
from . import ninja_syntax
+from .ninja_syntax import serialize_path
if sys.platform == "cygwin":
sys.exit(
@@ -28,51 +30,86 @@ if sys.platform == "cygwin":
)
+class Object:
+ def __init__(self, completed: bool, name: str, **options: Any) -> None:
+ self.name = name
+ self.base_name = Path(name).with_suffix("")
+ self.completed = completed
+ self.options: Dict[str, Any] = {
+ "add_to_all": True,
+ "asflags": None,
+ "extra_asflags": None,
+ "cflags": None,
+ "extra_cflags": None,
+ "mw_version": None,
+ "shift_jis": None,
+ "source": name,
+ }
+ self.options.update(options)
+
+
class ProjectConfig:
- def __init__(self):
+ def __init__(self) -> None:
# Paths
- self.build_dir = Path("build")
- self.src_dir = Path("src")
- self.tools_dir = Path("tools")
+ self.build_dir: Path = Path("build") # Output build files
+ self.src_dir: Path = Path("src") # C/C++/asm source files
+ self.tools_dir: Path = Path("tools") # Python scripts
+ self.asm_dir: Optional[Path] = Path(
+ "asm"
+ ) # Override incomplete objects (for modding)
# Tooling
- self.dtk_tag = None # Git tag
- self.build_dtk_path = None # If None, download
- self.compilers_tag = None # 1
- self.compilers_path = None # If None, download
- self.wibo_tag = None # Git tag
- self.wrapper = None # If None, download wibo on Linux
- self.sjiswrap_tag = None # Git tag
- self.sjiswrap_path = None # If None, download
+ self.binutils_tag: Optional[str] = None # Git tag
+ self.binutils_path: Optional[Path] = None # If None, download
+ self.dtk_tag: Optional[str] = None # Git tag
+ self.dtk_path: Optional[Path] = None # If None, download
+ self.compilers_tag: Optional[str] = None # 1
+ self.compilers_path: Optional[Path] = None # If None, download
+ self.wibo_tag: Optional[str] = None # Git tag
+ self.wrapper: Optional[Path] = None # If None, download wibo on Linux
+ self.sjiswrap_tag: Optional[str] = None # Git tag
+ self.sjiswrap_path: Optional[Path] = None # If None, download
# Project config
- self.build_rels = True # Build REL files
- self.check_sha_path = None # Path to version.sha1
- self.config_path = None # Path to config.yml
- self.debug = False # Build with debug info
- self.generate_map = False # Generate map file(s)
- self.ldflags = None # Linker flags
- self.libs = None # List of libraries
- self.linker_version = None # mwld version
- self.version = None # Version name
- self.warn_missing_config = False # Warn on missing unit configuration
- self.warn_missing_source = False # Warn on missing source file
- self.rel_strip_partial = True # Generate PLFs with -strip_partial
- self.rel_empty_file = None # Path to empty.c for generating empty RELs
+ self.non_matching: bool = False
+ self.build_rels: bool = True # Build REL files
+ self.check_sha_path: Optional[Path] = None # Path to version.sha1
+ self.config_path: Optional[Path] = None # Path to config.yml
+ self.debug: bool = False # Build with debug info
+ self.generate_map: bool = False # Generate map file(s)
+ self.asflags: Optional[List[str]] = None # Assembler flags
+ self.ldflags: Optional[List[str]] = None # Linker flags
+ self.libs: Optional[List[Dict[str, Any]]] = None # List of libraries
+ self.linker_version: Optional[str] = None # mwld version
+ self.version: Optional[str] = None # Version name
+ self.warn_missing_config: bool = False # Warn on missing unit configuration
+ self.warn_missing_source: bool = False # Warn on missing source file
+ self.rel_strip_partial: bool = True # Generate PLFs with -strip_partial
+ self.rel_empty_file: Optional[
+ str
+ ] = None # Object name for generating empty RELs
+ self.shift_jis = (
+ True # Convert source files from UTF-8 to Shift JIS automatically
+ )
+ self.reconfig_deps: Optional[List[Path]] = (
+ None # Additional re-configuration dependency files
+ )
# Progress output and progress.json config
- self.progress_all = True # Include combined "all" category
- self.progress_modules = True # Include combined "modules" category
- self.progress_each_module = True # Include individual modules, disable for large numbers of modules
+ self.progress_all: bool = True # Include combined "all" category
+ self.progress_modules: bool = True # Include combined "modules" category
+ self.progress_each_module: bool = (
+ True # Include individual modules, disable for large numbers of modules
+ )
# Progress fancy printing
- self.progress_use_fancy = False
- self.progress_code_fancy_frac = 0
- self.progress_code_fancy_item = ""
- self.progress_data_fancy_frac = 0
- self.progress_data_fancy_item = ""
+ self.progress_use_fancy: bool = False
+ self.progress_code_fancy_frac: int = 0
+ self.progress_code_fancy_item: str = ""
+ self.progress_data_fancy_frac: int = 0
+ self.progress_data_fancy_item: str = ""
- def validate(self):
+ def validate(self) -> None:
required_attrs = [
"build_dir",
"src_dir",
@@ -88,33 +125,18 @@ class ProjectConfig:
if getattr(self, attr) is None:
sys.exit(f"ProjectConfig.{attr} missing")
- def find_object(self, name):
- for lib in self.libs:
+ def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]:
+ for lib in self.libs or {}:
for obj in lib["objects"]:
if obj.name == name:
- return [lib, obj]
+ return lib, obj
return None
- def out_path(self):
- return self.build_dir / self.version
-
-
-class Object:
- def __init__(self, completed, name, **options):
- self.name = name
- self.completed = completed
- self.options = {
- "add_to_all": True,
- "cflags": None,
- "extra_cflags": None,
- "mw_version": None,
- "shiftjis": True,
- "source": name,
- }
- self.options.update(options)
+ def out_path(self) -> Path:
+ return self.build_dir / str(self.version)
-def is_windows():
+def is_windows() -> bool:
return os.name == "nt"
@@ -124,36 +146,25 @@ CHAIN = "cmd /c " if is_windows() else ""
EXE = ".exe" if is_windows() else ""
-# Replace forward slashes with backslashes on Windows
-def os_str(value):
- return str(value).replace("/", os.sep)
-
-
-# Replace backslashes with forward slashes on Windows
-def unix_str(value):
- return str(value).replace(os.sep, "/")
-
-
-# Stringify paths for ninja_syntax
-def path(value):
- if value is None:
- return None
- elif isinstance(value, list):
- return list(map(os_str, filter(lambda x: x is not None, value)))
+def make_flags_str(cflags: Union[str, List[str]]) -> str:
+ if isinstance(cflags, list):
+ return " ".join(cflags)
else:
- return [os_str(value)]
+ return cflags
# Load decomp-toolkit generated config.json
-def load_build_config(config, build_config_path):
+def load_build_config(
+ config: ProjectConfig, build_config_path: Path
+) -> Optional[Dict[str, Any]]:
if not build_config_path.is_file():
return None
- def versiontuple(v):
+ def versiontuple(v: str) -> Tuple[int, ...]:
return tuple(map(int, (v.split("."))))
f = open(build_config_path, "r", encoding="utf-8")
- build_config = json.load(f)
+ build_config: Dict[str, Any] = json.load(f)
config_version = build_config.get("version")
if not config_version:
# Invalid config.json
@@ -161,7 +172,7 @@ def load_build_config(config, build_config_path):
os.remove(build_config_path)
return None
- dtk_version = config.dtk_tag[1:] # Strip v
+ dtk_version = str(config.dtk_tag)[1:] # Strip v
if versiontuple(config_version) < versiontuple(dtk_version):
# Outdated config.json
f.close()
@@ -173,14 +184,16 @@ def load_build_config(config, build_config_path):
# Generate build.ninja and objdiff.json
-def generate_build(config):
+def generate_build(config: ProjectConfig) -> None:
build_config = load_build_config(config, config.out_path() / "config.json")
generate_build_ninja(config, build_config)
generate_objdiff_config(config, build_config)
# Generate build.ninja
-def generate_build_ninja(config, build_config):
+def generate_build_ninja(
+ config: ProjectConfig, build_config: Optional[Dict[str, Any]]
+) -> None:
config.validate()
out = io.StringIO()
@@ -188,9 +201,9 @@ def generate_build_ninja(config, build_config):
n.variable("ninja_required_version", "1.3")
n.newline()
- configure_script = os.path.relpath(os.path.abspath(sys.argv[0]))
- python_lib = os.path.relpath(__file__)
- python_lib_dir = os.path.dirname(python_lib)
+ configure_script = Path(os.path.relpath(os.path.abspath(sys.argv[0])))
+ python_lib = Path(os.path.relpath(__file__))
+ python_lib_dir = python_lib.parent
n.comment("The arguments passed to configure.py, for rerunning it.")
n.variable("configure_args", sys.argv[1:])
n.variable("python", f'"{sys.executable}"')
@@ -200,13 +213,15 @@ def generate_build_ninja(config, build_config):
# Variables
###
n.comment("Variables")
- ldflags = " ".join(config.ldflags)
+ ldflags = " ".join(config.ldflags or [])
if config.generate_map:
ldflags += " -mapunused"
if config.debug:
ldflags += " -g"
n.variable("ldflags", ldflags)
- n.variable("mw_version", config.linker_version)
+ if not config.linker_version:
+ sys.exit("ProjectConfig.linker_version missing")
+ n.variable("mw_version", Path(config.linker_version))
n.newline()
###
@@ -215,6 +230,7 @@ def generate_build_ninja(config, build_config):
n.comment("Tooling")
build_path = config.out_path()
+ progress_path = build_path / "progress.json"
build_tools_path = config.build_dir / "tools"
download_tool = config.tools_dir / "download_tool.py"
n.rule(
@@ -223,20 +239,31 @@ def generate_build_ninja(config, build_config):
description="TOOL $out",
)
- if config.build_dtk_path:
+ decompctx = config.tools_dir / "decompctx.py"
+ n.rule(
+ name="decompctx",
+ command=f"$python {decompctx} $in -o $out -d $out.d",
+ description="CTX $in",
+ depfile="$out.d",
+ deps="gcc",
+ )
+
+ if config.dtk_path is not None and config.dtk_path.is_file():
+ dtk = config.dtk_path
+ elif config.dtk_path is not None:
dtk = build_tools_path / "release" / f"dtk{EXE}"
n.rule(
name="cargo",
command="cargo build --release --manifest-path $in --bin $bin --target-dir $target",
description="CARGO $bin",
- depfile=path(Path("$target") / "release" / "$bin.d"),
+ depfile=Path("$target") / "release" / "$bin.d",
deps="gcc",
)
n.build(
- outputs=path(dtk),
+ outputs=dtk,
rule="cargo",
- inputs=path(config.build_dtk_path / "Cargo.toml"),
- implicit=path(config.build_dtk_path / "Cargo.lock"),
+ inputs=config.dtk_path / "Cargo.toml",
+ implicit=config.dtk_path / "Cargo.lock",
variables={
"bin": "dtk",
"target": build_tools_path,
@@ -245,9 +272,9 @@ def generate_build_ninja(config, build_config):
elif config.dtk_tag:
dtk = build_tools_path / f"dtk{EXE}"
n.build(
- outputs=path(dtk),
+ outputs=dtk,
rule="download_tool",
- implicit=path(download_tool),
+ implicit=download_tool,
variables={
"tool": "dtk",
"tag": config.dtk_tag,
@@ -261,9 +288,9 @@ def generate_build_ninja(config, build_config):
elif config.sjiswrap_tag:
sjiswrap = build_tools_path / "sjiswrap.exe"
n.build(
- outputs=path(sjiswrap),
+ outputs=sjiswrap,
rule="download_tool",
- implicit=path(download_tool),
+ implicit=download_tool,
variables={
"tool": "sjiswrap",
"tag": config.sjiswrap_tag,
@@ -274,7 +301,7 @@ def generate_build_ninja(config, build_config):
# Only add an implicit dependency on wibo if we download it
wrapper = config.wrapper
- wrapper_implicit = None
+ wrapper_implicit: Optional[Path] = None
if (
config.wibo_tag is not None
and sys.platform == "linux"
@@ -284,33 +311,53 @@ def generate_build_ninja(config, build_config):
wrapper = build_tools_path / "wibo"
wrapper_implicit = wrapper
n.build(
- outputs=path(wrapper),
+ outputs=wrapper,
rule="download_tool",
- implicit=path(download_tool),
+ implicit=download_tool,
variables={
"tool": "wibo",
"tag": config.wibo_tag,
},
)
if not is_windows() and wrapper is None:
- wrapper = "wine"
+ wrapper = Path("wine")
wrapper_cmd = f"{wrapper} " if wrapper else ""
- compilers_implicit = None
+ compilers_implicit: Optional[Path] = None
if config.compilers_path:
compilers = config.compilers_path
elif config.compilers_tag:
compilers = config.build_dir / "compilers"
compilers_implicit = compilers
n.build(
- outputs=path(compilers),
+ outputs=compilers,
rule="download_tool",
- implicit=path(download_tool),
+ implicit=download_tool,
variables={
"tool": "compilers",
"tag": config.compilers_tag,
},
)
+ else:
+ sys.exit("ProjectConfig.compilers_tag missing")
+
+ binutils_implicit = None
+ if config.binutils_path:
+ binutils = config.binutils_path
+ elif config.binutils_tag:
+ binutils = config.build_dir / "binutils"
+ binutils_implicit = binutils
+ n.build(
+ outputs=binutils,
+ rule="download_tool",
+ implicit=download_tool,
+ variables={
+ "tool": "binutils",
+ "tag": config.binutils_tag,
+ },
+ )
+ else:
+ sys.exit("ProjectConfig.binutils_tag missing")
n.newline()
@@ -322,16 +369,24 @@ def generate_build_ninja(config, build_config):
# MWCC
mwcc = compiler_path / "mwcceppc.exe"
mwcc_cmd = f"{wrapper_cmd}{mwcc} $cflags -MMD -c $in -o $basedir"
- mwcc_implicit = [compilers_implicit or mwcc, wrapper_implicit]
+ mwcc_implicit: List[Optional[Path]] = [compilers_implicit or mwcc, wrapper_implicit]
# MWCC with UTF-8 to Shift JIS wrapper
mwcc_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir"
- mwcc_sjis_implicit = [*mwcc_implicit, sjiswrap]
+ mwcc_sjis_implicit: List[Optional[Path]] = [*mwcc_implicit, sjiswrap]
# MWLD
mwld = compiler_path / "mwldeppc.exe"
mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp"
- mwld_implicit = [compilers_implicit or mwld, wrapper_implicit]
+ mwld_implicit: List[Optional[Path]] = [compilers_implicit or mwld, wrapper_implicit]
+
+ # GNU as
+ gnu_as = binutils / f"powerpc-eabi-as{EXE}"
+ gnu_as_cmd = (
+ f"{CHAIN}{gnu_as} $asflags -o $out $in -MD $out.d"
+ + f" && {dtk} elf fixup $out $out"
+ )
+ gnu_as_implicit = [binutils_implicit or gnu_as, dtk]
if os.name != "nt":
transform_dep = config.tools_dir / "transform_dep.py"
@@ -358,17 +413,6 @@ def generate_build_ninja(config, build_config):
)
n.newline()
- n.comment("Generate REL(s)")
- makerel_rsp = build_path / "makerel.rsp"
- n.rule(
- name="makerel",
- command=f"{dtk} rel make -w -c $config @{makerel_rsp}",
- description="REL",
- rspfile=path(makerel_rsp),
- rspfile_content="$in_newline",
- )
- n.newline()
-
n.comment("MWCC build")
n.rule(
name="mwcc",
@@ -389,6 +433,16 @@ def generate_build_ninja(config, build_config):
)
n.newline()
+ n.comment("Assemble asm")
+ n.rule(
+ name="as",
+ command=gnu_as_cmd,
+ description="AS $out",
+ depfile="$out.d",
+ deps="gcc",
+ )
+ n.newline()
+
n.comment("Host build")
n.variable("host_cflags", "-I include -Wno-trigraphs")
n.variable(
@@ -411,66 +465,67 @@ def generate_build_ninja(config, build_config):
# Source files
###
n.comment("Source files")
+ build_asm_path = build_path / "mod"
build_src_path = build_path / "src"
build_host_path = build_path / "host"
build_config_path = build_path / "config.json"
- def map_path(path):
+ def map_path(path: Path) -> Path:
return path.parent / (path.name + ".MAP")
class LinkStep:
- def __init__(self, config):
- self.name = config["name"]
- self.module_id = config["module_id"]
- self.ldscript = config["ldscript"]
+ def __init__(self, config: Dict[str, Any]) -> None:
+ self.name: str = config["name"]
+ self.module_id: int = config["module_id"]
+ self.ldscript: Optional[Path] = Path(config["ldscript"])
self.entry = config["entry"]
- self.inputs = []
+ self.inputs: List[str] = []
- def add(self, obj):
- self.inputs.append(obj)
+ def add(self, obj: Path) -> None:
+ self.inputs.append(serialize_path(obj))
- def output(self):
+ def output(self) -> Path:
if self.module_id == 0:
return build_path / f"{self.name}.dol"
else:
return build_path / self.name / f"{self.name}.rel"
- def partial_output(self):
+ def partial_output(self) -> Path:
if self.module_id == 0:
return build_path / f"{self.name}.elf"
else:
return build_path / self.name / f"{self.name}.plf"
- def write(self, n):
+ def write(self, n: ninja_syntax.Writer) -> None:
n.comment(f"Link {self.name}")
if self.module_id == 0:
elf_path = build_path / f"{self.name}.elf"
dol_path = build_path / f"{self.name}.dol"
- elf_ldflags = f"$ldflags -lcf {self.ldscript}"
+ elf_ldflags = f"$ldflags -lcf {serialize_path(self.ldscript)}"
if config.generate_map:
elf_map = map_path(elf_path)
- elf_ldflags += f" -map {elf_map}"
+ elf_ldflags += f" -map {serialize_path(elf_map)}"
else:
elf_map = None
n.build(
- outputs=path(elf_path),
+ outputs=elf_path,
rule="link",
- inputs=path(self.inputs),
- implicit=path([self.ldscript, *mwld_implicit]),
- implicit_outputs=path(elf_map),
+ inputs=self.inputs,
+ implicit=[self.ldscript, *mwld_implicit],
+ implicit_outputs=elf_map,
variables={"ldflags": elf_ldflags},
)
n.build(
- outputs=path(dol_path),
+ outputs=dol_path,
rule="elf2dol",
- inputs=path(elf_path),
- implicit=path(dtk),
+ inputs=elf_path,
+ implicit=dtk,
)
else:
preplf_path = build_path / self.name / f"{self.name}.preplf"
plf_path = build_path / self.name / f"{self.name}.plf"
- preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r"
- plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}"
+ preplf_ldflags = "$ldflags -sdata 0 -sdata2 0 -r"
+ plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {serialize_path(self.ldscript)}"
if self.entry:
plf_ldflags += f" -m {self.entry}"
# -strip_partial is only valid with -m
@@ -478,44 +533,140 @@ def generate_build_ninja(config, build_config):
plf_ldflags += " -strip_partial"
if config.generate_map:
preplf_map = map_path(preplf_path)
- preplf_ldflags += f" -map {preplf_map}"
+ preplf_ldflags += f" -map {serialize_path(preplf_map)}"
plf_map = map_path(plf_path)
- plf_ldflags += f" -map {plf_map}"
+ plf_ldflags += f" -map {serialize_path(plf_map)}"
else:
preplf_map = None
plf_map = None
n.build(
- outputs=path(preplf_path),
+ outputs=preplf_path,
rule="link",
- inputs=path(self.inputs),
- implicit=path(mwld_implicit),
- implicit_outputs=path(preplf_map),
+ inputs=self.inputs,
+ implicit=mwld_implicit,
+ implicit_outputs=preplf_map,
variables={"ldflags": preplf_ldflags},
)
n.build(
- outputs=path(plf_path),
+ outputs=plf_path,
rule="link",
- inputs=path(self.inputs),
- implicit=path([self.ldscript, preplf_path, *mwld_implicit]),
- implicit_outputs=path(plf_map),
+ inputs=self.inputs,
+ implicit=[self.ldscript, preplf_path, *mwld_implicit],
+ implicit_outputs=plf_map,
variables={"ldflags": plf_ldflags},
)
n.newline()
+ link_outputs: List[Path] = []
if build_config:
- link_steps = []
- used_compiler_versions = set()
- source_inputs = []
- host_source_inputs = []
- source_added = set()
-
- def make_cflags_str(cflags):
- if isinstance(cflags, list):
- return " ".join(cflags)
- else:
- return cflags
+ link_steps: List[LinkStep] = []
+ used_compiler_versions: Set[str] = set()
+ source_inputs: List[Path] = []
+ host_source_inputs: List[Path] = []
+ source_added: Set[Path] = set()
+
+ def c_build(
+ obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path
+ ) -> Optional[Path]:
+ cflags_str = make_flags_str(options["cflags"])
+ if options["extra_cflags"] is not None:
+ extra_cflags_str = make_flags_str(options["extra_cflags"])
+ cflags_str += " " + extra_cflags_str
+ used_compiler_versions.add(options["mw_version"])
+
+ src_obj_path = build_src_path / f"{obj.base_name}.o"
+ src_base_path = build_src_path / obj.base_name
+
+ # Avoid creating duplicate build rules
+ if src_obj_path in source_added:
+ return src_obj_path
+ source_added.add(src_obj_path)
- def add_unit(build_obj, link_step):
+ shift_jis = options["shift_jis"]
+ if shift_jis is None:
+ shift_jis = config.shift_jis
+
+ # Add MWCC build rule
+ n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})")
+ n.build(
+ outputs=src_obj_path,
+ rule="mwcc_sjis" if shift_jis else "mwcc",
+ inputs=src_path,
+ variables={
+ "mw_version": Path(options["mw_version"]),
+ "cflags": cflags_str,
+ "basedir": os.path.dirname(src_base_path),
+ "basefile": src_base_path,
+ },
+ implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit,
+ )
+
+ # Add ctx build rule
+ ctx_path = build_src_path / f"{obj.base_name}.ctx"
+ n.build(
+ outputs=ctx_path,
+ rule="decompctx",
+ inputs=src_path,
+ implicit=decompctx,
+ )
+
+ # Add host build rule
+ if options.get("host", False):
+ host_obj_path = build_host_path / f"{obj.base_name}.o"
+ host_base_path = build_host_path / obj.base_name
+ n.build(
+ outputs=host_obj_path,
+ rule="host_cc" if src_path.suffix == ".c" else "host_cpp",
+ inputs=src_path,
+ variables={
+ "basedir": os.path.dirname(host_base_path),
+ "basefile": host_base_path,
+ },
+ )
+ if options["add_to_all"]:
+ host_source_inputs.append(host_obj_path)
+ n.newline()
+
+ if options["add_to_all"]:
+ source_inputs.append(src_obj_path)
+
+ return src_obj_path
+
+ def asm_build(
+ obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path
+ ) -> Optional[Path]:
+ asflags = options["asflags"] or config.asflags
+ if asflags is None:
+ sys.exit("ProjectConfig.asflags missing")
+ asflags_str = make_flags_str(asflags)
+ if options["extra_asflags"] is not None:
+ extra_asflags_str = make_flags_str(options["extra_asflags"])
+ asflags_str += " " + extra_asflags_str
+
+ asm_obj_path = build_asm_path / f"{obj.base_name}.o"
+
+ # Avoid creating duplicate build rules
+ if asm_obj_path in source_added:
+ return asm_obj_path
+ source_added.add(asm_obj_path)
+
+ # Add assembler build rule
+ n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})")
+ n.build(
+ outputs=asm_obj_path,
+ rule="as",
+ inputs=src_path,
+ variables={"asflags": asflags_str},
+ implicit=gnu_as_implicit,
+ )
+ n.newline()
+
+ if options["add_to_all"]:
+ source_inputs.append(asm_obj_path)
+
+ return asm_obj_path
+
+ def add_unit(build_obj, link_step: LinkStep):
obj_path, obj_name = build_obj["object"], build_obj["name"]
result = config.find_object(obj_name)
if not result:
@@ -526,71 +677,50 @@ def generate_build_ninja(config, build_config):
lib, obj = result
lib_name = lib["lib"]
- src_dir = Path(lib.get("src_dir", config.src_dir))
-
- options = obj.options
- completed = obj.completed
- unit_src_path = src_dir / options["source"]
-
- if not unit_src_path.exists():
- if config.warn_missing_source or completed:
+ # Use object options, then library options
+ options = lib.copy()
+ for key, value in obj.options.items():
+ if value is not None or key not in options:
+ options[key] = value
+
+ unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"]
+
+ unit_asm_path: Optional[Path] = None
+ if config.asm_dir is not None:
+ unit_asm_path = (
+ Path(lib.get("asm_dir", config.asm_dir)) / options["source"]
+ ).with_suffix(".s")
+
+ link_built_obj = obj.completed
+ built_obj_path: Optional[Path] = None
+ if unit_src_path.exists():
+ if unit_src_path.suffix in (".c", ".cp", ".cpp"):
+ # Add MWCC & host build rules
+ built_obj_path = c_build(obj, options, lib_name, unit_src_path)
+ elif unit_src_path.suffix == ".s":
+ # Add assembler build rule
+ built_obj_path = asm_build(obj, options, lib_name, unit_src_path)
+ else:
+ sys.exit(f"Unknown source file type {unit_src_path}")
+ else:
+ if config.warn_missing_source or obj.completed:
print(f"Missing source file {unit_src_path}")
+ link_built_obj = False
+
+ # Assembly overrides
+ if unit_asm_path is not None and unit_asm_path.exists():
+ link_built_obj = True
+ built_obj_path = asm_build(obj, options, lib_name, unit_asm_path)
+
+ if link_built_obj and built_obj_path is not None:
+ # Use the source-built object
+ link_step.add(built_obj_path)
+ elif obj_path is not None:
+ # Use the original (extracted) object
link_step.add(obj_path)
- return
-
- mw_version = options["mw_version"] or lib["mw_version"]
- cflags_str = make_cflags_str(options["cflags"] or lib["cflags"])
- if options["extra_cflags"] is not None:
- extra_cflags_str = make_cflags_str(options["extra_cflags"])
- cflags_str += " " + extra_cflags_str
- used_compiler_versions.add(mw_version)
-
- base_object = Path(obj.name).with_suffix("")
- src_obj_path = build_src_path / f"{base_object}.o"
- src_base_path = build_src_path / base_object
-
- if src_obj_path not in source_added:
- source_added.add(src_obj_path)
-
- n.comment(f"{obj_name}: {lib_name} (linked {completed})")
- n.build(
- outputs=path(src_obj_path),
- rule="mwcc_sjis" if options["shiftjis"] else "mwcc",
- inputs=path(unit_src_path),
- variables={
- "mw_version": path(Path(mw_version)),
- "cflags": cflags_str,
- "basedir": os.path.dirname(src_base_path),
- "basefile": path(src_base_path),
- },
- implicit=path(
- mwcc_sjis_implicit if options["shiftjis"] else mwcc_implicit
- ),
- )
-
- if lib["host"]:
- host_obj_path = build_host_path / f"{base_object}.o"
- host_base_path = build_host_path / base_object
- n.build(
- outputs=path(host_obj_path),
- rule="host_cc" if unit_src_path.suffix == ".c" else "host_cpp",
- inputs=path(unit_src_path),
- variables={
- "basedir": os.path.dirname(host_base_path),
- "basefile": path(host_base_path),
- },
- )
- if options["add_to_all"]:
- host_source_inputs.append(host_obj_path)
- n.newline()
-
- if options["add_to_all"]:
- source_inputs.append(src_obj_path)
-
- if completed:
- obj_path = src_obj_path
- link_step.add(obj_path)
+ else:
+ sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}")
# Add DOL link step
link_step = LinkStep(build_config)
@@ -626,7 +756,7 @@ def generate_build_ninja(config, build_config):
sys.exit(f"Compiler {mw_path} does not exist")
# Check if linker exists
- mw_path = compilers / config.linker_version / "mwldeppc.exe"
+ mw_path = compilers / str(config.linker_version) / "mwldeppc.exe"
if config.compilers_path and not os.path.exists(mw_path):
sys.exit(f"Linker {mw_path} does not exist")
@@ -635,13 +765,25 @@ def generate_build_ninja(config, build_config):
###
for step in link_steps:
step.write(n)
+ link_outputs.append(step.output())
n.newline()
###
# Generate RELs
###
+ n.comment("Generate REL(s)")
+ flags = "-w"
+ if len(build_config["links"]) > 1:
+ flags += " -q"
+ n.rule(
+ name="makerel",
+ command=f"{dtk} rel make {flags} -c $config $names @$rspfile",
+ description="REL",
+ rspfile="$rspfile",
+ rspfile_content="$in_newline",
+ )
generated_rels = []
- for link in build_config["links"]:
+ for idx, link in enumerate(build_config["links"]):
# Map module names to link steps
link_steps_local = list(
filter(
@@ -655,7 +797,7 @@ def generate_build_ninja(config, build_config):
rels_to_generate = list(
filter(
lambda step: step.module_id != 0
- and not step.name in generated_rels,
+ and step.name not in generated_rels,
link_steps_local,
)
)
@@ -668,15 +810,23 @@ def generate_build_ninja(config, build_config):
rels_to_generate,
)
)
- n.comment("Generate RELs")
+ rel_names = list(
+ map(
+ lambda step: step.name,
+ link_steps_local,
+ )
+ )
+ rel_names_arg = " ".join(map(lambda name: f"-n {name}", rel_names))
n.build(
- outputs=path(rel_outputs),
+ outputs=rel_outputs,
rule="makerel",
- inputs=path(
- list(map(lambda step: step.partial_output(), link_steps_local))
- ),
- implicit=path([dtk, config.config_path]),
- variables={"config": path(config.config_path)},
+ inputs=list(map(lambda step: step.partial_output(), link_steps_local)),
+ implicit=[dtk, config.config_path],
+ variables={
+ "config": config.config_path,
+ "rspfile": config.out_path() / f"rel{idx}.rsp",
+ "names": rel_names_arg,
+ },
)
n.newline()
@@ -687,7 +837,7 @@ def generate_build_ninja(config, build_config):
n.build(
outputs="all_source",
rule="phony",
- inputs=path(source_inputs),
+ inputs=source_inputs,
)
n.newline()
@@ -698,7 +848,7 @@ def generate_build_ninja(config, build_config):
n.build(
outputs="all_source_host",
rule="phony",
- inputs=path(host_source_inputs),
+ inputs=host_source_inputs,
)
n.newline()
@@ -714,10 +864,10 @@ def generate_build_ninja(config, build_config):
description="CHECK $in",
)
n.build(
- outputs=path(ok_path),
+ outputs=ok_path,
rule="check",
- inputs=path(config.check_sha_path),
- implicit=path([dtk, *map(lambda step: step.output(), link_steps)]),
+ inputs=config.check_sha_path,
+ implicit=[dtk, *link_outputs],
)
n.newline()
@@ -725,16 +875,15 @@ def generate_build_ninja(config, build_config):
# Calculate progress
###
n.comment("Calculate progress")
- progress_path = build_path / "progress.json"
n.rule(
name="progress",
command=f"$python {configure_script} $configure_args progress",
description="PROGRESS",
)
n.build(
- outputs=path(progress_path),
+ outputs=progress_path,
rule="progress",
- implicit=path([ok_path, configure_script, python_lib, config.config_path]),
+ implicit=[ok_path, configure_script, python_lib, config.config_path],
)
###
@@ -750,7 +899,7 @@ def generate_build_ninja(config, build_config):
description=f"DIFF {dol_elf_path}",
)
n.build(
- inputs=path([config.config_path, dol_elf_path]),
+ inputs=[config.config_path, dol_elf_path],
outputs="dol_diff",
rule="dol_diff",
)
@@ -768,10 +917,10 @@ def generate_build_ninja(config, build_config):
description=f"APPLY {dol_elf_path}",
)
n.build(
- inputs=path([config.config_path, dol_elf_path]),
+ inputs=[config.config_path, dol_elf_path],
outputs="dol_apply",
rule="dol_apply",
- implicit=path([ok_path]),
+ implicit=[ok_path],
)
n.build(
outputs="apply",
@@ -792,11 +941,11 @@ def generate_build_ninja(config, build_config):
deps="gcc",
)
n.build(
- inputs=path(config.config_path),
- outputs=path(build_config_path),
+ inputs=config.config_path,
+ outputs=build_config_path,
rule="split",
- implicit=path(dtk),
- variables={"out_dir": path(build_path)},
+ implicit=dtk,
+ variables={"out_dir": build_path},
)
n.newline()
@@ -813,14 +962,13 @@ def generate_build_ninja(config, build_config):
n.build(
outputs="build.ninja",
rule="configure",
- implicit=path(
- [
- build_config_path,
- configure_script,
- python_lib,
- Path(python_lib_dir) / "ninja_syntax.py",
- ]
- ),
+ implicit=[
+ build_config_path,
+ configure_script,
+ python_lib,
+ python_lib_dir / "ninja_syntax.py",
+ *(config.reconfig_deps or [])
+ ],
)
n.newline()
@@ -829,9 +977,12 @@ def generate_build_ninja(config, build_config):
###
n.comment("Default rule")
if build_config:
- n.default(path(progress_path))
+ if config.non_matching:
+ n.default(link_outputs)
+ else:
+ n.default(progress_path)
else:
- n.default(path(build_config_path))
+ n.default(build_config_path)
# Write build.ninja
with open("build.ninja", "w", encoding="utf-8") as f:
@@ -840,12 +991,14 @@ def generate_build_ninja(config, build_config):
# Generate objdiff.json
-def generate_objdiff_config(config, build_config):
+def generate_objdiff_config(
+ config: ProjectConfig, build_config: Optional[Dict[str, Any]]
+) -> None:
if not build_config:
return
- objdiff_config = {
- "min_version": "0.4.3",
+ objdiff_config: Dict[str, Any] = {
+ "min_version": "1.0.0",
"custom_make": "ninja",
"build_target": False,
"watch_patterns": [
@@ -863,18 +1016,50 @@ def generate_objdiff_config(config, build_config):
"units": [],
}
+ # decomp.me compiler name mapping
+ # Commented out versions have not been added to decomp.me yet
+ COMPILER_MAP = {
+ "GC/1.0": "mwcc_233_144",
+ "GC/1.1": "mwcc_233_159",
+ "GC/1.2.5": "mwcc_233_163",
+ "GC/1.2.5e": "mwcc_233_163e",
+ "GC/1.2.5n": "mwcc_233_163n",
+ "GC/1.3.2": "mwcc_242_81",
+ "GC/1.3.2r": "mwcc_242_81r",
+ "GC/2.0": "mwcc_247_92",
+ "GC/2.5": "mwcc_247_105",
+ "GC/2.6": "mwcc_247_107",
+ "GC/2.7": "mwcc_247_108",
+ "GC/3.0": "mwcc_41_60831",
+ # "GC/3.0a3": "mwcc_41_51213",
+ "GC/3.0a3.2": "mwcc_41_60126",
+ # "GC/3.0a3.3": "mwcc_41_60209",
+ # "GC/3.0a3.4": "mwcc_42_60308",
+ # "GC/3.0a5": "mwcc_42_60422",
+ "GC/3.0a5.2": "mwcc_41_60831",
+ "Wii/0x4201_127": "mwcc_42_142",
+ # "Wii/1.0": "mwcc_43_145",
+ # "Wii/1.0RC1": "mwcc_42_140",
+ "Wii/1.0a": "mwcc_42_142",
+ "Wii/1.1": "mwcc_43_151",
+ "Wii/1.3": "mwcc_43_172",
+ # "Wii/1.5": "mwcc_43_188",
+ "Wii/1.6": "mwcc_43_202",
+ "Wii/1.7": "mwcc_43_213",
+ }
+
build_path = config.out_path()
- def add_unit(build_obj, module_name):
+ def add_unit(build_obj: Dict[str, Any], module_name: str) -> None:
if build_obj["autogenerated"]:
# Skip autogenerated objects
return
obj_path, obj_name = build_obj["object"], build_obj["name"]
base_object = Path(obj_name).with_suffix("")
- unit_config = {
- "name": unix_str(Path(module_name) / base_object),
- "target_path": unix_str(obj_path),
+ unit_config: Dict[str, Any] = {
+ "name": Path(module_name) / base_object,
+ "target_path": obj_path,
}
result = config.find_object(obj_name)
@@ -885,14 +1070,21 @@ def generate_objdiff_config(config, build_config):
lib, obj = result
src_dir = Path(lib.get("src_dir", config.src_dir))
- unit_src_path = src_dir / obj.options["source"]
+ # Use object options, then library options
+ options = lib.copy()
+ for key, value in obj.options.items():
+ if value is not None or key not in options:
+ options[key] = value
+
+ unit_src_path = src_dir / str(options["source"])
if not unit_src_path.exists():
objdiff_config["units"].append(unit_config)
return
- cflags = obj.options["cflags"] or lib["cflags"]
- src_obj_path = build_path / "src" / f"{base_object}.o"
+ cflags = options["cflags"]
+ src_obj_path = build_path / "src" / f"{obj.base_name}.o"
+ src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx"
reverse_fn_order = False
if type(cflags) is list:
@@ -905,9 +1097,32 @@ def generate_objdiff_config(config, build_config):
elif value == "nodeferred":
reverse_fn_order = False
- unit_config["base_path"] = unix_str(src_obj_path)
+ # Filter out include directories
+ def keep_flag(flag):
+ return not flag.startswith("-i ") and not flag.startswith("-I ")
+
+ cflags = list(filter(keep_flag, cflags))
+
+ # Add appropriate lang flag
+ if unit_src_path.suffix in (".cp", ".cpp"):
+ cflags.insert(0, "-lang=c++")
+ else:
+ cflags.insert(0, "-lang=c")
+
+ unit_config["base_path"] = src_obj_path
unit_config["reverse_fn_order"] = reverse_fn_order
unit_config["complete"] = obj.completed
+ compiler_version = COMPILER_MAP.get(options["mw_version"])
+ if compiler_version is None:
+ print(f"Missing scratch compiler mapping for {options['mw_version']}")
+ else:
+ unit_config["scratch"] = {
+ "platform": "gc_wii",
+ "compiler": compiler_version,
+ "c_flags": make_flags_str(cflags),
+ "ctx_path": src_ctx_path,
+ "build_ctx": True,
+ }
objdiff_config["units"].append(unit_config)
# Add DOL units
@@ -921,32 +1136,36 @@ def generate_objdiff_config(config, build_config):
# Write objdiff.json
with open("objdiff.json", "w", encoding="utf-8") as w:
- json.dump(objdiff_config, w, indent=4)
+
+ def unix_path(input: Any) -> str:
+ return str(input).replace(os.sep, "/") if input else ""
+
+ json.dump(objdiff_config, w, indent=4, default=unix_path)
# Calculate, print and write progress to progress.json
-def calculate_progress(config):
+def calculate_progress(config: ProjectConfig) -> None:
out_path = config.out_path()
build_config = load_build_config(config, out_path / "config.json")
if not build_config:
return
class ProgressUnit:
- def __init__(self, name):
- self.name = name
- self.code_total = 0
- self.code_fancy_frac = config.progress_code_fancy_frac
- self.code_fancy_item = config.progress_code_fancy_item
- self.code_progress = 0
- self.data_total = 0
- self.data_fancy_frac = config.progress_data_fancy_frac
- self.data_fancy_item = config.progress_data_fancy_item
- self.data_progress = 0
- self.objects_progress = 0
- self.objects_total = 0
- self.objects = set()
-
- def add(self, build_obj):
+ def __init__(self, name: str) -> None:
+ self.name: str = name
+ self.code_total: int = 0
+ self.code_fancy_frac: int = config.progress_code_fancy_frac
+ self.code_fancy_item: str = config.progress_code_fancy_item
+ self.code_progress: int = 0
+ self.data_total: int = 0
+ self.data_fancy_frac: int = config.progress_data_fancy_frac
+ self.data_fancy_item: str = config.progress_data_fancy_item
+ self.data_progress: int = 0
+ self.objects_progress: int = 0
+ self.objects_total: int = 0
+ self.objects: Set[Object] = set()
+
+ def add(self, build_obj: Dict[str, Any]) -> None:
self.code_total += build_obj["code_size"]
self.data_total += build_obj["data_size"]
@@ -973,10 +1192,10 @@ def calculate_progress(config):
if include_object:
self.objects_progress += 1
- def code_frac(self):
+ def code_frac(self) -> float:
return self.code_progress / self.code_total
- def data_frac(self):
+ def data_frac(self) -> float:
return self.data_progress / self.data_total
# Add DOL units
@@ -989,7 +1208,7 @@ def calculate_progress(config):
# Add REL units
rels_progress = ProgressUnit("Modules") if config.progress_modules else None
- modules_progress = []
+ modules_progress: List[ProgressUnit] = []
for module in build_config["modules"]:
progress = ProgressUnit(module["name"])
modules_progress.append(progress)
@@ -1003,7 +1222,7 @@ def calculate_progress(config):
# Print human-readable progress
print("Progress:")
- def print_category(unit):
+ def print_category(unit: Optional[ProgressUnit]) -> None:
if unit is None:
return
@@ -1037,9 +1256,9 @@ def calculate_progress(config):
print_category(progress)
# Generate and write progress.json
- progress_json = {}
+ progress_json: Dict[str, Any] = {}
- def add_category(category, unit):
+ def add_category(category: str, unit: ProgressUnit) -> None:
progress_json[category] = {
"code": unit.code_progress,
"code/total": unit.code_total,
diff --git a/tools/transform_dep.py b/tools/transform_dep.py
index 86bd2ecb..124de04b 100644
--- a/tools/transform_dep.py
+++ b/tools/transform_dep.py
@@ -25,7 +25,7 @@ def in_wsl() -> bool:
return "microsoft-standard" in uname().release
-def import_d_file(in_file) -> str:
+def import_d_file(in_file: str) -> str:
out_text = ""
with open(in_file) as file:
@@ -60,7 +60,7 @@ def import_d_file(in_file) -> str:
return out_text
-def main():
+def main() -> None:
parser = argparse.ArgumentParser(
description="""Transform a .d file from Wine paths to normal paths"""
)
@@ -81,4 +81,4 @@ def main():
if __name__ == "__main__":
- main() \ No newline at end of file
+ main()