ZAPD fixes in `sys_initial_check`, update subrepos (#507)

* git subrepo pull --force tools/ZAPD

subrepo:
  subdir:   "tools/ZAPD"
  merged:   "a3363333d"
upstream:
  origin:   "https://github.com/zeldaret/ZAPD.git"
  branch:   "master"
  commit:   "a3363333d"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

* git subrepo pull tools/asm-differ --force

subrepo:
  subdir:   "tools/asm-differ"
  merged:   "70c33cc12"
upstream:
  origin:   "https://github.com/simonlindholm/asm-differ.git"
  branch:   "main"
  commit:   "70c33cc12"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

* git subrepo pull (merge) tools/z64compress --force

subrepo:
  subdir:   "tools/z64compress"
  merged:   "ac5b1a0d0"
upstream:
  origin:   "https://github.com/z64me/z64compress.git"
  branch:   "main"
  commit:   "ac5b1a0d0"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

* Use defines for texture sizes in sys_initial_check

* Update extract_assets.py

* Add null check

* git subrepo pull --force tools/ZAPD

subrepo:
  subdir:   "tools/ZAPD"
  merged:   "50242eca9"
upstream:
  origin:   "https://github.com/zeldaret/ZAPD.git"
  branch:   "master"
  commit:   "50242eca9"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

1 files changed, 187 insertions, 117 deletions

diff --git a/tools/asm-differ/diff.py b/tools/asm-differ/diff.py
index d7ac3f344..2ec117a12 100755
--- a/tools/asm-differ/diff.py
+++ b/tools/asm-differ/diff.py
@@ -640,6 +640,7 @@ class Text:
 class TableMetadata:
     headers: Tuple[Text, ...]
     current_score: int
+    max_score: int
     previous_score: Optional[int]
 
 
@@ -832,6 +833,7 @@ class JsonFormatter(Formatter):
             for h, name in zip(meta.headers, ("base", "current", "previous"))
         }
         output["current_score"] = meta.current_score
+        output["max_score"] = meta.max_score
         if meta.previous_score is not None:
             output["previous_score"] = meta.previous_score
         output_rows: List[Dict[str, Any]] = []
@@ -1184,8 +1186,9 @@ def parse_elf_data_references(data: bytes) -> List[Tuple[int, int, str]]:
     assert len(symtab_sections) == 1
     symtab = sections[symtab_sections[0]]
 
-    text_sections = [i for i in range(e_shnum) if sec_names[i] == b".text"]
-    assert len(text_sections) == 1
+    text_sections = [i for i in range(e_shnum) if sec_names[i] == b".text" and sections[i].sh_size != 0]
+    if len(text_sections) != 1:
+        return []
     text_section = text_sections[0]
 
     ret: List[Tuple[int, int, str]] = []
@@ -1327,11 +1330,23 @@ def dump_binary(
         (objdump_flags + flags2, project.myimg, None),
     )
 
+DATA_POOL_PLACEHOLDER = "DATA_POOL_PLACEHOLDER-OFFSET_{}"
+DATA_POOL_PLACEHOLDER_PATTERN = re.compile(
+    r"DATA_POOL_PLACEHOLDER-OFFSET_([a-zA-z0-9]+)"
+)
+
+# Example: "ldr r4, [pc, #56]    ; (4c <AddCoins+0x4c>)"
+ARM32_LOAD_POOL_PATTERN = r"(ldr\s+r([0-9]|1[0-3]),\s+\[pc,.*;\s*)(\([a-fA-F0-9]+.*\))"
 
-class DifferenceNormalizer:
+
+# The base class is a no-op.
+class AsmProcessor:
     def __init__(self, config: Config) -> None:
         self.config = config
 
+    def process_reloc(self, row: str, prev: str) -> str:
+        return prev
+
     def normalize(self, mnemonic: str, row: str) -> str:
         """This should be called exactly once for each line."""
         arch = self.config.arch
@@ -1342,9 +1357,143 @@ class DifferenceNormalizer:
 
     def _normalize_arch_specific(self, mnemonic: str, row: str) -> str:
         return row
+    
+    def post_process(self, lines: List["Line"]) -> None:
+        return
+
+
+class AsmProcessorMIPS(AsmProcessor):
+    def process_reloc(self, row: str, prev: str) -> str:
+        arch = self.config.arch
+        if "R_MIPS_NONE" in row:
+            # GNU as emits no-op relocations immediately after real ones when
+            # assembling with -mabi=64. Return without trying to parse 'imm' as an
+            # integer.
+            return prev
+        before, imm, after = parse_relocated_line(prev)
+        repl = row.split()[-1]
+        if imm != "0":
+            # MIPS uses relocations with addends embedded in the code as immediates.
+            # If there is an immediate, show it as part of the relocation. Ideally
+            # we'd show this addend in both %lo/%hi, but annoyingly objdump's output
+            # doesn't include enough information to pair up %lo's and %hi's...
+            # TODO: handle unambiguous cases where all addends for a symbol are the
+            # same, or show "+???".
+            mnemonic = prev.split()[0]
+            if (
+                mnemonic in arch.instructions_with_address_immediates
+                and not imm.startswith("0x")
+            ):
+                imm = "0x" + imm
+            repl += "+" + imm if int(imm, 0) > 0 else imm
+        if "R_MIPS_LO16" in row:
+            repl = f"%lo({repl})"
+        elif "R_MIPS_HI16" in row:
+            # Ideally we'd pair up R_MIPS_LO16 and R_MIPS_HI16 to generate a
+            # correct addend for each, but objdump doesn't give us the order of
+            # the relocations, so we can't find the right LO16. :(
+            repl = f"%hi({repl})"
+        elif "R_MIPS_26" in row:
+            # Function calls
+            pass
+        elif "R_MIPS_PC16" in row:
+            # Branch to glabel. This gives confusing output, but there's not much
+            # we can do here.
+            pass
+        else:
+            assert False, f"unknown relocation type '{row}' for line '{prev}'"
+        return before + repl + after
+
+
+class AsmProcessorPPC(AsmProcessor):
+    def process_reloc(self, row: str, prev: str) -> str:
+        arch = self.config.arch
+        assert any(
+            r in row for r in ["R_PPC_REL24", "R_PPC_ADDR16", "R_PPC_EMB_SDA21"]
+        ), f"unknown relocation type '{row}' for line '{prev}'"
+        before, imm, after = parse_relocated_line(prev)
+        repl = row.split()[-1]
+        if "R_PPC_REL24" in row:
+            # function calls
+            pass
+        elif "R_PPC_ADDR16_HI" in row:
+            # absolute hi of addr
+            repl = f"{repl}@h"
+        elif "R_PPC_ADDR16_HA" in row:
+            # adjusted hi of addr
+            repl = f"{repl}@ha"
+        elif "R_PPC_ADDR16_LO" in row:
+            # lo of addr
+            repl = f"{repl}@l"
+        elif "R_PPC_ADDR16" in row:
+            # 16-bit absolute addr
+            if "+0x7" in repl:
+                # remove the very large addends as they are an artifact of (label-_SDA(2)_BASE_)
+                # computations and are unimportant in a diff setting.
+                if int(repl.split("+")[1], 16) > 0x70000000:
+                    repl = repl.split("+")[0]
+        elif "R_PPC_EMB_SDA21" in row:
+            # small data area
+            pass
+        return before + repl + after
+
+
+class AsmProcessorARM32(AsmProcessor):
+    def process_reloc(self, row: str, prev: str) -> str:
+        arch = self.config.arch
+        before, imm, after = parse_relocated_line(prev)
+        repl = row.split()[-1]
+        return before + repl + after
+
+    def _normalize_arch_specific(self, mnemonic: str, row: str) -> str:
+        if self.config.ignore_addr_diffs:
+            row = self._normalize_bl(mnemonic, row)
+        row = self._normalize_data_pool(row)
+        return row
+
+    def _normalize_bl(self, mnemonic: str, row: str) -> str:
+        if mnemonic != "bl":
+            return row
+
+        row, _ = split_off_address(row)
+        return row + "<ignore>"
+
+    def _normalize_data_pool(self, row: str) -> str:
+        pool_match = re.search(ARM32_LOAD_POOL_PATTERN, row)
+        if pool_match:
+            offset = pool_match.group(3).split(" ")[0][1:]
+            repl = DATA_POOL_PLACEHOLDER.format(offset)
+            return pool_match.group(1) + repl
+        return row
 
+    def post_process(self, lines: List["Line"]) -> None:
+        lines_by_line_number = {}
+        for line in lines:
+            lines_by_line_number[line.line_num] = line
+        for line in lines:
+            reloc_match = re.search(
+                DATA_POOL_PLACEHOLDER_PATTERN, line.normalized_original
+            )
+            if reloc_match is None:
+                continue
 
-class DifferenceNormalizerAArch64(DifferenceNormalizer):
+            # Get value at relocation
+            reloc = reloc_match.group(0)
+            line_number = re.search(
+                DATA_POOL_PLACEHOLDER_PATTERN, reloc).group(1)
+            line_original = lines_by_line_number[int(line_number, 16)].original
+            value = line_original.split()[1]
+
+            # Replace relocation placeholder with value
+            replaced = re.sub(
+                DATA_POOL_PLACEHOLDER_PATTERN,
+                f"={value} ({line_number})",
+                line.normalized_original,
+            )
+            line.original = replaced
+
+
+class AsmProcessorAArch64(AsmProcessor):
     def __init__(self, config: Config) -> None:
         super().__init__(config)
         self._adrp_pair_registers: Set[str] = set()
@@ -1394,23 +1543,6 @@ class DifferenceNormalizerAArch64(DifferenceNormalizer):
         return row
 
 
-class DifferenceNormalizerARM32(DifferenceNormalizer):
-    def __init__(self, config: Config) -> None:
-        super().__init__(config)
-
-    def _normalize_arch_specific(self, mnemonic: str, row: str) -> str:
-        if self.config.ignore_addr_diffs:
-            row = self._normalize_bl(mnemonic, row)
-        return row
-
-    def _normalize_bl(self, mnemonic: str, row: str) -> str:
-        if mnemonic != "bl":
-            return row
-
-        row, _ = split_off_address(row)
-        return row + "<ignore>"
-
-
 @dataclass
 class ArchSettings:
     name: str
@@ -1420,14 +1552,15 @@ class ArchSettings:
     re_sprel: Pattern[str]
     re_large_imm: Pattern[str]
     re_imm: Pattern[str]
+    re_reloc: Pattern[str]
     branch_instructions: Set[str]
     instructions_with_address_immediates: Set[str]
     forbidden: Set[str] = field(default_factory=lambda: set(string.ascii_letters + "_"))
     arch_flags: List[str] = field(default_factory=list)
     branch_likely_instructions: Set[str] = field(default_factory=set)
-    difference_normalizer: Type[DifferenceNormalizer] = DifferenceNormalizer
+    proc: Type[AsmProcessor] = AsmProcessor
     big_endian: Optional[bool] = True
-
+    delay_slot_instructions: Set[str] = field(default_factory=set)
 
 MIPS_BRANCH_LIKELY_INSTRUCTIONS = {
     "beql",
@@ -1543,10 +1676,13 @@ MIPS_SETTINGS = ArchSettings(
     re_sprel=re.compile(r"(?<=,)([0-9]+|0x[0-9a-f]+)\(sp\)"),
     re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"),
     re_imm=re.compile(r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(sp)|%(lo|hi)\([^)]*\)"),
+    re_reloc=re.compile(r"R_MIPS_"),
     arch_flags=["-m", "mips:4300"],
     branch_likely_instructions=MIPS_BRANCH_LIKELY_INSTRUCTIONS,
     branch_instructions=MIPS_BRANCH_INSTRUCTIONS,
     instructions_with_address_immediates=MIPS_BRANCH_INSTRUCTIONS.union({"jal", "j"}),
+    delay_slot_instructions=MIPS_BRANCH_INSTRUCTIONS.union({"j", "jal", "jr", "jalr"}),
+    proc=AsmProcessorMIPS,
 )
 
 MIPSEL_SETTINGS = replace(MIPS_SETTINGS, name="mipsel", big_endian=False)
@@ -1566,9 +1702,10 @@ ARM32_SETTINGS = ArchSettings(
     re_sprel=re.compile(r"sp, #-?(0x[0-9a-fA-F]+|[0-9]+)\b"),
     re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"),
     re_imm=re.compile(r"(?<!sp, )#-?(0x[0-9a-fA-F]+|[0-9]+)\b"),
+    re_reloc=re.compile(r"R_ARM_"),
     branch_instructions=ARM32_BRANCH_INSTRUCTIONS,
     instructions_with_address_immediates=ARM32_BRANCH_INSTRUCTIONS.union({"adr"}),
-    difference_normalizer=DifferenceNormalizerARM32,
+    proc=AsmProcessorARM32,
 )
 
 AARCH64_SETTINGS = ArchSettings(
@@ -1581,9 +1718,10 @@ AARCH64_SETTINGS = ArchSettings(
     re_sprel=re.compile(r"sp, #-?(0x[0-9a-fA-F]+|[0-9]+)\b"),
     re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"),
     re_imm=re.compile(r"(?<!sp, )#-?(0x[0-9a-fA-F]+|[0-9]+)\b"),
+    re_reloc=re.compile(r"R_AARCH64_"),
     branch_instructions=AARCH64_BRANCH_INSTRUCTIONS,
     instructions_with_address_immediates=AARCH64_BRANCH_INSTRUCTIONS.union({"bl", "adrp"}),
-    difference_normalizer=DifferenceNormalizerAArch64,
+    proc=AsmProcessorAArch64,
 )
 
 PPC_SETTINGS = ArchSettings(
@@ -1594,8 +1732,10 @@ PPC_SETTINGS = ArchSettings(
     re_sprel=re.compile(r"(?<=,)(-?[0-9]+|-?0x[0-9a-f]+)\(r1\)"),
     re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"),
     re_imm=re.compile(r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(r1)|[^@]*@(ha|h|lo)"),
+    re_reloc=re.compile(r"R_PPC_"),
     branch_instructions=PPC_BRANCH_INSTRUCTIONS,
     instructions_with_address_immediates=PPC_BRANCH_INSTRUCTIONS.union({"bl"}),
+    proc=AsmProcessorPPC,
 )
 
 ARCH_SETTINGS = [
@@ -1639,84 +1779,6 @@ def parse_relocated_line(line: str) -> Tuple[str, str, str]:
     return before, imm, after
 
 
-def process_mips_reloc(row: str, prev: str, arch: ArchSettings) -> str:
-    if "R_MIPS_NONE" in row:
-        # GNU as emits no-op relocations immediately after real ones when
-        # assembling with -mabi=64. Return without trying to parse 'imm' as an
-        # integer.
-        return prev
-    before, imm, after = parse_relocated_line(prev)
-    repl = row.split()[-1]
-    if imm != "0":
-        # MIPS uses relocations with addends embedded in the code as immediates.
-        # If there is an immediate, show it as part of the relocation. Ideally
-        # we'd show this addend in both %lo/%hi, but annoyingly objdump's output
-        # doesn't include enough information to pair up %lo's and %hi's...
-        # TODO: handle unambiguous cases where all addends for a symbol are the
-        # same, or show "+???".
-        mnemonic = prev.split()[0]
-        if (
-            mnemonic in arch.instructions_with_address_immediates
-            and not imm.startswith("0x")
-        ):
-            imm = "0x" + imm
-        repl += "+" + imm if int(imm, 0) > 0 else imm
-    if "R_MIPS_LO16" in row:
-        repl = f"%lo({repl})"
-    elif "R_MIPS_HI16" in row:
-        # Ideally we'd pair up R_MIPS_LO16 and R_MIPS_HI16 to generate a
-        # correct addend for each, but objdump doesn't give us the order of
-        # the relocations, so we can't find the right LO16. :(
-        repl = f"%hi({repl})"
-    elif "R_MIPS_26" in row:
-        # Function calls
-        pass
-    elif "R_MIPS_PC16" in row:
-        # Branch to glabel. This gives confusing output, but there's not much
-        # we can do here.
-        pass
-    else:
-        assert False, f"unknown relocation type '{row}' for line '{prev}'"
-    return before + repl + after
-
-
-def process_ppc_reloc(row: str, prev: str) -> str:
-    assert any(
-        r in row for r in ["R_PPC_REL24", "R_PPC_ADDR16", "R_PPC_EMB_SDA21"]
-    ), f"unknown relocation type '{row}' for line '{prev}'"
-    before, imm, after = parse_relocated_line(prev)
-    repl = row.split()[-1]
-    if "R_PPC_REL24" in row:
-        # function calls
-        pass
-    elif "R_PPC_ADDR16_HI" in row:
-        # absolute hi of addr
-        repl = f"{repl}@h"
-    elif "R_PPC_ADDR16_HA" in row:
-        # adjusted hi of addr
-        repl = f"{repl}@ha"
-    elif "R_PPC_ADDR16_LO" in row:
-        # lo of addr
-        repl = f"{repl}@l"
-    elif "R_PPC_ADDR16" in row:
-        # 16-bit absolute addr
-        if "+0x7" in repl:
-            # remove the very large addends as they are an artifact of (label-_SDA(2)_BASE_)
-            # computations and are unimportant in a diff setting.
-            if int(repl.split("+")[1], 16) > 0x70000000:
-                repl = repl.split("+")[0]
-    elif "R_PPC_EMB_SDA21" in row:
-        # small data area
-        pass
-    return before + repl + after
-
-
-def process_arm_reloc(row: str, prev: str, arch: ArchSettings) -> str:
-    before, imm, after = parse_relocated_line(prev)
-    repl = row.split()[-1]
-    return before + repl + after
-
-
 def pad_mnemonic(line: str) -> str:
     if "\t" not in line:
         return line
@@ -1741,7 +1803,7 @@ class Line:
 
 def process(dump: str, config: Config) -> List[Line]:
     arch = config.arch
-    normalizer = arch.difference_normalizer(config)
+    processor = arch.proc(config)
     skip_next = False
     source_lines = []
     source_filename = None
@@ -1783,7 +1845,7 @@ def process(dump: str, config: Config) -> List[Line]:
             )
             break
 
-        if not re.match(r"^ +[0-9a-f]+:\t", row):
+        if not re.match(r"^\s+[0-9a-f]+:\s+", row):
             # This regex is conservative, and assumes the file path does not contain "weird"
             # characters like colons, tabs, or angle brackets.
             if re.match(
@@ -1797,10 +1859,11 @@ def process(dump: str, config: Config) -> List[Line]:
         m_comment = re.search(arch.re_comment, row)
         comment = m_comment[0] if m_comment else None
         row = re.sub(arch.re_comment, "", row)
+        line_num_str = row.split(":")[0]
         row = row.rstrip()
         tabs = row.split("\t")
         row = "\t".join(tabs[2:])
-        line_num = eval_line_num(tabs[0].strip())
+        line_num = eval_line_num(line_num_str.strip())
 
         if line_num in data_refs:
             refs = data_refs[line_num]
@@ -1835,20 +1898,13 @@ def process(dump: str, config: Config) -> List[Line]:
 
         while i < len(lines):
             reloc_row = lines[i]
-            if "R_AARCH64_" in reloc_row:
-                # TODO: handle relocation
-                pass
-            elif "R_MIPS_" in reloc_row:
-                original = process_mips_reloc(reloc_row, original, arch)
-            elif "R_PPC_" in reloc_row:
-                original = process_ppc_reloc(reloc_row, original)
-            elif "R_ARM_" in reloc_row:
-                original = process_arm_reloc(reloc_row, original, arch)
+            if re.search(arch.re_reloc, reloc_row):
+                original = processor.process_reloc(reloc_row, original)
             else:
                 break
             i += 1
 
-        normalized_original = normalizer.normalize(mnemonic, original)
+        normalized_original = processor.normalize(mnemonic, original)
 
         scorable_line = normalized_original
         if not config.score_stack_differences:
@@ -1904,6 +1960,7 @@ def process(dump: str, config: Config) -> List[Line]:
         elif stop_after_delay_slot:
             break
 
+    processor.post_process(output)
     return output
 
 
@@ -2123,8 +2180,15 @@ class OutputLine:
 class Diff:
     lines: List[OutputLine]
     score: int
+    max_score: int
 
 
+def trim_nops(lines: List[Line], arch: ArchSettings) -> List[Line]:
+    lines = lines[:]
+    while lines and lines[-1].mnemonic == "nop" and (len(lines) == 1 or lines[-2].mnemonic not in arch.delay_slot_instructions):
+        lines.pop()
+    return lines
+
 def do_diff(lines1: List[Line], lines2: List[Line], config: Config) -> Diff:
     if config.show_source:
         import cxxfilt
@@ -2152,8 +2216,12 @@ def do_diff(lines1: List[Line], lines2: List[Line], config: Config) -> Diff:
                     btset.add(bt)
                     sc(str(bt))
 
+    lines1 = trim_nops(lines1, arch)
+    lines2 = trim_nops(lines2, arch)
+
     diffed_lines = diff_lines(lines1, lines2, config.algorithm)
     score = score_diff_lines(diffed_lines, config)
+    max_score = len(lines1) * config.penalty_deletion
 
     line_num_base = -1
     line_num_offset = 0
@@ -2385,7 +2453,7 @@ def do_diff(lines1: List[Line], lines2: List[Line], config: Config) -> Diff:
         )
 
     output = output[config.skip_lines :]
-    return Diff(lines=output, score=score)
+    return Diff(lines=output, score=score, max_score=max_score)
 
 
 def chunk_diff_lines(
@@ -2461,6 +2529,7 @@ def align_diffs(
                 Text(f"{padding}PREVIOUS ({old_diff.score})"),
             ),
             current_score=new_diff.score,
+            max_score=new_diff.max_score,
             previous_score=old_diff.score,
         )
         old_chunks = chunk_diff_lines(old_diff.lines)
@@ -2504,6 +2573,7 @@ def align_diffs(
                 Text(f"{padding}CURRENT ({new_diff.score})"),
             ),
             current_score=new_diff.score,
+            max_score=new_diff.max_score,
             previous_score=None,
         )
         diff_lines = [(line, line) for line in new_diff.lines]