Several improvements to debug_map_diff.py

Support checking linkage diffs, and checking if REL data symbols are missing/misnamed.

1 files changed, 96 insertions, 38 deletions

diff --git a/tools/utilities/debug_map_diff.py b/tools/utilities/debug_map_diff.py
index 678e0ab8..bec1e986 100755
--- a/tools/utilities/debug_map_diff.py
+++ b/tools/utilities/debug_map_diff.py
@@ -8,6 +8,7 @@ from pathlib import Path
 import re
 import subprocess
 import argparse
+from collections import defaultdict
 
 arg_parse = argparse.ArgumentParser()
 arg_parse.add_argument("object_name", help="Name of the object to compare, e.g. d_a_bridge for a REL, or main/d_a_npc_fa1 for a main.dol object")
@@ -35,6 +36,13 @@ assert retcode == 0, "Failed to configure"
 retcode = subprocess.call(["ninja", base_map_path.relative_to(decomp_root_path)], cwd=decomp_root_path)
 assert retcode == 0, "Ninja build call failed"
 
+class Symbol:
+  def __init__(self, name: str, size: int, sym_type: str | None = None, linkage: str | None = None):
+    self.name = name
+    self.size = size
+    self.sym_type = sym_type
+    self.linkage = linkage
+
 def get_main_symbols(framework_map_contents: str, valid_obj_names = None):
   symbols = {}
   matches = re.findall(r"^  [0-9a-f]{8} ([0-9a-f]{6}) (?:[0-9a-f]{8})(?: +\d+)? (.+?)(?: \(entry of [^)]+\))? \t(?:\S+\.a )?([^\s\.]+)\.\S+ ?$", framework_map_contents, re.IGNORECASE | re.MULTILINE)
@@ -51,38 +59,63 @@ def get_main_symbols(framework_map_contents: str, valid_obj_names = None):
     if valid_obj_names is not None and obj_name not in valid_obj_names:
       continue
     
-    symbols[name] = size
+    symbols[name] = Symbol(name, size)
   if len(symbols) == 0:
     raise Exception("Failed to find object matching the given name")
   return symbols
 
 def get_rel_symbols(rel_map_data: str):
   rel_map_lines = rel_map_data.splitlines()
-  found_memory_map = False
-  next_section_index = 0
-  section_name_to_section_index = {}
-  for line in rel_map_lines:
-    if line.strip() == "Memory map:":
-      found_memory_map = True
-    if found_memory_map:
-      section_match = re.search(r"^ +\.(text|ctors|dtors|rodata|data|bss)  [0-9a-f]{8} ([0-9a-f]{8}) [0-9a-f]{8}$", line)
-      if section_match:
-        section_name = section_match.group(1)
-        section_size = int(section_match.group(2), 16)
-        if section_size > 0:
-          section_name_to_section_index[section_name] = next_section_index
-          next_section_index += 1
-  if not found_memory_map:
-    raise Exception("Failed to find memory map")
+  
+  line = rel_map_lines.pop(0)
+  assert line == "Link map of _prolog"
+  symbol_name_to_type_linkage_obj = {}
+  while True:
+    line = rel_map_lines.pop(0)
+    if line == "":
+      # End of link map
+      break
+    if line.startswith(">>> SYMBOL NOT FOUND: "):
+      continue
+    linker_symbol_match = re.search(r"^  +(\d+)\] (\S+) found as linker generated symbol$", line)
+    if linker_symbol_match:
+      continue
+    normal_symbol_match = re.search(r"^  +(\d+)\] (\S+) \((section|func|object|notype),(global|local|weak)\) found in (\S+) $", line)
+    assert normal_symbol_match is not None, f"Unknown line: {repr(line)}"
+    link_map_depth = int(normal_symbol_match.group(1))
+    symbol_name = normal_symbol_match.group(2)
+    symbol_type = normal_symbol_match.group(3)
+    linkage = normal_symbol_match.group(4)
+    object_name = normal_symbol_match.group(5)
+    assert symbol_name not in symbol_name_to_type_linkage_obj
+    symbol_name_to_type_linkage_obj[symbol_name] = (symbol_type, linkage, object_name)
+  
+  # found_memory_map = False
+  # next_section_index = 0
+  # section_name_to_section_index = {}
+  # for line in rel_map_lines:
+  #   if line.strip() == "Memory map:":
+  #     found_memory_map = True
+  #   if found_memory_map:
+  #     section_match = re.search(r"^ +\.(text|ctors|dtors|rodata|data|bss)  [0-9a-f]{8} ([0-9a-f]{8}) [0-9a-f]{8}$", line)
+  #     if section_match:
+  #       section_name = section_match.group(1)
+  #       section_size = int(section_match.group(2), 16)
+  #       if section_size > 0:
+  #         section_name_to_section_index[section_name] = next_section_index
+  #         next_section_index += 1
+  # if not found_memory_map:
+  #   raise Exception("Failed to find memory map")
   
   symbols = {}
-  current_section_name = None
+  # current_section_name = None
+  localstatic_counters = defaultdict(int)
   for line in rel_map_lines:
-    section_header_match = re.search(r"^\.(text|ctors|dtors|rodata|data|bss) section layout$", line)
-    if section_header_match:
-      current_section_name = section_header_match.group(1)
-    if current_section_name != "text":
-        continue
+    # section_header_match = re.search(r"^\.(text|ctors|dtors|rodata|data|bss) section layout$", line)
+    # if section_header_match:
+    #   current_section_name = section_header_match.group(1)
+    # if current_section_name != "text":
+    #     continue
     symbol_entry_match = re.search(r"^  ([0-9a-f]{8}|UNUSED  ) ([0-9a-f]{6}) ([0-9a-f]{8}|\.{8})(?: +\d+)? (.+?)(?: \(entry of [^)]+\))? \t?(\S+)", line, re.IGNORECASE)
     if symbol_entry_match:
       symbol_address = symbol_entry_match.group(1)
@@ -98,9 +131,21 @@ def get_rel_symbols(rel_map_data: str):
         symbol_offset = int(symbol_offset, 16)
       symbol_name = symbol_entry_match.group(4)
       object_name = symbol_entry_match.group(5)
+      
       if object_name in ["executor.o", "global_destructor_chain.o"]:
         continue
-      symbols[symbol_name] = symbol_size
+      if re.search(r"^@\d+$", symbol_name):
+        continue
+      if localstatic_match := re.search(r"^([^\s\$]+)\$\d+$", symbol_name):
+        localstatic_name = localstatic_match.group(1)
+        localstatic_counters[localstatic_name] += 1
+        symbol_name = f"{localstatic_name}${localstatic_counters[localstatic_name]}"
+      
+      if symbol_name in symbol_name_to_type_linkage_obj:
+        symbol_type, linkage, object_name = symbol_name_to_type_linkage_obj[symbol_name]
+      else:
+        symbol_type = linkage = object_name = None
+      symbols[symbol_name] = Symbol(symbol_name, symbol_size, sym_type=symbol_type, linkage=linkage)
       #print("%08X  %s" % (symbol_offset, symbol_name))
   
   #print(rel_symbol_names)
@@ -127,18 +172,20 @@ total_missing = 0
 total_fake = 0
 total_right_size = 0
 total_wrong_size = 0
-for symbol_name, target_size in target_symbols.items():
-    if target_size == 0:
-      continue
-    if symbol_name not in base_symbols:
-      base_size = 0
-    else:
-      base_size = base_symbols[symbol_name]
-    size_diff = abs(target_size - base_size)
-    ratio = size_diff / target_size
-    if size_diff != 0:
-      total_wrong_size += 1
-    symbol_size_diffs.append((symbol_name, target_size, base_size, ratio))
+total_wrong_linkage = 0
+
+for symbol_name, target_symbol in target_symbols.items():
+  if target_symbol.size == 0:
+    continue
+  if symbol_name not in base_symbols:
+    base_size = 0
+  else:
+    base_size = base_symbols[symbol_name].size
+  size_diff = abs(target_symbol.size - base_size)
+  ratio = size_diff / target_symbol.size
+  if size_diff != 0:
+    total_wrong_size += 1
+  symbol_size_diffs.append((symbol_name, target_symbol.size, base_size, ratio))
 
 symbol_size_diffs.sort(key=lambda x: x[-1])
 
@@ -151,13 +198,13 @@ for symbol_name, target_size, base_size, ratio in symbol_size_diffs:
   elif size_diff == 0:
     prefix = "GOOD: "
     total_right_size += 1
-  elif ratio < 0.03:
+  elif ratio < 0.05 and size_diff <= 0x10:
     prefix = "CLOSE: "
   else:
     prefix = "WRONG: "
   print(prefix + symbol_name, "0x%X" % target_size, "0x%X" % base_size, ratio)
   
-for symbol_name, base_size in base_symbols.items():
+for symbol_name, base_symbol in base_symbols.items():
   if symbol_name in target_symbols:
     continue
   print("FAKE:", symbol_name)
@@ -172,8 +219,19 @@ for symbol_name, target_size, base_size, ratio in symbol_size_diffs:
     total_missing += 1
     print(prefix + symbol_name, "0x%X" % target_size)
 
+for symbol_name, target_symbol in target_symbols.items():
+  if target_symbol.size == 0:
+    continue
+  if symbol_name not in base_symbols:
+    continue
+  base_symbol = base_symbols[symbol_name]
+  if target_symbol.linkage != base_symbol.linkage:
+    total_wrong_linkage += 1
+    print(f"LINKAGE: {symbol_name} (should be {target_symbol.linkage}, is {base_symbol.linkage})")
+
 print("==================================================")
 print(f"Total right size: {total_right_size}")
 print(f"Total wrong size: {total_wrong_size}")
 print(f"Total fake: {total_fake}")
 print(f"Total missing: {total_missing}")
+print(f"Total wrong linkage: {total_wrong_linkage}")