Make debug_map_diff.py much more accurate for main.dol TUs

Functions are now marked as "FAKE?" instead of "FAKE" in cases where it's possible that they only seem fake due to the linker map not showing duplicate symbols multiple times (inlines used in multiple TUs).

"UNREFERENCED DUPLICATE" data in the base maps are now also taken into account. Also, limited linkage diffing can be done for main.dol symbols (based on the naming convention of l_ and g_ prefixes).

1 files changed, 170 insertions, 118 deletions

diff --git a/tools/utilities/debug_map_diff.py b/tools/utilities/debug_map_diff.py
index 02a7f0cb..64d283bb 100755
--- a/tools/utilities/debug_map_diff.py
+++ b/tools/utilities/debug_map_diff.py
@@ -17,8 +17,8 @@ from collections import defaultdict
 arg_parse = argparse.ArgumentParser()
 arg_parse.add_argument("object_name", help="Name of the object to compare, e.g. d_a_bridge or d_a_npc_fa1")
 args = arg_parse.parse_args()
-object_name: str = args.object_name
-assert "/" not in object_name and "." not in object_name, "The object name should not contain slashes or dots"
+target_object_name: str = args.object_name
+assert "/" not in target_object_name and "." not in target_object_name, "The object name should not contain slashes or dots"
 
 debug_maps_root_path = Path("orig/D44J01/files/maps")
 decomp_root_path = Path(".")
@@ -32,7 +32,7 @@ for ninja_target in subprocess.check_output(["ninja", "-t", "targets", "all"]).d
   all_ninja_outputs.append(ninja_output)
 
 target_map_path_dol = debug_maps_root_path / "frameworkD.map"
-target_map_path_rel = debug_maps_root_path / f"{object_name}D.map"
+target_map_path_rel = debug_maps_root_path / f"{target_object_name}D.map"
 if target_map_path_rel.exists():
   target_is_rel = True
   target_map_path = target_map_path_rel
@@ -44,7 +44,7 @@ del target_map_path_dol
 del target_map_path_rel
 
 base_map_path_dol = decomp_root_path / "build/D44J01/framework.elf.MAP"
-base_map_path_rel = decomp_root_path / f"build/D44J01/{object_name}/{object_name}.plf.MAP"
+base_map_path_rel = decomp_root_path / f"build/D44J01/{target_object_name}/{target_object_name}.plf.MAP"
 if str(base_map_path_rel) in all_ninja_outputs:
   base_is_rel = True
   base_map_path = base_map_path_rel
@@ -59,121 +59,135 @@ retcode = subprocess.call(["ninja", base_map_path.relative_to(decomp_root_path)]
 assert retcode == 0, "Ninja build call failed"
 
 class Symbol:
-  def __init__(self, name: str, size: int, sym_type: str | None = None, linkage: str | None = None):
+  def __init__(self, name: str, size: int, sym_type: str | None = None, linkage: str | None = None, stripped: bool | None = None):
     self.name = name
     self.size = size
     self.sym_type = sym_type
     self.linkage = linkage
+    self.stripped = stripped
+  
+  def __repr__(self):
+    return f"Symbol(name={self.name}, size={self.size}, sym_type={self.sym_type}, linkage={self.linkage}, stripped={self.stripped})"
 
-def get_main_symbols(framework_map_contents: str, valid_obj_names = None):
-  symbols = {}
-  localstatic_counters = defaultdict(int)
-  matches = re.findall(r"^  [0-9a-f]{8} ([0-9a-f]{6}) (?:[0-9a-f]{8})(?: +\d+)? (.+?)(?: \(entry of [^)]+\))? \t(?:\S+\.a )?([^\s\.]+)\.\S+ ?$", framework_map_contents, re.IGNORECASE | re.MULTILINE)
-  for match in matches:
-    size, symbol_name, obj_name = match
-    size = int(size, 16)
-    
-    if valid_obj_names is not None and obj_name not in valid_obj_names:
-      continue
-    if symbol_name.startswith("."):
-      continue
-    if re.search(r"^@\d+$", symbol_name):
-      continue
-    if localstatic_match := re.search(r"^([^\s\$]+)\$\d+$", symbol_name):
-      localstatic_name = localstatic_match.group(1)
-      localstatic_counters[localstatic_name] += 1
-      symbol_name = f"{localstatic_name}${localstatic_counters[localstatic_name]}"
-    
-    symbols[symbol_name] = Symbol(symbol_name, size)
-  if len(symbols) == 0:
-    raise Exception("Failed to find object matching the given name (check for typos)")
-  return symbols
-
-def get_rel_symbols(rel_map_data: str):
-  rel_map_lines = rel_map_data.splitlines()
+def get_symbols_from_linker_map(map_contents: str, missing_tree_and_stripped=False):
+  map_lines = map_contents.splitlines()
   
-  line = rel_map_lines.pop(0)
-  assert line == "Link map of _prolog"
-  symbol_name_to_type_linkage_obj = {}
-  while True:
-    line = rel_map_lines.pop(0)
-    if line == "":
-      # End of link map
-      break
-    if line.startswith(">>> SYMBOL NOT FOUND: "):
-      continue
-    linker_symbol_match = re.search(r"^  +(\d+)\] (\S+) found as linker generated symbol$", line)
-    if linker_symbol_match:
-      continue
-    normal_symbol_match = re.search(r"^  +(\d+)\] (\S+) \((section|func|object|notype),(global|local|weak)\) found in (\S+) $", line)
-    assert normal_symbol_match is not None, f"Unknown line: {repr(line)}"
-    link_map_depth = int(normal_symbol_match.group(1))
-    symbol_name = normal_symbol_match.group(2)
-    symbol_type = normal_symbol_match.group(3)
-    linkage = normal_symbol_match.group(4)
-    object_name = normal_symbol_match.group(5)
-    assert symbol_name not in symbol_name_to_type_linkage_obj
-    symbol_name_to_type_linkage_obj[symbol_name] = (symbol_type, linkage, object_name)
+  object_name_to_ext = {}
+  def remove_object_ext(object_name: str):
+    object_name, object_ext = object_name.split(".", 1)
+    if object_name in object_name_to_ext:
+      # Check to be sure we never have two objects with the same name but different extensions
+      assert object_name_to_ext[object_name] == object_ext
+    else:
+      object_name_to_ext[object_name] = object_ext
+    return object_name
   
-  # found_memory_map = False
-  # next_section_index = 0
-  # section_name_to_section_index = {}
-  # for line in rel_map_lines:
-  #   if line.strip() == "Memory map:":
-  #     found_memory_map = True
-  #   if found_memory_map:
-  #     section_match = re.search(r"^ +\.(text|ctors|dtors|rodata|data|bss)  [0-9a-f]{8} ([0-9a-f]{8}) [0-9a-f]{8}$", line)
-  #     if section_match:
-  #       section_name = section_match.group(1)
-  #       section_size = int(section_match.group(2), 16)
-  #       if section_size > 0:
-  #         section_name_to_section_index[section_name] = next_section_index
-  #         next_section_index += 1
-  # if not found_memory_map:
-  #   raise Exception("Failed to find memory map")
+  object_name_to_symbol_name_to_type_and_linkage = defaultdict(dict)
+  localstatic_counters = defaultdict(lambda: defaultdict(int))
+  unref_dupe_symbol_names_to_object_name_to_type_linkage = {}
+  if not missing_tree_and_stripped:
+    line = map_lines.pop(0)
+    assert line == "Link map of _prolog" or line == "Link map of __start"
+    curr_unref_dupe_symbol_name = None
+    while True:
+      line = map_lines.pop(0)
+      if line == "":
+        # End of link map
+        break
+      
+      unref_dupe_symbol_match = re.search(r"^  +(\d+)\] >>> UNREFERENCED DUPLICATE (\S+)$", line)
+      if unref_dupe_symbol_match:
+        link_map_depth = int(unref_dupe_symbol_match.group(1))
+        curr_unref_dupe_symbol_name = unref_dupe_symbol_match.group(2)
+        assert curr_unref_dupe_symbol_name not in unref_dupe_symbol_names_to_object_name_to_type_linkage
+        unref_dupe_symbol_names_to_object_name_to_type_linkage[curr_unref_dupe_symbol_name] = {}
+        continue
+      unref_dupe_linkage_match = re.search(r"^  +(\d+)\] >>> \((func|object),(weak)\) found in (\S+) $", line)
+      if unref_dupe_linkage_match:
+        assert curr_unref_dupe_symbol_name is not None
+        link_map_depth = int(unref_dupe_linkage_match.group(1))
+        symbol_type = unref_dupe_linkage_match.group(2)
+        linkage = unref_dupe_linkage_match.group(3)
+        object_name = remove_object_ext(unref_dupe_linkage_match.group(4))
+        
+        # A symbol shouldn't be duplicates multiple times in the same object, but it's possible for multiple objects to have the exact same name.
+        # e.g. getRaw__Q37JGadget6binary10TParseDataCFv is in 5 different TUs named control.o
+        # Not really any way to detect this, so just ignore it and check that the types all match.
+        # assert object_name not in unref_dupe_symbol_names_to_object_name_to_type_linkage[curr_unref_dupe_symbol_name]
+        if object_name in unref_dupe_symbol_names_to_object_name_to_type_linkage[curr_unref_dupe_symbol_name]:
+          assert (symbol_type, linkage) == unref_dupe_symbol_names_to_object_name_to_type_linkage[curr_unref_dupe_symbol_name][object_name]
+        
+        unref_dupe_symbol_names_to_object_name_to_type_linkage[curr_unref_dupe_symbol_name][object_name] = (symbol_type, linkage)
+        continue
+      else:
+        curr_unref_dupe_symbol_name = None
+      
+      if line.startswith(">>> SYMBOL NOT FOUND: "):
+        continue
+      linker_symbol_match = re.search(r"^  +(\d+)\] (\S+) found as linker generated symbol$", line)
+      if linker_symbol_match:
+        continue
+      normal_symbol_match = re.search(r"^  +(\d+)\] (\S+) \((section|func|object|notype),(global|local|weak)\) found in (\S+) $", line)
+      assert normal_symbol_match is not None, f"Unknown line: {repr(line)}"
+      link_map_depth = int(normal_symbol_match.group(1))
+      symbol_name = normal_symbol_match.group(2)
+      symbol_type = normal_symbol_match.group(3)
+      linkage = normal_symbol_match.group(4)
+      object_name = remove_object_ext(normal_symbol_match.group(5))
+      
+      if localstatic_match := re.search(r"^([^\s\$]+)\$\d+$", symbol_name):
+        localstatic_name = localstatic_match.group(1)
+        localstatic_counters[object_name][localstatic_name] += 1
+        symbol_name = f"{localstatic_name}${localstatic_counters[object_name][localstatic_name]}"
+      
+      assert symbol_name not in object_name_to_symbol_name_to_type_and_linkage[object_name], f"Duplicate symbol: {repr(symbol_name)}"
+      object_name_to_symbol_name_to_type_and_linkage[object_name][symbol_name] = (symbol_type, linkage)
   
-  symbols = {}
-  # current_section_name = None
-  localstatic_counters = defaultdict(int)
-  for line in rel_map_lines:
-    # section_header_match = re.search(r"^\.(text|ctors|dtors|rodata|data|bss) section layout$", line)
-    # if section_header_match:
-    #   current_section_name = section_header_match.group(1)
-    # if current_section_name != "text":
-    #     continue
+  symbols: dict[str, dict[str, Symbol]] = defaultdict(dict)
+  localstatic_counters = defaultdict(lambda: defaultdict(int))
+  unref_dupe_symbol_names_already_added = set()
+  for line in map_lines:
     symbol_entry_match = re.search(r"^  ([0-9a-f]{8}|UNUSED  ) ([0-9a-f]{6}) ([0-9a-f]{8}|\.{8})(?: +\d+)? (.+?)(?: \(entry of [^)]+\))? \t?(\S+)", line, re.IGNORECASE)
     if symbol_entry_match:
-      symbol_address = symbol_entry_match.group(1)
+      symbol_offset = symbol_entry_match.group(1)
       symbol_size = symbol_entry_match.group(2)
       symbol_size = int(symbol_size, 16)
-      symbol_offset = symbol_entry_match.group(3)
-      if symbol_address == "UNUSED  ":
-        assert symbol_offset == "........"
-      if symbol_offset == "........":
-        assert symbol_address == "UNUSED  "
-        symbol_offset = None
+      symbol_address = symbol_entry_match.group(3)
+      if symbol_offset == "UNUSED  ":
+        assert symbol_address == "........"
+      if symbol_address == "........":
+        assert symbol_offset == "UNUSED  "
+        symbol_address = None
+        stripped = True
       else:
-        symbol_offset = int(symbol_offset, 16)
+        symbol_address = int(symbol_address, 16)
+        stripped = False
       symbol_name = symbol_entry_match.group(4)
-      object_name = symbol_entry_match.group(5)
+      object_name = remove_object_ext(symbol_entry_match.group(5))
       
-      if object_name in ["executor.o", "global_destructor_chain.o"]:
+      if symbol_name.startswith(".") or symbol_name in ["extab", "extabindex"]:
+        # e.g. Section symbol (.text) or pool symbol (...data)
         continue
       if re.search(r"^@\d+$", symbol_name):
         continue
       if localstatic_match := re.search(r"^([^\s\$]+)\$\d+$", symbol_name):
         localstatic_name = localstatic_match.group(1)
-        localstatic_counters[localstatic_name] += 1
-        symbol_name = f"{localstatic_name}${localstatic_counters[localstatic_name]}"
+        localstatic_counters[object_name][localstatic_name] += 1
+        symbol_name = f"{localstatic_name}${localstatic_counters[object_name][localstatic_name]}"
       
-      if symbol_name in symbol_name_to_type_linkage_obj:
-        symbol_type, linkage, object_name = symbol_name_to_type_linkage_obj[symbol_name]
+      if symbol_name in object_name_to_symbol_name_to_type_and_linkage[object_name]:
+        symbol_type, linkage = object_name_to_symbol_name_to_type_and_linkage[object_name][symbol_name]
       else:
-        symbol_type = linkage = object_name = None
-      symbols[symbol_name] = Symbol(symbol_name, symbol_size, sym_type=symbol_type, linkage=linkage)
-      #print("%08X  %s" % (symbol_offset, symbol_name))
-  
-  #print(rel_symbol_names)
+        assert stripped or missing_tree_and_stripped, f"Symbol {repr(symbol_name)} is missing linkage information in object {repr(object_name)}"
+        symbol_type = linkage = None
+      symbols[object_name][symbol_name] = Symbol(symbol_name, symbol_size, sym_type=symbol_type, linkage=linkage, stripped=stripped)
+      
+      if symbol_name in unref_dupe_symbol_names_to_object_name_to_type_linkage and symbol_name not in unref_dupe_symbol_names_already_added:
+        for other_object_name in unref_dupe_symbol_names_to_object_name_to_type_linkage[symbol_name]:
+          symbol_type, linkage = unref_dupe_symbol_names_to_object_name_to_type_linkage[symbol_name][other_object_name]
+          assert stripped == False, "Shouldn't reach a stripped duplicate here"
+          symbols[other_object_name][symbol_name] = Symbol(symbol_name, symbol_size, sym_type=symbol_type, linkage=linkage, stripped=stripped)
+        unref_dupe_symbol_names_already_added.add(symbol_name)
   
   return symbols
 
@@ -182,21 +196,35 @@ def is_debug_only_symbol(symbol_name: str):
     return True
   return False
 
-if target_is_rel:
-  target_symbols = get_rel_symbols(target_map_path.read_text())
-else:
-  target_symbols = get_main_symbols(target_map_path.read_text(), valid_obj_names=[object_name])
+target_missing_tree_and_stripped = not target_is_rel
 
-if base_is_rel:
-  base_symbols = get_rel_symbols(base_map_path.read_text())
-else:
-  base_symbols = get_main_symbols(base_map_path.read_text(), valid_obj_names=[object_name])
+all_target_symbols = get_symbols_from_linker_map(target_map_path.read_text(), missing_tree_and_stripped=target_missing_tree_and_stripped)
+target_symbols = all_target_symbols[target_object_name]
+if len(target_symbols) == 0:
+  raise Exception("Failed to find object matching the given name (check for typos)")
+
+all_base_symbols = get_symbols_from_linker_map(base_map_path.read_text())
+base_symbols = all_base_symbols[target_object_name]
+if len(base_symbols) == 0:
+  raise Exception("Failed to find object matching the given name (check for typos)")
 
 print(len(target_symbols), len(base_symbols))
 
+target_symbol_names_in_previous_objects = set()
+if target_missing_tree_and_stripped:
+  # This handles the logic for checking if a symbol already appeared earlier on in framework.map.
+  # Note: I'm not sure if the logic here is 100% accurate for edge cases since it just relies on dict insertion order.
+  # Might need to add more robust logic later that keeps track of symbol section, address, or line number within the map file...?
+  for object_name, symbols in all_target_symbols.items():
+    if object_name == target_object_name:
+      break
+    for symbol_name, symbol in symbols.items():
+      target_symbol_names_in_previous_objects.add(symbol.name)
+
 symbol_size_diffs = []
 total_missing = 0
 total_fake = 0
+total_maybe_fake = 0
 total_right_size = 0
 total_wrong_size = 0
 total_wrong_linkage = 0
@@ -210,7 +238,7 @@ for symbol_name, target_symbol in target_symbols.items():
     base_size = base_symbols[symbol_name].size
   size_diff = abs(target_symbol.size - base_size)
   ratio = size_diff / target_symbol.size
-  if size_diff != 0:
+  if symbol_name in base_symbols and size_diff != 0:
     total_wrong_size += 1
   symbol_size_diffs.append((symbol_name, target_symbol.size, base_size, ratio))
 
@@ -231,11 +259,23 @@ for symbol_name, target_size, base_size, ratio in symbol_size_diffs:
     prefix = "WRONG: "
   print(prefix + symbol_name, "0x%X" % target_size, "0x%X" % base_size, ratio)
   
+maybe_fake_symbols = []
+fake_symbols = []
 for symbol_name, base_symbol in base_symbols.items():
   if symbol_name in target_symbols:
     continue
-  print("FAKE:", symbol_name, "0x%X" % base_symbol.size)
-  total_fake += 1
+  if target_missing_tree_and_stripped and (base_symbol.stripped or symbol_name in target_symbol_names_in_previous_objects):
+    maybe_fake_symbols.append(base_symbol)
+  else:
+    fake_symbols.append(base_symbol)
+
+for base_symbol in maybe_fake_symbols:
+    print("FAKE?:", base_symbol.name, "0x%X" % base_symbol.size)
+    total_maybe_fake += 1
+
+for base_symbol in fake_symbols:
+    print("FAKE:", base_symbol.name, "0x%X" % base_symbol.size)
+    total_fake += 1
 
 for symbol_name, target_size, base_size, ratio in symbol_size_diffs:
   prefix = ""
@@ -246,20 +286,32 @@ for symbol_name, target_size, base_size, ratio in symbol_size_diffs:
     total_missing += 1
     print(prefix + symbol_name, "0x%X" % target_size)
 
-if target_is_rel:
-  for symbol_name, target_symbol in target_symbols.items():
-    if target_symbol.size == 0:
-      continue
-    if symbol_name not in base_symbols:
-      continue
-    base_symbol = base_symbols[symbol_name]
-    if target_symbol.linkage != base_symbol.linkage:
-      total_wrong_linkage += 1
-      print(f"LINKAGE: {symbol_name} (should be {target_symbol.linkage}, is {base_symbol.linkage})")
+for symbol_name, target_symbol in target_symbols.items():
+  if target_symbol.size == 0:
+    continue
+  if symbol_name not in base_symbols:
+    continue
+  base_symbol = base_symbols[symbol_name]
+  wrong_linkage = False
+  if target_symbol.linkage is None and base_symbol.sym_type == "object":
+    # The official framework.map for main.dol doesn't include linkage, but we can guess it based off of certain symbol name prefixes.
+    if base_symbol.name.startswith("l_") and base_symbol.linkage != "local":
+      wrong_linkage = True
+      target_linkage = "local"
+    elif base_symbol.name.startswith("g_") and base_symbol.linkage != "global":
+      wrong_linkage = True
+      target_linkage = "global"
+  elif target_symbol.linkage is not None and target_symbol.linkage != base_symbol.linkage:
+    wrong_linkage = True
+    target_linkage = target_symbol.linkage
+  if wrong_linkage:
+    total_wrong_linkage += 1
+    print(f"LINKAGE: {symbol_name} (should be {target_linkage}, is {base_symbol.linkage})")
 
 print("==================================================")
 print(f"Total right size: {total_right_size}")
 print(f"Total wrong size: {total_wrong_size}")
 print(f"Total fake: {total_fake}")
+print(f"Total maybe fake: {total_maybe_fake}")
 print(f"Total missing: {total_missing}")
 print(f"Total wrong linkage: {total_wrong_linkage}")