Full Disasm (#55)

* git subrepo pull --force tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "6ec2b39"
upstream:
  origin:   "git@github.com:ethteck/splat.git"
  branch:   "master"
  commit:   "6ec2b39"
git-subrepo:
  version:  "0.4.6"
  origin:   "git@github.com:ingydotnet/git-subrepo.git"
  commit:   "110b9eb"

* Add FULL_DISASM toggle

* Fix bootclear

12 files changed, 208 insertions, 46 deletions

diff --git a/tools/splat/.gitrepo b/tools/splat/.gitrepo
index 24399af..2cad301 100644
--- a/tools/splat/.gitrepo
+++ b/tools/splat/.gitrepo
@@ -6,7 +6,7 @@
 [subrepo]
 	remote = git@github.com:ethteck/splat.git
 	branch = master
-	commit = 20922caeffe970c179d4aa5f009d8b174d6afda3
-	parent = 814c2590fabd186dbe3db269b4d5b1af3d40dff8
+	commit = 6ec2b39108ac3891824896dc33a4de56765f962d
+	parent = f19df74c6ee597f01c3d7def1fdef105d9bdef92
 	method = merge
-	cmdver = 0.4.3
+	cmdver = 0.4.6
diff --git a/tools/splat/CHANGELOG.md b/tools/splat/CHANGELOG.md
index b92ea15..a72a12e 100644
--- a/tools/splat/CHANGELOG.md
+++ b/tools/splat/CHANGELOG.md
@@ -1,5 +1,47 @@
 # splat Release Notes
 
+### 0.16.9
+
+* Add command line argument `--disassemble-all`, which has the same effect as the `disassemble_all` yaml option so will disamble already matched functions as well as migrated data.
+  * Note: the command line argument takes precedence over the yaml, so will take effect even if the yaml option is set to false.
+
+### 0.16.8
+
+* Avoid ignoring the `align` defined in a segment for `code` segments
+
+### 0.16.7
+
+* Use `pylibyaml` to speed-up yaml parsing
+
+### 0.16.6
+
+* Add option `ld_rom_start`.
+  * Allows offsetting rom address linker symbols by some arbitrary value.
+    * Useful for SN64 games which often have rom addresses offset by 0xB0000000.
+  * Defaults to 0.
+
+### 0.16.5
+
+* Add option `segment_symbols_style`.
+  * Allows changing the style of the generated segment symbols in the linker script.
+  * Possible values:
+    * `splat`: The current style for segment symbols.
+    * `makerom`: Style that aims to be compatible with makerom generated symbols.
+  * Defaults to `splat`.
+
+### 0.16.4
+
+* Add `get_section_flags` method to the `Segment` class.
+  * Useful for providing linker section flags when creating a custom section when making splat extensions.
+  * This may be necessary for some custom section types, because sections unrecognized by the linker will not link its data properly.
+  * More info about section flags: <https://sourceware.org/binutils/docs/as/Section.html#ELF-Version>
+
+### 0.16.3
+
+* Add `--stdout-only` flag. Redirects the progress bar output to `stdout` instead of `stderr`.
+* Add a check to prevent relocs with duplicated rom addresses.
+* Check empty functions only have 2 instructions before autodecompiling them.
+
 ### 0.16.2
 
 * Add option `disassemble_all`. If enabled then already matched functions and migrated data will be disassembled to files anyways.
diff --git a/tools/splat/segtypes/common/c.py b/tools/splat/segtypes/common/c.py
index 21a155d..f23e9cd 100644
--- a/tools/splat/segtypes/common/c.py
+++ b/tools/splat/segtypes/common/c.py
@@ -342,6 +342,7 @@ class CommonSegC(CommonSegCodeSubsegment):
         # Terrible hack to "auto-decompile" empty functions
         if (
             options.opts.auto_decompile_empty_functions
+            and len(func.instructions) == 2
             and func.instructions[0].isReturn()
             and func.instructions[1].isNop()
         ):
diff --git a/tools/splat/segtypes/common/code.py b/tools/splat/segtypes/common/code.py
index eda9936..295776f 100644
--- a/tools/splat/segtypes/common/code.py
+++ b/tools/splat/segtypes/common/code.py
@@ -7,7 +7,7 @@ from util.range import Range
 from util.symbols import Symbol
 
 from segtypes.common.group import CommonSegGroup
-from segtypes.segment import Segment
+from segtypes.segment import Segment, parse_segment_align
 
 CODE_TYPES = ["c", "asm", "hasm"]
 
@@ -44,7 +44,10 @@ class CommonSegCode(CommonSegGroup):
         self.jtbl_glabels_to_add: Set[int] = set()
         self.jumptables: Dict[int, Tuple[int, int]] = {}
         self.rodata_syms: Dict[int, List[Symbol]] = {}
-        self.align = 0x10
+
+        self.align = parse_segment_align(yaml)
+        if self.align is None:
+            self.align = 0x10
 
     @property
     def needs_symbols(self) -> bool:
diff --git a/tools/splat/segtypes/common/data.py b/tools/splat/segtypes/common/data.py
index cb09df4..81c648e 100644
--- a/tools/splat/segtypes/common/data.py
+++ b/tools/splat/segtypes/common/data.py
@@ -55,7 +55,12 @@ class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
             preamble = options.opts.generated_s_preamble
             if preamble:
                 f.write(preamble + "\n")
-            f.write(f".section {self.get_linker_section()}\n\n")
+
+            f.write(f".section {self.get_linker_section()}")
+            section_flags = self.get_section_flags()
+            if section_flags:
+                f.write(f', "{section_flags}"')
+            f.write("\n\n")
 
             f.write(self.spim_section.disassemble())
 
diff --git a/tools/splat/segtypes/linker_entry.py b/tools/splat/segtypes/linker_entry.py
index 981d3d3..fa49e57 100644
--- a/tools/splat/segtypes/linker_entry.py
+++ b/tools/splat/segtypes/linker_entry.py
@@ -68,8 +68,63 @@ def segment_cname(segment: Segment) -> str:
     return to_cname(name)
 
 
+def get_segment_rom_start(cname: str) -> str:
+    if options.opts.segment_symbols_style == "makerom":
+        return f"_{cname}SegmentRomStart"
+    return f"{cname}_ROM_START"
+
+
+def get_segment_rom_end(cname: str) -> str:
+    if options.opts.segment_symbols_style == "makerom":
+        return f"_{cname}SegmentRomEnd"
+    return f"{cname}_ROM_END"
+
+
+def get_segment_vram_start(cname: str) -> str:
+    if options.opts.segment_symbols_style == "makerom":
+        return f"_{cname}SegmentStart"
+    return f"{cname}_VRAM"
+
+
+def get_segment_vram_end(cname: str) -> str:
+    if options.opts.segment_symbols_style == "makerom":
+        return f"_{cname}SegmentEnd"
+    return f"{cname}_VRAM_END"
+
+
+def convert_section_name_to_linker_format(section_type: str) -> str:
+    assert section_type.startswith(".")
+    if options.opts.segment_symbols_style == "makerom":
+        if section_type == ".rodata":
+            return "RoData"
+        return section_type[1:].capitalize()
+
+    return to_cname(section_type.upper())
+
+
+def get_segment_section_start(segment_name: str, section_type: str) -> str:
+    sec = convert_section_name_to_linker_format(section_type)
+    if options.opts.segment_symbols_style == "makerom":
+        return f"_{segment_name}Segment{sec}Start"
+    return f"{segment_name}{sec}_START"
+
+
+def get_segment_section_end(segment_name: str, section_type: str) -> str:
+    sec = convert_section_name_to_linker_format(section_type)
+    if options.opts.segment_symbols_style == "makerom":
+        return f"_{segment_name}Segment{sec}End"
+    return f"{segment_name}{sec}_END"
+
+
+def get_segment_section_size(segment_name: str, section_type: str) -> str:
+    sec = convert_section_name_to_linker_format(section_type)
+    if options.opts.segment_symbols_style == "makerom":
+        return f"_{segment_name}Segment{sec}Size"
+    return f"{segment_name}{sec}_SIZE"
+
+
 def get_segment_vram_end_symbol_name(segment: Segment) -> str:
-    return segment_cname(segment) + "_VRAM_END"
+    return get_segment_vram_end(segment_cname(segment))
 
 
 @dataclass
@@ -120,7 +175,7 @@ class LinkerWriter:
 
         self._writeln("SECTIONS")
         self._begin_block()
-        self._writeln("__romPos = 0;")
+        self._writeln(f"__romPos = {options.opts.ld_rom_start};")
 
         if options.opts.gp is not None:
             self._writeln("_gp = " + f"0x{options.opts.gp:X};")
@@ -157,11 +212,13 @@ class LinkerWriter:
 
         # Start the first linker section
 
-        self._write_symbol(f"{seg_name}_ROM_START", "__romPos")
+        seg_rom_start = get_segment_rom_start(seg_name)
+        self._write_symbol(seg_rom_start, "__romPos")
 
         if entries[0].section_type == ".bss":
             self._begin_bss_segment(segment, is_first=True)
-            self._write_symbol(f"{seg_name}_BSS_START", ".")
+            seg_bss_start = get_segment_section_start(seg_name, ".bss")
+            self._write_symbol(seg_bss_start, ".")
             if ".bss" in section_labels:
                 section_labels[".bss"].started = True
         else:
@@ -196,12 +253,12 @@ class LinkerWriter:
                     elif cur_section == ".bss":
                         entering_bss = True
 
-                    if not (
-                        entering_bss or leaving_bss
-                    ):  # Don't write a START symbol if we are about to end the section
-                        self._write_symbol(
-                            f"{seg_name}{entry.section_type.upper()}_START", "."
+                    if not (entering_bss or leaving_bss):
+                        # Don't write a START symbol if we are about to end the section
+                        section_start = get_segment_section_start(
+                            seg_name, entry.section_type
                         )
+                        self._write_symbol(section_start, ".")
                         section_labels[entry.section_type].started = True
 
             if (
@@ -226,15 +283,9 @@ class LinkerWriter:
                     entry in last_seen_sections
                     and section_labels[entry.section_type].started
                 ):
-                    seg_name_section = to_cname(
-                        f"{seg_name}{last_seen_sections[entry].upper()}"
+                    self._end_section(
+                        seg_name, last_seen_sections[entry], section_labels
                     )
-                    self._write_symbol(f"{seg_name_section}_END", ".")
-                    self._write_symbol(
-                        f"{seg_name_section}_SIZE",
-                        f"ABSOLUTE({seg_name_section}_END - {seg_name_section}_START)",
-                    )
-                    section_labels[last_seen_sections[entry]].ended = True
 
                 self._end_block()
 
@@ -243,7 +294,8 @@ class LinkerWriter:
                 else:
                     self._begin_segment(segment)
 
-                self._write_symbol(f"{seg_name}{entry.section_type.upper()}_START", ".")
+                section_start = get_segment_section_start(seg_name, entry.section_type)
+                self._write_symbol(section_start, ".")
                 section_labels[cur_section].started = True
 
                 # Write THIS linker entry
@@ -257,25 +309,14 @@ class LinkerWriter:
 
                 # If this is the last entry of its type, add the END marker for the section we're ending
                 if entry in last_seen_sections:
-                    seg_name_section = to_cname(f"{seg_name}{cur_section.upper()}")
-                    self._write_symbol(f"{seg_name_section}_END", ".")
-                    self._write_symbol(
-                        f"{seg_name_section}_SIZE",
-                        f"ABSOLUTE({seg_name_section}_END - {seg_name_section}_START)",
-                    )
-                    section_labels[cur_section].ended = True
+                    self._end_section(seg_name, cur_section, section_labels)
 
             prev_section = cur_section
 
         # End all un-ended sections
         for section in section_labels.values():
             if section.started and not section.ended:
-                seg_name_section = to_cname(f"{seg_name}{section.name.upper()}")
-                self._write_symbol(f"{seg_name_section}_END", ".")
-                self._write_symbol(
-                    f"{seg_name_section}_SIZE",
-                    f"ABSOLUTE({seg_name_section}_END - {seg_name_section}_START)",
-                )
+                self._end_section(seg_name, section.name, section_labels)
 
         all_bss = all(e.section == ".bss" for e in entries)
         self._end_segment(segment, all_bss)
@@ -347,9 +388,11 @@ class LinkerWriter:
 
         name = segment_cname(segment)
 
-        self._write_symbol(f"{name}_VRAM", f"ADDR(.{name})")
+        seg_vram_start = get_segment_vram_start(name)
+        self._write_symbol(seg_vram_start, f"ADDR(.{name})")
 
-        line = f".{name} {vram_str}: AT({name}_ROM_START)"
+        seg_rom_start = get_segment_rom_start(name)
+        line = f".{name} {vram_str}: AT({seg_rom_start})"
         if segment.subalign != None:
             line += f" SUBALIGN({segment.subalign})"
 
@@ -368,7 +411,8 @@ class LinkerWriter:
 
         name = segment_cname(segment) + "_bss"
 
-        self._write_symbol(f"{name}_VRAM", f"ADDR(.{name})")
+        seg_vram_start = get_segment_vram_start(name)
+        self._write_symbol(seg_vram_start, f"ADDR(.{name})")
 
         if is_first:
             addr_str = vram_str + "(NOLOAD)"
@@ -395,7 +439,8 @@ class LinkerWriter:
             if segment.align:
                 self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});")
 
-        self._write_symbol(f"{name}_ROM_END", "__romPos")
+        seg_rom_end = get_segment_rom_end(name)
+        self._write_symbol(seg_rom_end, "__romPos")
         self._write_symbol(get_segment_vram_end_symbol_name(segment), ".")
 
         # Align directive
@@ -404,3 +449,19 @@ class LinkerWriter:
                 self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});")
 
         self._writeln("")
+
+    def _end_section(
+        self,
+        seg_name: str,
+        cur_section: str,
+        section_labels: OrderedDict[str, LinkerSection],
+    ) -> None:
+        section_start = get_segment_section_start(seg_name, cur_section)
+        section_end = get_segment_section_end(seg_name, cur_section)
+        section_size = get_segment_section_size(seg_name, cur_section)
+        self._write_symbol(section_end, ".")
+        self._write_symbol(
+            section_size,
+            f"ABSOLUTE({section_end} - {section_start})",
+        )
+        section_labels[cur_section].ended = True
diff --git a/tools/splat/segtypes/segment.py b/tools/splat/segtypes/segment.py
index 7d76be9..6f03426 100644
--- a/tools/splat/segtypes/segment.py
+++ b/tools/splat/segtypes/segment.py
@@ -428,6 +428,24 @@ class Segment:
     def get_linker_section(self) -> str:
         return ".data"
 
+    def get_section_flags(self) -> Optional[str]:
+        """
+        Allows specifying flags for a section.
+
+        This can be useful when creating a custom section, since sections not recognized by the linker will not be linked properly.
+
+        GNU as docs about the section directive and flags: https://sourceware.org/binutils/docs/as/Section.html#ELF-Version
+
+        Example:
+
+        ```
+        def get_section_flags(self) -> Optional[str]:
+            # Tells the linker to allocate this section
+            return "a"
+        ```
+        """
+        return None
+
     def out_path(self) -> Optional[Path]:
         return None
 
diff --git a/tools/splat/split.py b/tools/splat/split.py
index 9715c6f..8568450 100755
--- a/tools/splat/split.py
+++ b/tools/splat/split.py
@@ -7,7 +7,11 @@ import pickle
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 from disassembler import disassembler_instance
 from util import progress_bar
+
+# This unused import makes the yaml library faster. don't remove
+import pylibyaml  # pyright: ignore
 import yaml
+
 from colorama import Fore, Style
 from intervaltree import Interval, IntervalTree
 import sys
@@ -20,7 +24,7 @@ from segtypes.linker_entry import (
 from segtypes.segment import Segment
 from util import log, options, palettes, symbols, relocs
 
-VERSION = "0.16.2"
+VERSION = "0.16.9"
 
 parser = argparse.ArgumentParser(
     description="Split a rom given a rom, a config, and output directory"
@@ -39,6 +43,11 @@ parser.add_argument(
 parser.add_argument(
     "--stdout-only", help="Print all output to stdout", action="store_true"
 )
+parser.add_argument(
+    "--disassemble-all",
+    help="Disasemble matched functions and migrated data",
+    action="store_true",
+)
 
 linker_writer: LinkerWriter
 config: Dict[str, Any]
@@ -220,6 +229,7 @@ def main(
     use_cache=True,
     skip_version_check=False,
     stdout_only=False,
+    disassemble_all=False,
 ):
     global config
 
@@ -233,7 +243,7 @@ def main(
             additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
         config = merge_configs(config, additional_config)
 
-    options.initialize(config, config_path, modes, verbose)
+    options.initialize(config, config_path, modes, verbose, disassemble_all)
 
     disassembler_instance.create_disassembler_instance(options.opts.platform)
     disassembler_instance.get_instance().check_version(skip_version_check, VERSION)
@@ -492,4 +502,5 @@ if __name__ == "__main__":
         args.use_cache,
         args.skip_version_check,
         args.stdout_only,
+        args.disassemble_all,
     )
diff --git a/tools/splat/util/log.py b/tools/splat/util/log.py
index df6ef53..4f296da 100644
--- a/tools/splat/util/log.py
+++ b/tools/splat/util/log.py
@@ -1,5 +1,6 @@
 import sys
 from typing import NoReturn, Optional
+from pathlib import Path
 
 from colorama import Fore, init, Style
 
@@ -26,7 +27,7 @@ def error(*args, **kwargs) -> NoReturn:
 
 
 # The line_num is expected to be zero-indexed
-def parsing_error_preamble(path, line_num, line):
+def parsing_error_preamble(path: Path, line_num: int, line: str):
     write("")
     write(f"error reading {path}, line {line_num + 1}:", status="error")
     write(f"\t{line}")
diff --git a/tools/splat/util/options.py b/tools/splat/util/options.py
index 936d138..2229cad 100644
--- a/tools/splat/util/options.py
+++ b/tools/splat/util/options.py
@@ -104,6 +104,10 @@ class SplatOpts:
     ld_use_follows: bool
     # If enabled, the end symbol for each segment will be placed before the alignment directive for the segment
     segment_end_before_align: bool
+    # Controls the style of the auto-generated segment symbols in the linker script. Possible values: splat, makerom
+    segment_symbols_style: str
+    # Specifies the starting offset for rom address symbols in the linker script.
+    ld_rom_start: int
 
     ################################################################################
     # C file options
@@ -281,6 +285,7 @@ def _parse_yaml(
     config_paths: List[str],
     modes: List[str],
     verbose: bool = False,
+    disasm_all: bool = False,
 ) -> SplatOpts:
     p = OptParser(yaml)
 
@@ -374,6 +379,10 @@ def _parse_yaml(
         ld_wildcard_sections=p.parse_opt("ld_wildcard_sections", bool, False),
         ld_use_follows=p.parse_opt("ld_use_follows", bool, True),
         segment_end_before_align=p.parse_opt("segment_end_before_align", bool, False),
+        segment_symbols_style=p.parse_opt_within(
+            "segment_symbols_style", str, ["splat", "makerom"], "splat"
+        ),
+        ld_rom_start=p.parse_opt("ld_rom_start", int, 0),
         create_c_files=p.parse_opt("create_c_files", bool, True),
         auto_decompile_empty_functions=p.parse_opt(
             "auto_decompile_empty_functions", bool, True
@@ -443,7 +452,10 @@ def _parse_yaml(
         detect_redundant_function_end=p.parse_opt(
             "detect_redundant_function_end", bool, True
         ),
-        disassemble_all=p.parse_opt("disassemble_all", bool, False),
+        # Command line argument takes precedence over yaml option
+        disassemble_all=disasm_all
+        if disasm_all
+        else p.parse_opt("disassemble_all", bool, False),
     )
     p.check_no_unread_opts()
     return ret
@@ -454,10 +466,11 @@ def initialize(
     config_paths: List[str],
     modes: Optional[List[str]] = None,
     verbose=False,
+    disasm_all=False,
 ):
     global opts
 
     if not modes:
         modes = ["all"]
 
-    opts = _parse_yaml(config["options"], config_paths, modes, verbose)
+    opts = _parse_yaml(config["options"], config_paths, modes, verbose, disasm_all)
diff --git a/tools/splat/util/relocs.py b/tools/splat/util/relocs.py
index f0c60c4..08e668f 100644
--- a/tools/splat/util/relocs.py
+++ b/tools/splat/util/relocs.py
@@ -36,6 +36,7 @@ def initialize():
 
         prog_bar = progress_bar.get_progress_bar(sym_addrs_lines)
         prog_bar.set_description(f"Loading relocs ({path.stem})")
+        line: str
         for line_num, line in enumerate(prog_bar):
             line = line.strip()
             # Allow comments
@@ -107,6 +108,11 @@ def initialize():
             if addend is not None:
                 reloc.addend = addend
 
+            if reloc.rom_address in all_relocs:
+                log.parsing_error_preamble(path, line_num, line)
+                log.error(
+                    f"Duplicated 'rom' address for reloc: 0x{reloc.rom_address:X}"
+                )
             add_reloc(reloc)
 
 
diff --git a/tools/splat/util/symbols.py b/tools/splat/util/symbols.py
index fac5bb8..4f5b682 100644
--- a/tools/splat/util/symbols.py
+++ b/tools/splat/util/symbols.py
@@ -89,6 +89,7 @@ def handle_sym_addrs(
 
     prog_bar = progress_bar.get_progress_bar(sym_addrs_lines)
     prog_bar.set_description(f"Loading symbols ({path.stem})")
+    line: str
     for line_num, line in enumerate(prog_bar):
         line = line.strip()
         if not line == "" and not line.startswith("//"):