diff options
| author | Anghelo Carvajal <angheloalf95@gmail.com> | 2023-03-28 23:42:56 -0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-03-29 03:42:56 +0100 |
| commit | d66a21685f45d130f980e8a5ec7b08cfe189e40b (patch) | |
| tree | f11b3b09dc5772e419fe08d3d0ead4e778def57f /tools | |
| parent | 6a5e64cc4c26398cc607bb02cdc12a056ddf42d1 (diff) | |
Initial repo setup (#1)
* git subrepo clone git@github.com:ethteck/splat.git tools/splat
subrepo:
subdir: "tools/splat"
merged: "4fec014"
upstream:
origin: "git@github.com:ethteck/splat.git"
branch: "master"
commit: "4fec014"
git-subrepo:
version: "0.4.3"
origin: "https://github.com/ingydotnet/git-subrepo.git"
commit: "2f68596"
* gitignore
* decompress_baserom.py
* Initial yaml
* run flib
* more yaml stuff
* Builds an uncompressed ROM
* setup ido download
* git subrepo clone git@github.com:simonlindholm/asm-differ.git tools/asm-differ
subrepo:
subdir: "tools/asm-differ"
merged: "ae40866"
upstream:
origin: "git@github.com:simonlindholm/asm-differ.git"
branch: "main"
commit: "ae40866"
git-subrepo:
version: "0.4.3"
origin: "https://github.com/ingydotnet/git-subrepo.git"
commit: "2f68596"
* git subrepo clone git@github.com:simonlindholm/asm-processor.git tools/asm-processor
subrepo:
subdir: "tools/asm-processor"
merged: "bbd86ea"
upstream:
origin: "git@github.com:simonlindholm/asm-processor.git"
branch: "main"
commit: "bbd86ea"
git-subrepo:
version: "0.4.3"
origin: "https://github.com/ingydotnet/git-subrepo.git"
commit: "2f68596"
* git subrepo clone (merge) git@github.com:EllipticEllipsis/fado.git tools/fado
subrepo:
subdir: "tools/fado"
merged: "8d896ee"
upstream:
origin: "git@github.com:EllipticEllipsis/fado.git"
branch: "master"
commit: "8d896ee"
git-subrepo:
version: "0.4.3"
origin: "https://github.com/ingydotnet/git-subrepo.git"
commit: "2f68596"
* setup ido in makefile
* data and rodata
* add automatic file splits
* bootstrap most symbol_addrs
* git subrepo clone --branch=irix git@github.com:hensldm/ultralib.git lib/ultralib
subrepo:
subdir: "lib/ultralib"
merged: "0d1e095"
upstream:
origin: "git@github.com:hensldm/ultralib.git"
branch: "irix"
commit: "0d1e095"
git-subrepo:
version: "0.4.3"
origin: "https://github.com/ingydotnet/git-subrepo.git"
commit: "2f68596"
* setup ido building
* various tooling
* git subrepo pull tools/asm-differ
subrepo:
subdir: "tools/asm-differ"
merged: "857b398"
upstream:
origin: "git@github.com:simonlindholm/asm-differ.git"
branch: "main"
commit: "857b398"
git-subrepo:
version: "0.4.3"
origin: "https://github.com/ingydotnet/git-subrepo.git"
commit: "2f68596"
* Change defines to match IDO
* review
* newlin
* update gitignore
* Windows error
* D_01000000
* yeet settings.json
* yeet
* git subrepo clone git@github.com:decompals/ultralib.git lib/ultralib
subrepo:
subdir: "lib/ultralib"
merged: "d03b2a3"
upstream:
origin: "git@github.com:decompals/ultralib.git"
branch: "main"
commit: "d03b2a3"
git-subrepo:
version: "0.4.3"
origin: "https://github.com/ingydotnet/git-subrepo.git"
commit: "2f68596"
* clangformat and clangtidy
* Setup a bunch of headers
* bootstrap boot and code functions and variables headers
* global.h
* update format.py includes
* -D_MIPS_SZLONG=32
* format.py: --verbose
* Update tools/decompress_baserom.py
Co-authored-by: Derek Hensley <hensley.derek58@gmail.com>
* Add libc to global.h
* review
* yeet saved_regs
* yeet headers
* yeet _MSC_VER
* more header cleanup
* add readme licensing note
---------
Co-authored-by: Derek Hensley <hensley.derek58@gmail.com>
Diffstat (limited to 'tools')
204 files changed, 21479 insertions, 0 deletions
diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 0000000..9f68842 --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1,2 @@ +ido/ +*.tar.gz diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..2e155fc --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,47 @@ +UNAME_S := $(shell uname -s) +ifeq ($(OS),Windows_NT) + DETECTED_OS := windows + DOWNLOAD_OS := windows-latest +else ifeq ($(UNAME_S),Linux) + DETECTED_OS := linux + DOWNLOAD_OS := ubuntu-20.04 +else ifeq ($(UNAME_S),Darwin) + DETECTED_OS := macos + DOWNLOAD_OS := macos-latest +endif + + +IDO_5_3_DIR := ido/$(DETECTED_OS)/5.3 +IDO_5_3 := $(IDO_5_3_DIR)/cc + +IDO_7_1_DIR := ido/$(DETECTED_OS)/7.1 +IDO_7_1 := $(IDO_7_1_DIR)/cc + +all: $(IDO_5_3) $(IDO_7_1) + +clean: + $(RM) -rf $(IDO_5_3_DIR) $(IDO_7_1_DIR) + +distclean: clean + $(RM) -rf ido + +.PHONY: all clean distclean + + +$(IDO_5_3): | $(IDO_5_3_DIR) + wget https://github.com/decompals/ido-static-recomp/releases/latest/download/ido-5.3-recomp-$(DOWNLOAD_OS).tar.gz + tar xf ido-5.3-recomp-$(DOWNLOAD_OS).tar.gz -C $(IDO_5_3_DIR) + $(RM) ido-5.3-recomp-$(DOWNLOAD_OS).tar.gz + +$(IDO_7_1): | $(IDO_7_1_DIR) + wget https://github.com/decompals/ido-static-recomp/releases/latest/download/ido-7.1-recomp-$(DOWNLOAD_OS).tar.gz + tar xf ido-7.1-recomp-$(DOWNLOAD_OS).tar.gz -C $(IDO_7_1_DIR) + $(RM) ido-7.1-recomp-$(DOWNLOAD_OS).tar.gz + +$(IDO_5_3_DIR): + mkdir -p $@ + +$(IDO_7_1_DIR): + mkdir -p $@ + +distclean: clean diff --git a/tools/asm-differ/.gitignore b/tools/asm-differ/.gitignore new file mode 100644 index 0000000..a2b216f --- /dev/null +++ b/tools/asm-differ/.gitignore @@ -0,0 +1,4 @@ +.mypy_cache/ +__pycache__/ +.vscode/ +poetry.lock diff --git a/tools/asm-differ/.gitrepo b/tools/asm-differ/.gitrepo new file mode 100644 index 0000000..a98d53e --- /dev/null +++ b/tools/asm-differ/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme +; +[subrepo] + remote = git@github.com:simonlindholm/asm-differ.git + branch = main + commit = 857b39804a6f764634483cff7adc168a2f3e5c87 + parent = 6e36168761e33e6c33e6f15c2678733ce61ddffb + method = merge + cmdver = 0.4.3 diff --git a/tools/asm-differ/.pre-commit-config.yaml b/tools/asm-differ/.pre-commit-config.yaml new file mode 100644 index 0000000..67ba03d --- /dev/null +++ b/tools/asm-differ/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black diff --git a/tools/asm-differ/LICENSE b/tools/asm-differ/LICENSE new file mode 100644 index 0000000..cf1ab25 --- /dev/null +++ b/tools/asm-differ/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to <http://unlicense.org> diff --git a/tools/asm-differ/README.md b/tools/asm-differ/README.md new file mode 100644 index 0000000..1d0932c --- /dev/null +++ b/tools/asm-differ/README.md @@ -0,0 +1,56 @@ +# asm-differ + +Nice differ for assembly code. Currently supports MIPS, PPC, AArch64, and ARM32; should be easy to hack to support other instruction sets. + + + +## Dependencies + +- Python >= 3.6 +- `python3 -m pip install --user colorama watchdog levenshtein cxxfilt` (also `dataclasses` if on 3.6) + +## Usage + +Create a file `diff_settings.py` in some directory (see the one in this repo for an example). Then from that directory, run + +```bash +/path/to/diff.py [flags] (function|rom addr) +``` + +Recommended flags are `-mwo` (automatically run `make` on source file changes, and include symbols in diff). See `--help` for more details. + +### Tab completion + +[argcomplete](https://kislyuk.github.io/argcomplete/) can be optionally installed (with `python3 -m pip install argcomplete`) to enable tab completion in a bash shell, completing options and symbol names using the linker map. It also requires a bit more setup: + +If invoking the script **exactly** as `./diff.py`, the following should be added to the `.bashrc` according to argcomplete's instructions: + +```bash +eval "$(register-python-argcomplete ./diff.py)" +``` + +If that doesn't work, run `register-python-argcomplete ./diff.py` in your terminal and copy the output to `.bashrc`. + +If setup correctly (don't forget to restart the shell), `complete | grep ./diff.py` should output: + +```bash +complete -o bashdefault -o default -o nospace -F _python_argcomplete ./diff.py +``` + +Note for developers or for general troubleshooting: run `export _ARC_DEBUG=` to enable debug output during tab-completion, it may show otherwise silenced errors. Use `unset _ARC_DEBUG` or restart the terminal to disable. + +### Contributing + +Contributions are very welcome! Some notes on workflow: + +`black` is used for code formatting. You can either run `black diff.py` manually, or set up a pre-commit hook: +```bash +pip install pre-commit black +pre-commit install +``` + +Type annotations are used for all Python code. `mypy` should pass without any errors. + +PRs that skip the above are still welcome, however. + +The targeted Python version is 3.6. There are currently no tests. diff --git a/tools/asm-differ/diff-stylesheet.css b/tools/asm-differ/diff-stylesheet.css new file mode 100644 index 0000000..79da120 --- /dev/null +++ b/tools/asm-differ/diff-stylesheet.css @@ -0,0 +1,67 @@ +table.diff { + border: none; + font-family: Monospace; + white-space: pre; +} +tr.data-ref { + background-color: gray; +} +.immediate { + color: lightblue; +} +.stack { + color: yellow; +} +.register { + color: yellow; +} +.delay-slot { + font-weight: bold; + color: gray; +} +.diff-change { + color: lightblue; +} +.diff-add { + color: green; +} +.diff-remove { + color: red; +} +.source-filename { + font-weight: bold; +} +.source-function { + font-weight: bold; + text-decoration: underline; +} +.source-other { + font-style: italic; +} +.rotation-0 { + color: magenta; +} +.rotation-1 { + color: cyan; +} +.rotation-2 { + color: green; +} +.rotation-3 { + color: red; +} +.rotation-4 { + color: yellow; +} +.rotation-5 { + color: pink; +} +.rotation-6 { + color: blue; +} +.rotation-7 { + color: lime; +} +.rotation-8 { + color: gray; +} diff --git a/tools/asm-differ/diff.py b/tools/asm-differ/diff.py new file mode 100755 index 0000000..1b12f9a --- /dev/null +++ b/tools/asm-differ/diff.py @@ -0,0 +1,3465 @@ +#!/usr/bin/env python3 +# PYTHON_ARGCOMPLETE_OK +import argparse +import enum +import sys +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Match, + NoReturn, + Optional, + Pattern, + Set, + Tuple, + Type, + Union, +) + + +def fail(msg: str) -> NoReturn: + print(msg, file=sys.stderr) + sys.exit(1) + + +def static_assert_unreachable(x: NoReturn) -> NoReturn: + raise Exception("Unreachable! " + repr(x)) + + +class DiffMode(enum.Enum): + SINGLE = "single" + SINGLE_BASE = "single_base" + NORMAL = "normal" + THREEWAY_PREV = "3prev" + THREEWAY_BASE = "3base" + + +# ==== COMMAND-LINE ==== + +if __name__ == "__main__": + # Prefer to use diff_settings.py from the current working directory + sys.path.insert(0, ".") + try: + import diff_settings + except ModuleNotFoundError: + fail("Unable to find diff_settings.py in the same directory.") + sys.path.pop(0) + + try: + import argcomplete + except ModuleNotFoundError: + argcomplete = None + + parser = argparse.ArgumentParser( + description="Diff MIPS, PPC, AArch64, or ARM32 assembly." + ) + + start_argument = parser.add_argument( + "start", + help="Function name or address to start diffing from.", + ) + + if argcomplete: + + def complete_symbol( + prefix: str, parsed_args: argparse.Namespace, **kwargs: object + ) -> List[str]: + if not prefix or prefix.startswith("-"): + # skip reading the map file, which would + # result in a lot of useless completions + return [] + config: Dict[str, Any] = {} + diff_settings.apply(config, parsed_args) # type: ignore + mapfile = config.get("mapfile") + if not mapfile: + return [] + completes = [] + with open(mapfile) as f: + data = f.read() + # assume symbols are prefixed by a space character + search = f" {prefix}" + pos = data.find(search) + while pos != -1: + # skip the space character in the search string + pos += 1 + # assume symbols are suffixed by either a space + # character or a (unix-style) line return + spacePos = data.find(" ", pos) + lineReturnPos = data.find("\n", pos) + if lineReturnPos == -1: + endPos = spacePos + elif spacePos == -1: + endPos = lineReturnPos + else: + endPos = min(spacePos, lineReturnPos) + if endPos == -1: + match = data[pos:] + pos = -1 + else: + match = data[pos:endPos] + pos = data.find(search, endPos) + completes.append(match) + return completes + + setattr(start_argument, "completer", complete_symbol) + + parser.add_argument( + "end", + nargs="?", + help="Address to end diff at.", + ) + parser.add_argument( + "-o", + dest="diff_obj", + action="store_true", + help="""Diff .o files rather than a whole binary. This makes it possible to + see symbol names. (Recommended)""", + ) + parser.add_argument( + "-f", + "--objfile", + dest="objfile", + type=str, + help="""File path for an object file being diffed. When used + the map file isn't searched for the function given. Useful for dynamically + linked libraries.""", + ) + parser.add_argument( + "-e", + "--elf", + dest="diff_elf_symbol", + metavar="SYMBOL", + help="""Diff a given function in two ELFs, one being stripped and the other + one non-stripped. Requires objdump from binutils 2.33+.""", + ) + parser.add_argument( + "-c", + "--source", + dest="show_source", + action="store_true", + help="Show source code (if possible). Only works with -o or -e.", + ) + parser.add_argument( + "-C", + "--source-old-binutils", + dest="source_old_binutils", + action="store_true", + help="""Tweak --source handling to make it work with binutils < 2.33. + Implies --source.""", + ) + parser.add_argument( + "-j", + "--section", + dest="diff_section", + default=".text", + metavar="SECTION", + help="Diff restricted to a given output section.", + ) + parser.add_argument( + "-L", + "--line-numbers", + dest="show_line_numbers", + action="store_const", + const=True, + help="""Show source line numbers in output, when available. May be enabled by + default depending on diff_settings.py.""", + ) + parser.add_argument( + "--no-line-numbers", + dest="show_line_numbers", + action="store_const", + const=False, + help="Hide source line numbers in output.", + ) + parser.add_argument( + "--inlines", + dest="inlines", + action="store_true", + help="Show inline function calls (if possible). Only works with -o or -e.", + ) + parser.add_argument( + "--base-asm", + dest="base_asm", + metavar="FILE", + help="Read assembly from given file instead of configured base img.", + ) + parser.add_argument( + "--write-asm", + dest="write_asm", + metavar="FILE", + help="Write the current assembly output to file, e.g. for use with --base-asm.", + ) + parser.add_argument( + "-m", + "--make", + dest="make", + action="store_true", + help="Automatically run 'make' on the .o file or binary before diffing.", + ) + parser.add_argument( + "-l", + "--skip-lines", + dest="skip_lines", + metavar="LINES", + type=int, + default=0, + help="Skip the first LINES lines of output.", + ) + parser.add_argument( + "-s", + "--stop-at-ret", + dest="stop_at_ret", + action="store_true", + help="""Stop disassembling at the first return instruction. + Some functions have multiple return points, so use with care!""", + ) + parser.add_argument( + "-i", + "--ignore-large-imms", + dest="ignore_large_imms", + action="store_true", + help="Pretend all large enough immediates are the same.", + ) + parser.add_argument( + "-I", + "--ignore-addr-diffs", + dest="ignore_addr_diffs", + action="store_true", + help="Ignore address differences. Currently only affects AArch64 and ARM32.", + ) + parser.add_argument( + "-B", + "--no-show-branches", + dest="show_branches", + action="store_false", + help="Don't visualize branches/branch targets.", + ) + parser.add_argument( + "-R", + "--no-show-rodata-refs", + dest="show_rodata_refs", + action="store_false", + help="Don't show .rodata -> .text references (typically from jump tables).", + ) + parser.add_argument( + "-S", + "--base-shift", + dest="base_shift", + metavar="N", + type=str, + default="0", + help="""Diff position N in our img against position N + shift in the base img. + Arithmetic is allowed, so e.g. |-S "0x1234 - 0x4321"| is a reasonable + flag to pass if it is known that position 0x1234 in the base img syncs + up with position 0x4321 in our img. Not supported together with -o.""", + ) + parser.add_argument( + "-w", + "--watch", + dest="watch", + action="store_true", + help="""Automatically update when source/object files change. + Recommended in combination with -m.""", + ) + parser.add_argument( + "-0", + "--diff_mode=single_base", + dest="diff_mode", + action="store_const", + const=DiffMode.SINGLE_BASE, + help="""View the base asm only (not a diff).""", + ) + parser.add_argument( + "-1", + "--diff_mode=single", + dest="diff_mode", + action="store_const", + const=DiffMode.SINGLE, + help="""View the current asm only (not a diff).""", + ) + parser.add_argument( + "-3", + "--threeway=prev", + dest="diff_mode", + action="store_const", + const=DiffMode.THREEWAY_PREV, + help="""Show a three-way diff between target asm, current asm, and asm + prior to -w rebuild. Requires -w.""", + ) + parser.add_argument( + "-b", + "--threeway=base", + dest="diff_mode", + action="store_const", + const=DiffMode.THREEWAY_BASE, + help="""Show a three-way diff between target asm, current asm, and asm + when diff.py was started. Requires -w.""", + ) + parser.add_argument( + "--width", + dest="column_width", + metavar="COLS", + type=int, + default=50, + help="Sets the width of the left and right view column.", + ) + parser.add_argument( + "--algorithm", + dest="algorithm", + default="levenshtein", + choices=["levenshtein", "difflib"], + help="""Diff algorithm to use. Levenshtein gives the minimum diff, while difflib + aims for long sections of equal opcodes. Defaults to %(default)s.""", + ) + parser.add_argument( + "--max-size", + "--max-lines", + metavar="LINES", + dest="max_lines", + type=int, + default=1024, + help="The maximum length of the diff, in lines.", + ) + parser.add_argument( + "--no-pager", + dest="no_pager", + action="store_true", + help="""Disable the pager; write output directly to stdout, then exit. + Incompatible with --watch.""", + ) + parser.add_argument( + "--format", + choices=("color", "plain", "html", "json"), + default="color", + help="Output format, default is color. --format=html or json implies --no-pager.", + ) + parser.add_argument( + "-U", + "--compress-matching", + metavar="N", + dest="compress_matching", + type=int, + help="""Compress streaks of matching lines, leaving N lines of context + around non-matching parts.""", + ) + parser.add_argument( + "-V", + "--compress-sameinstr", + metavar="N", + dest="compress_sameinstr", + type=int, + help="""Compress streaks of lines with same instructions (but possibly + different regalloc), leaving N lines of context around other parts.""", + ) + + # Project-specific flags, e.g. different versions/make arguments. + add_custom_arguments_fn = getattr(diff_settings, "add_custom_arguments", None) + if add_custom_arguments_fn: + add_custom_arguments_fn(parser) + + if argcomplete: + argcomplete.autocomplete(parser) + +# ==== IMPORTS ==== + +# (We do imports late to optimize auto-complete performance.) + +import abc +from collections import Counter, defaultdict +from dataclasses import asdict, dataclass, field, replace +import difflib +import html +import itertools +import json +import os +import queue +import re +import string +import struct +import subprocess +import threading +import time +import traceback + + +MISSING_PREREQUISITES = ( + "Missing prerequisite python module {}. " + "Run `python3 -m pip install --user colorama watchdog levenshtein cxxfilt` to install prerequisites (cxxfilt only needed with --source)." +) + +try: + from colorama import Back, Fore, Style + import watchdog +except ModuleNotFoundError as e: + fail(MISSING_PREREQUISITES.format(e.name)) + +# ==== CONFIG ==== + + +@dataclass +class ProjectSettings: + arch_str: str + objdump_executable: str + objdump_flags: List[str] + build_command: List[str] + map_format: str + build_dir: str + ms_map_address_offset: int + baseimg: Optional[str] + myimg: Optional[str] + mapfile: Optional[str] + source_directories: Optional[List[str]] + source_extensions: List[str] + show_line_numbers_default: bool + disassemble_all: bool + reg_categories: Dict[str, int] + expected_dir: str + + +@dataclass +class Compress: + context: int + same_instr: bool + + +@dataclass +class Config: + arch: "ArchSettings" + + # Build/objdump options + diff_obj: bool + objfile: Optional[str] + make: bool + source_old_binutils: bool + diff_section: str + inlines: bool + max_function_size_lines: int + max_function_size_bytes: int + + # Display options + formatter: "Formatter" + diff_mode: DiffMode + base_shift: int + skip_lines: int + compress: Optional[Compress] + show_rodata_refs: bool + show_branches: bool + show_line_numbers: bool + show_source: bool + stop_at_ret: bool + ignore_large_imms: bool + ignore_addr_diffs: bool + algorithm: str + reg_categories: Dict[str, int] + + # Score options + score_stack_differences = True + penalty_stackdiff = 1 + penalty_regalloc = 5 + penalty_reordering = 60 + penalty_insertion = 100 + penalty_deletion = 100 + + +def create_project_settings(settings: Dict[str, Any]) -> ProjectSettings: + return ProjectSettings( + arch_str=settings.get("arch", "mips"), + baseimg=settings.get("baseimg"), + myimg=settings.get("myimg"), + mapfile=settings.get("mapfile"), + build_command=settings.get( + "make_command", ["make", *settings.get("makeflags", [])] + ), + source_directories=settings.get("source_directories"), + source_extensions=settings.get( + "source_extensions", [".c", ".h", ".cpp", ".hpp", ".s"] + ), + objdump_executable=get_objdump_executable(settings.get("objdump_executable")), + objdump_flags=settings.get("objdump_flags", []), + expected_dir=settings.get("expected_dir", "expected/"), + map_format=settings.get("map_format", "gnu"), + ms_map_address_offset=settings.get("ms_map_address_offset", 0), + build_dir=settings.get("build_dir", settings.get("mw_build_dir", "build/")), + show_line_numbers_default=settings.get("show_line_numbers_default", True), + disassemble_all=settings.get("disassemble_all", False), + reg_categories=settings.get("reg_categories", {}), + ) + + +def create_config(args: argparse.Namespace, project: ProjectSettings) -> Config: + arch = get_arch(project.arch_str) + + formatter: Formatter + if args.format == "plain": + formatter = PlainFormatter(column_width=args.column_width) + elif args.format == "color": + formatter = AnsiFormatter(column_width=args.column_width) + elif args.format == "html": + formatter = HtmlFormatter() + elif args.format == "json": + formatter = JsonFormatter(arch_str=arch.name) + else: + raise ValueError(f"Unsupported --format: {args.format}") + + compress = None + if args.compress_matching is not None: + compress = Compress(args.compress_matching, False) + if args.compress_sameinstr is not None: + if compress is not None: + raise ValueError( + "Cannot pass both --compress-matching and --compress-sameinstr" + ) + compress = Compress(args.compress_sameinstr, True) + + show_line_numbers = args.show_line_numbers + if show_line_numbers is None: + show_line_numbers = project.show_line_numbers_default + + return Config( + arch=arch, + # Build/objdump options + diff_obj=args.diff_obj, + objfile=args.objfile, + make=args.make, + source_old_binutils=args.source_old_binutils, + diff_section=args.diff_section, + inlines=args.inlines, + max_function_size_lines=args.max_lines, + max_function_size_bytes=args.max_lines * 4, + # Display options + formatter=formatter, + diff_mode=args.diff_mode or DiffMode.NORMAL, + base_shift=eval_int( + args.base_shift, "Failed to parse --base-shift (-S) argument as an integer." + ), + skip_lines=args.skip_lines, + compress=compress, + show_rodata_refs=args.show_rodata_refs, + show_branches=args.show_branches, + show_line_numbers=show_line_numbers, + show_source=args.show_source or args.source_old_binutils, + stop_at_ret=args.stop_at_ret, + ignore_large_imms=args.ignore_large_imms, + ignore_addr_diffs=args.ignore_addr_diffs, + algorithm=args.algorithm, + reg_categories=project.reg_categories, + ) + + +def get_objdump_executable(objdump_executable: Optional[str]) -> str: + if objdump_executable is not None: + return objdump_executable + + objdump_candidates = [ + "mips-linux-gnu-objdump", + "mips64-elf-objdump", + "mips-elf-objdump", + ] + for objdump_cand in objdump_candidates: + try: + subprocess.check_call( + [objdump_cand, "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return objdump_cand + except subprocess.CalledProcessError: + pass + except FileNotFoundError: + pass + + return fail( + f"Missing binutils; please ensure {' or '.join(objdump_candidates)} exists, or configure objdump_executable." + ) + + +def get_arch(arch_str: str) -> "ArchSettings": + for settings in ARCH_SETTINGS: + if arch_str == settings.name: + return settings + raise ValueError(f"Unknown architecture: {arch_str}") + + +BUFFER_CMD: List[str] = ["tail", "-c", str(10**9)] + +# -S truncates long lines instead of wrapping them +# -R interprets color escape sequences +# -i ignores case when searching +# -c something about how the screen gets redrawn; I don't remember the purpose +# -#6 makes left/right arrow keys scroll by 6 characters +LESS_CMD: List[str] = ["less", "-SRic", "-#6"] + +DEBOUNCE_DELAY: float = 0.1 + +# ==== FORMATTING ==== + + +@enum.unique +class BasicFormat(enum.Enum): + NONE = enum.auto() + IMMEDIATE = enum.auto() + STACK = enum.auto() + REGISTER = enum.auto() + REGISTER_CATEGORY = enum.auto() + DELAY_SLOT = enum.auto() + DIFF_CHANGE = enum.auto() + DIFF_ADD = enum.auto() + DIFF_REMOVE = enum.auto() + SOURCE_FILENAME = enum.auto() + SOURCE_FUNCTION = enum.auto() + SOURCE_LINE_NUM = enum.auto() + SOURCE_OTHER = enum.auto() + + +@dataclass(frozen=True) +class RotationFormat: + group: str + index: int + key: str + + +Format = Union[BasicFormat, RotationFormat] +FormatFunction = Callable[[str], Format] + + +class Text: + segments: List[Tuple[str, Format]] + + def __init__(self, line: str = "", f: Format = BasicFormat.NONE) -> None: + self.segments = [(line, f)] if line else [] + + def reformat(self, f: Format) -> "Text": + return Text(self.plain(), f) + + def plain(self) -> str: + return "".join(s for s, f in self.segments) + + def __repr__(self) -> str: + return f"<Text: {self.plain()!r}>" + + def __bool__(self) -> bool: + return any(s for s, f in self.segments) + + def __str__(self) -> str: + # Use Formatter.apply(...) instead + return NotImplemented + + def __eq__(self, other: object) -> bool: + return NotImplemented + + def __add__(self, other: Union["Text", str]) -> "Text": + if isinstance(other, str): + other = Text(other) + result = Text() + # If two adjacent segments have the same format, merge their lines + if ( + self.segments + and other.segments + and self.segments[-1][1] == other.segments[0][1] + ): + result.segments = ( + self.segments[:-1] + + [(self.segments[-1][0] + other.segments[0][0], self.segments[-1][1])] + + other.segments[1:] + ) + else: + result.segments = self.segments + other.segments + return result + + def __radd__(self, other: Union["Text", str]) -> "Text": + if isinstance(other, str): + other = Text(other) + return other + self + + def finditer(self, pat: Pattern[str]) -> Iterator[Match[str]]: + """Replacement for `pat.finditer(text)` that operates on the inner text, + and returns the exact same matches as `Text.sub(pat, ...)`.""" + for chunk, f in self.segments: + for match in pat.finditer(chunk): + yield match + + def sub(self, pat: Pattern[str], sub_fn: Callable[[Match[str]], "Text"]) -> "Text": + result = Text() + for chunk, f in self.segments: + i = 0 + for match in pat.finditer(chunk): + start, end = match.start(), match.end() + assert i <= start <= end <= len(chunk) + sub = sub_fn(match) + if i != start: + result.segments.append((chunk[i:start], f)) + result.segments.extend(sub.segments) + i = end + if chunk[i:]: + result.segments.append((chunk[i:], f)) + return result + + def ljust(self, column_width: int) -> "Text": + length = sum(len(x) for x, _ in self.segments) + return self + " " * max(column_width - length, 0) + + +@dataclass +class TableLine: + key: Optional[str] + is_data_ref: bool + cells: Tuple[Tuple[Text, Optional["Line"]], ...] + + +@dataclass +class TableData: + headers: Tuple[Text, ...] + current_score: int + max_score: int + previous_score: Optional[int] + lines: List[TableLine] + + +class Formatter(abc.ABC): + @abc.abstractmethod + def apply_format(self, chunk: str, f: Format) -> str: + """Apply the formatting `f` to `chunk` and escape the contents.""" + ... + + @abc.abstractmethod + def table(self, data: TableData) -> str: + """Format a multi-column table with metadata""" + ... + + def apply(self, text: Text) -> str: + return "".join(self.apply_format(chunk, f) for chunk, f in text.segments) + + @staticmethod + def outputline_texts(line: TableLine) -> Tuple[Text, ...]: + return tuple(cell[0] for cell in line.cells) + + +@dataclass +class PlainFormatter(Formatter): + column_width: int + + def apply_format(self, chunk: str, f: Format) -> str: + return chunk + + def table(self, data: TableData) -> str: + rows = [data.headers] + [self.outputline_texts(line) for line in data.lines] + return "\n".join( + "".join(self.apply(x.ljust(self.column_width)) for x in row) for row in rows + ) + + +@dataclass +class AnsiFormatter(Formatter): + # Additional ansi escape codes not in colorama. See: + # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_(Select_Graphic_Rendition)_parameters + STYLE_UNDERLINE = "\x1b[4m" + STYLE_NO_UNDERLINE = "\x1b[24m" + STYLE_INVERT = "\x1b[7m" + STYLE_RESET = "\x1b[0m" + + BASIC_ANSI_CODES = { + BasicFormat.NONE: "", + BasicFormat.IMMEDIATE: Fore.LIGHTBLUE_EX, + BasicFormat.STACK: Fore.YELLOW, + BasicFormat.REGISTER: Fore.YELLOW, + BasicFormat.REGISTER_CATEGORY: Fore.LIGHTYELLOW_EX, + BasicFormat.DIFF_CHANGE: Fore.LIGHTBLUE_EX, + BasicFormat.DIFF_ADD: Fore.GREEN, + BasicFormat.DIFF_REMOVE: Fore.RED, + BasicFormat.SOURCE_FILENAME: Style.DIM + Style.BRIGHT, + BasicFormat.SOURCE_FUNCTION: Style.DIM + Style.BRIGHT + STYLE_UNDERLINE, + BasicFormat.SOURCE_LINE_NUM: Fore.LIGHTBLACK_EX, + BasicFormat.SOURCE_OTHER: Style.DIM, + } + + BASIC_ANSI_CODES_UNDO = { + BasicFormat.NONE: "", + BasicFormat.SOURCE_FILENAME: Style.NORMAL, + BasicFormat.SOURCE_FUNCTION: Style.NORMAL + STYLE_NO_UNDERLINE, + BasicFormat.SOURCE_OTHER: Style.NORMAL, + } + + ROTATION_ANSI_COLORS = [ + Fore.MAGENTA, + Fore.CYAN, + Fore.GREEN, + Fore.RED, + Fore.LIGHTYELLOW_EX, + Fore.LIGHTMAGENTA_EX, + Fore.LIGHTCYAN_EX, + Fore.LIGHTGREEN_EX, + Fore.LIGHTBLACK_EX, + ] + + column_width: int + + def apply_format(self, chunk: str, f: Format) -> str: + if f == BasicFormat.NONE: + return chunk + undo_ansi_code = Fore.RESET + if isinstance(f, BasicFormat): + ansi_code = self.BASIC_ANSI_CODES[f] + undo_ansi_code = self.BASIC_ANSI_CODES_UNDO.get(f, undo_ansi_code) + elif isinstance(f, RotationFormat): + ansi_code = self.ROTATION_ANSI_COLORS[ + f.index % len(self.ROTATION_ANSI_COLORS) + ] + else: + static_assert_unreachable(f) + return f"{ansi_code}{chunk}{undo_ansi_code}" + + def table(self, data: TableData) -> str: + rows = [(data.headers, False)] + [ + ( + self.outputline_texts(line), + line.is_data_ref, + ) + for line in data.lines + ] + return "\n".join( + "".join( + (self.STYLE_INVERT if is_data_ref else "") + + self.apply(x.ljust(self.column_width)) + + (self.STYLE_RESET if is_data_ref else "") + for x in row + ) + for (row, is_data_ref) in rows + ) + + +@dataclass +class HtmlFormatter(Formatter): + rotation_formats: int = 9 + + def apply_format(self, chunk: str, f: Format) -> str: + chunk = html.escape(chunk) + if f == BasicFormat.NONE: + return chunk + if isinstance(f, BasicFormat): + class_name = f.name.lower().replace("_", "-") + data_attr = "" + elif isinstance(f, RotationFormat): + class_name = f"rotation-{f.index % self.rotation_formats}" + rotation_key = html.escape(f"{f.group};{f.key}", quote=True) + data_attr = f'data-rotation="{rotation_key}"' + else: + static_assert_unreachable(f) + return f"<span class='{class_name}' {data_attr}>{chunk}</span>" + + def table(self, data: TableData) -> str: + def table_row(line: Tuple[Text, ...], is_data_ref: bool, cell_el: str) -> str: + tr_attrs = " class='data-ref'" if is_data_ref else "" + output_row = f" <tr{tr_attrs}>" + for cell in line: + cell_html = self.apply(cell) + output_row += f"<{cell_el}>{cell_html}</{cell_el}>" + output_row += "</tr>\n" + return output_row + + output = "<table class='diff'>\n" + output += " <thead>\n" + output += table_row(data.headers, False, "th") + output += " </thead>\n" + output += " <tbody>\n" + output += "".join( + table_row(self.outputline_texts(line), line.is_data_ref, "td") + for line in data.lines + ) + output += " </tbody>\n" + output += "</table>\n" + return output + + +@dataclass +class JsonFormatter(Formatter): + arch_str: str + + def apply_format(self, chunk: str, f: Format) -> str: + # This method is unused by this formatter + return NotImplemented + + def table(self, data: TableData) -> str: + def serialize_format(s: str, f: Format) -> Dict[str, Any]: + if f == BasicFormat.NONE: + return {"text": s} + elif isinstance(f, BasicFormat): + return {"text": s, "format": f.name.lower()} + elif isinstance(f, RotationFormat): + attrs = asdict(f) + attrs.update({"text": s, "format": "rotation"}) + return attrs + else: + static_assert_unreachable(f) + + def serialize(text: Optional[Text]) -> List[Dict[str, Any]]: + if text is None: + return [] + return [serialize_format(s, f) for s, f in text.segments] + + output: Dict[str, Any] = {} + output["arch_str"] = self.arch_str + output["header"] = { + name: serialize(h) + for h, name in zip(data.headers, ("base", "current", "previous")) + } + output["current_score"] = data.current_score + output["max_score"] = data.max_score + if data.previous_score is not None: + output["previous_score"] = data.previous_score + output_rows: List[Dict[str, Any]] = [] + for row in data.lines: + output_row: Dict[str, Any] = {} + output_row["key"] = row.key + output_row["is_data_ref"] = row.is_data_ref + iters: List[Tuple[str, Text, Optional[Line]]] = [ + (label, *cell) + for label, cell in zip(("base", "current", "previous"), row.cells) + ] + if all(line is None for _, _, line in iters): + # Skip rows that were only for displaying source code + continue + for column_name, text, line in iters: + column: Dict[str, Any] = {} + column["text"] = serialize(text) + if line: + if line.line_num is not None: + column["line"] = line.line_num + if line.branch_target is not None: + column["branch"] = line.branch_target + if line.source_lines: + column["src"] = line.source_lines + if line.comment is not None: + column["src_comment"] = line.comment + if line.source_line_num is not None: + column["src_line"] = line.source_line_num + if line or column["text"]: + output_row[column_name] = column + output_rows.append(output_row) + output["rows"] = output_rows + return json.dumps(output) + + +def format_fields( + pat: Pattern[str], + out1: Text, + out2: Text, + color1: FormatFunction, + color2: Optional[FormatFunction] = None, +) -> Tuple[Text, Text]: + diffs = [ + of.group() != nf.group() + for (of, nf) in zip(out1.finditer(pat), out2.finditer(pat)) + ] + + it = iter(diffs) + + def maybe_color(color: FormatFunction, s: str) -> Text: + return Text(s, color(s)) if next(it, False) else Text(s) + + out1 = out1.sub(pat, lambda m: maybe_color(color1, m.group())) + it = iter(diffs) + out2 = out2.sub(pat, lambda m: maybe_color(color2 or color1, m.group())) + + return out1, out2 + + +def symbol_formatter(group: str, base_index: int) -> FormatFunction: + symbol_formats: Dict[str, Format] = {} + + def symbol_format(s: str) -> Format: + # TODO: it would be nice to use a unique Format for each symbol, so we could + # add extra UI elements in the HTML version + f = symbol_formats.get(s) + if f is None: + index = len(symbol_formats) + base_index + f = RotationFormat(key=s, index=index, group=group) + symbol_formats[s] = f + return f + + return symbol_format + + +# ==== LOGIC ==== + +ObjdumpCommand = Tuple[List[str], str, Optional[str]] + +# eval_expr adapted from https://stackoverflow.com/a/9558001 + +import ast +import operator as op + +# supported operators +operators: Dict[Type[Union[ast.operator, ast.unaryop]], Any] = { + ast.Add: op.add, + ast.Sub: op.sub, + ast.Mult: op.mul, + ast.Div: op.truediv, + ast.Pow: op.pow, + ast.BitXor: op.xor, + ast.USub: op.neg, +} + + +def eval_expr(expr: str) -> Any: + return eval_(ast.parse(expr, mode="eval").body) + + +def eval_(node: ast.AST) -> Any: + if isinstance(node, ast.Num): # <number> + return node.n + elif isinstance(node, ast.BinOp): # <left> <operator> <right> + return operators[type(node.op)](eval_(node.left), eval_(node.right)) + elif isinstance(node, ast.UnaryOp): # <operator> <operand> e.g., -1 + return operators[type(node.op)](eval_(node.operand)) + else: + raise TypeError(node) + + +def maybe_eval_int(expr: str) -> Optional[int]: + try: + ret = eval_expr(expr) + if not isinstance(ret, int): + raise Exception("not an integer") + return ret + except Exception: + return None + + +def eval_int(expr: str, emsg: str) -> int: + ret = maybe_eval_int(expr) + if ret is None: + fail(emsg) + return ret + + +def eval_line_num(expr: str) -> Optional[int]: + expr = expr.strip().replace(":", "") + if expr == "": + return None + return int(expr, 16) + + +def run_make(target: str, project: ProjectSettings) -> None: + subprocess.check_call(project.build_command + [target]) + + +def run_make_capture_output( + target: str, project: ProjectSettings +) -> "subprocess.CompletedProcess[bytes]": + return subprocess.run( + project.build_command + [target], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +def restrict_to_function(dump: str, fn_name: str) -> str: + try: + ind = dump.index("\n", dump.index(f"<{fn_name}>:")) + return dump[ind + 1 :] + except ValueError: + return "" + + +def serialize_rodata_references(references: List[Tuple[int, int, str]]) -> str: + return "".join( + f"DATAREF {text_offset} {from_offset} {from_section}\n" + for (text_offset, from_offset, from_section) in references + ) + + +def maybe_get_objdump_source_flags(config: Config) -> List[str]: + flags = [] + + if config.show_line_numbers or config.show_source: + flags.append("--line-numbers") + + if config.show_source: + flags.append("--source") + + if not config.source_old_binutils: + flags.append("--source-comment=│ ") + + if config.inlines: + flags.append("--inlines") + + return flags + + +def run_objdump(cmd: ObjdumpCommand, config: Config, project: ProjectSettings) -> str: + flags, target, restrict = cmd + try: + out = subprocess.run( + [project.objdump_executable] + + config.arch.arch_flags + + project.objdump_flags + + flags + + [target], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ).stdout + except subprocess.CalledProcessError as e: + print(e.stdout) + print(e.stderr) + if "unrecognized option '--source-comment" in e.stderr: + fail("** Try using --source-old-binutils instead of --source **") + raise e + + obj_data: Optional[bytes] = None + if config.diff_obj: + with open(target, "rb") as f: + obj_data = f.read() + + return preprocess_objdump_out(restrict, obj_data, out, config) + + +def preprocess_objdump_out( + restrict: Optional[str], obj_data: Optional[bytes], objdump_out: str, config: Config +) -> str: + """ + Preprocess the output of objdump into a format that `process()` expects. + This format is suitable for saving to disk with `--write-asm`. + + - Optionally filter the output to a single function (`restrict`) + - Otherwise, strip objdump header (7 lines) + - Prepend .data references ("DATAREF" lines) when working with object files + """ + out = objdump_out + + if restrict is not None: + out = restrict_to_function(out, restrict) + else: + for i in range(7): + out = out[out.find("\n") + 1 :] + out = out.rstrip("\n") + + if obj_data and config.show_rodata_refs: + out = ( + serialize_rodata_references(parse_elf_rodata_references(obj_data, config)) + + out + ) + + return out + + +def search_build_objects(objname: str, project: ProjectSettings) -> Optional[str]: + objfiles = [ + os.path.join(dirpath, f) + for dirpath, _, filenames in os.walk(project.build_dir) + for f in filenames + if f == objname + ] + if len(objfiles) > 1: + all_objects = "\n".join(objfiles) + fail( + f"Found multiple objects of the same name {objname} in {project.build_dir}, " + f"cannot determine which to diff against: \n{all_objects}" + ) + if len(objfiles) == 1: + return objfiles[0] + + return None + + +def search_map_file( + fn_name: str, project: ProjectSettings, config: Config, *, for_binary: bool +) -> Tuple[Optional[str], Optional[int]]: + if not project.mapfile: + fail(f"No map file configured; cannot find function {fn_name}.") + + try: + with open(project.mapfile) as f: + contents = f.read() + except Exception: + fail(f"Failed to open map file {project.mapfile} for reading.") + + if project.map_format == "gnu": + if for_binary and "load address" not in contents: + fail( + 'Failed to find "load address" in map file. Maybe you need to add\n' + '"export LANG := C" to your Makefile to avoid localized output?' + ) + + lines = contents.split("\n") + + try: + cur_objfile = None + ram_to_rom = None + cands = [] + last_line = "" + for line in lines: + if line.startswith(" " + config.diff_section): + cur_objfile = line.split()[3] + if "load address" in line: + tokens = last_line.split() + line.split() + ram = int(tokens[1], 0) + rom = int(tokens[5], 0) + ram_to_rom = rom - ram + if line.endswith(" " + fn_name) or f" {fn_name} = 0x" in line: + ram = int(line.split()[0], 0) + if (for_binary and ram_to_rom is not None) or ( + not for_binary and cur_objfile is not None + ): + cands.append((cur_objfile, ram + (ram_to_rom or 0))) + last_line = line + except Exception as e: + traceback.print_exc() + fail(f"Internal error while parsing map file") + + if len(cands) > 1: + fail(f"Found multiple occurrences of function {fn_name} in map file.") + if len(cands) == 1: + return cands[0] + elif project.map_format == "mw": + find = re.findall( + # ram elf rom alignment + r" \S+ \S+ (\S+) (\S+) +\S+ " + + re.escape(fn_name) + + r"(?: \(entry of " + + re.escape(config.diff_section) + + r"\))? \t" + # object name + + "(\S+)", + contents, + ) + if len(find) > 1: + fail(f"Found multiple occurrences of function {fn_name} in map file.") + if len(find) == 1: + rom = int(find[0][1], 16) + objname = find[0][2] + objfile = search_build_objects(objname, project) + + # TODO Currently the ram-rom conversion only works for diffing ELF + # executables, but it would likely be more convenient to diff DOLs. + # At this time it is recommended to always use -o when running the diff + # script as this mode does not make use of the ram-rom conversion. + if objfile is not None: + return objfile, rom + elif project.map_format == "ms": + load_address_find = re.search( + r"Preferred load address is ([0-9a-f]+)", + contents, + ) + if not load_address_find: + fail(f"Couldn't find module load address in map file.") + load_address = int(load_address_find.group(1), 16) + + diff_segment_find = re.search( + r"([0-9a-f]+):[0-9a-f]+ [0-9a-f]+H " + re.escape(config.diff_section), + contents, + ) + if not diff_segment_find: + fail(f"Couldn't find segment for section in map file.") + diff_segment = diff_segment_find.group(1) + + find = re.findall( + r" (?:" + + re.escape(diff_segment) + + r")\S+\s+(?:" + + re.escape(fn_name) + + r")\s+\S+ ... \S+", + contents, + ) + if len(find) > 1: + fail(f"Found multiple occurrences of function {fn_name} in map file.") + if len(find) == 1: + names_find = re.search(r"(\S+) ... (\S+)", find[0]) + assert names_find is not None + fileofs = ( + int(names_find.group(1), 16) + - load_address + + project.ms_map_address_offset + ) + if for_binary: + return None, fileofs + + objname = names_find.group(2) + objfile = search_build_objects(objname, project) + if objfile is not None: + return objfile, fileofs + else: + fail(f"Linker map format {project.map_format} unrecognised.") + return None, None + + +def parse_elf_rodata_references( + data: bytes, config: Config +) -> List[Tuple[int, int, str]]: + e_ident = data[:16] + if e_ident[:4] != b"\x7FELF": + return [] + + SHT_SYMTAB = 2 + SHT_REL = 9 + SHT_RELA = 4 + R_MIPS_32 = 2 + R_MIPS_GPREL32 = 12 + + is_32bit = e_ident[4] == 1 + is_little_endian = e_ident[5] == 1 + str_end = "<" if is_little_endian else ">" + str_off = "I" if is_32bit else "Q" + + def read(spec: str, offset: int) -> Tuple[int, ...]: + spec = spec.replace("P", str_off) + size = struct.calcsize(spec) + return struct.unpack(str_end + spec, data[offset : offset + size]) + + ( + e_type, + e_machine, + e_version, + e_entry, + e_phoff, + e_shoff, + e_flags, + e_ehsize, + e_phentsize, + e_phnum, + e_shentsize, + e_shnum, + e_shstrndx, + ) = read("HHIPPPIHHHHHH", 16) + if e_type != 1: # relocatable + return [] + assert e_shoff != 0 + assert e_shnum != 0 # don't support > 0xFF00 sections + assert e_shstrndx != 0 + + @dataclass + class Section: + sh_name: int + sh_type: int + sh_flags: int + sh_addr: int + sh_offset: int + sh_size: int + sh_link: int + sh_info: int + sh_addralign: int + sh_entsize: int + + sections = [ + Section(*read("IIPPPPIIPP", e_shoff + i * e_shentsize)) for i in range(e_shnum) + ] + shstr = sections[e_shstrndx] + sec_name_offs = [shstr.sh_offset + s.sh_name for s in sections] + sec_names = [data[offset : data.index(b"\0", offset)] for offset in sec_name_offs] + + symtab_sections = [i for i in range(e_shnum) if sections[i].sh_type == SHT_SYMTAB] + assert len(symtab_sections) == 1 + symtab = sections[symtab_sections[0]] + + section_name = config.diff_section.encode("utf-8") + text_sections = [ + i + for i in range(e_shnum) + if sec_names[i] == section_name and sections[i].sh_size != 0 + ] + if len(text_sections) != 1: + return [] + text_section = text_sections[0] + + ret: List[Tuple[int, int, str]] = [] + for s in sections: + if s.sh_type == SHT_REL or s.sh_type == SHT_RELA: + if s.sh_info == text_section: + # Skip section_name -> section_name references + continue + sec_name = sec_names[s.sh_info].decode("latin1") + if sec_name not in (".rodata", ".late_rodata"): + continue + sec_base = sections[s.sh_info].sh_offset + for i in range(0, s.sh_size, s.sh_entsize): + if s.sh_type == SHT_REL: + r_offset, r_info = read("PP", s.sh_offset + i) + else: + r_offset, r_info, r_addend = read("PPP", s.sh_offset + i) + + if is_32bit: + r_sym = r_info >> 8 + r_type = r_info & 0xFF + sym_offset = symtab.sh_offset + symtab.sh_entsize * r_sym + st_name, st_value, st_size, st_info, st_other, st_shndx = read( + "IIIBBH", sym_offset + ) + else: + r_sym = r_info >> 32 + r_type = r_info & 0xFFFFFFFF + sym_offset = symtab.sh_offset + symtab.sh_entsize * r_sym + st_name, st_info, st_other, st_shndx, st_value, st_size = read( + "IBBHQQ", sym_offset + ) + if st_shndx == text_section: + if s.sh_type == SHT_REL: + if e_machine == 8 and r_type in (R_MIPS_32, R_MIPS_GPREL32): + (r_addend,) = read("I", sec_base + r_offset) + else: + continue + text_offset = (st_value + r_addend) & 0xFFFFFFFF + ret.append((text_offset, r_offset, sec_name)) + return ret + + +def dump_elf( + start: str, + end: Optional[str], + diff_elf_symbol: str, + config: Config, + project: ProjectSettings, +) -> Tuple[str, ObjdumpCommand, ObjdumpCommand]: + if not project.baseimg or not project.myimg: + fail("Missing myimg/baseimg in config.") + if config.base_shift: + fail("--base-shift not compatible with -e") + + start_addr = eval_int(start, "Start address must be an integer expression.") + + if end is not None: + end_addr = eval_int(end, "End address must be an integer expression.") + else: + end_addr = start_addr + config.max_function_size_bytes + + flags1 = [ + f"--start-address={start_addr}", + f"--stop-address={end_addr}", + ] + + if project.disassemble_all: + disassemble_flag = "-D" + else: + disassemble_flag = "-d" + + flags2 = [ + f"--disassemble={diff_elf_symbol}", + ] + + objdump_flags = [disassemble_flag, "-rz", "-j", config.diff_section] + return ( + project.myimg, + (objdump_flags + flags1, project.baseimg, None), + ( + objdump_flags + flags2 + maybe_get_objdump_source_flags(config), + project.myimg, + None, + ), + ) + + +def dump_objfile( + start: str, end: Optional[str], config: Config, project: ProjectSettings +) -> Tuple[str, ObjdumpCommand, ObjdumpCommand]: + if config.base_shift: + fail("--base-shift not compatible with -o") + if end is not None: + fail("end address not supported together with -o") + if start.startswith("0"): + fail("numerical start address not supported with -o; pass a function name") + + objfile = config.objfile + if not objfile: + objfile, _ = search_map_file(start, project, config, for_binary=False) + + if not objfile: + fail("Not able to find .o file for function.") + + if config.make: + run_make(objfile, project) + + if not os.path.isfile(objfile): + fail(f"Not able to find .o file for function: {objfile} is not a file.") + + refobjfile = os.path.join(project.expected_dir, objfile) + if config.diff_mode != DiffMode.SINGLE and not os.path.isfile(refobjfile): + fail(f'Please ensure an OK .o file exists at "{refobjfile}".') + + if project.disassemble_all: + disassemble_flag = "-D" + else: + disassemble_flag = "-d" + + objdump_flags = [disassemble_flag, "-rz", "-j", config.diff_section] + return ( + objfile, + (objdump_flags, refobjfile, start), + (objdump_flags + maybe_get_objdump_source_flags(config), objfile, start), + ) + + +def dump_binary( + start: str, end: Optional[str], config: Config, project: ProjectSettings +) -> Tuple[str, ObjdumpCommand, ObjdumpCommand]: + if not project.baseimg or not project.myimg: + fail("Missing myimg/baseimg in config.") + if config.make: + run_make(project.myimg, project) + start_addr = maybe_eval_int(start) + if start_addr is None: + _, start_addr = search_map_file(start, project, config, for_binary=True) + if start_addr is None: + fail("Not able to find function in map file.") + if end is not None: + end_addr = eval_int(end, "End address must be an integer expression.") + else: + end_addr = start_addr + config.max_function_size_bytes + objdump_flags = ["-Dz", "-bbinary"] + ["-EB" if config.arch.big_endian else "-EL"] + flags1 = [ + f"--start-address={start_addr + config.base_shift}", + f"--stop-address={end_addr + config.base_shift}", + ] + flags2 = [f"--start-address={start_addr}", f"--stop-address={end_addr}"] + return ( + project.myimg, + (objdump_flags + flags1, project.baseimg, None), + (objdump_flags + flags2, project.myimg, None), + ) + + +# Example: "ldr r4, [pc, #56] ; (4c <AddCoins+0x4c>)" +ARM32_LOAD_POOL_PATTERN = r"(ldr\s+r([0-9]|1[0-3]),\s+\[pc,.*;\s*)(\([a-fA-F0-9]+.*\))" + + +# The base class is a no-op. +class AsmProcessor: + def __init__(self, config: Config) -> None: + self.config = config + + def pre_process( + self, mnemonic: str, args: str, next_row: Optional[str] + ) -> Tuple[str, str]: + return mnemonic, args + + def process_reloc(self, row: str, prev: str) -> Tuple[str, Optional[str]]: + return prev, None + + def normalize(self, mnemonic: str, row: str) -> str: + """This should be called exactly once for each line.""" + arch = self.config.arch + row = self._normalize_arch_specific(mnemonic, row) + if self.config.ignore_large_imms and mnemonic not in arch.branch_instructions: + row = re.sub(self.config.arch.re_large_imm, "<imm>", row) + return row + + def _normalize_arch_specific(self, mnemonic: str, row: str) -> str: + return row + + def post_process(self, lines: List["Line"]) -> None: + return + + def is_end_of_function(self, mnemonic: str, args: str) -> bool: + return False + + +class AsmProcessorMIPS(AsmProcessor): + def __init__(self, config: Config) -> None: + super().__init__(config) + self.seen_jr_ra = False + + def process_reloc(self, row: str, prev: str) -> Tuple[str, Optional[str]]: + arch = self.config.arch + if "R_MIPS_NONE" in row or "R_MIPS_JALR" in row: + # GNU as emits no-op relocations immediately after real ones when + # assembling with -mabi=64. Return without trying to parse 'imm' as an + # integer. + return prev, None + before, imm, after = parse_relocated_line(prev) + repl = row.split()[-1] + reloc_addend_from_imm(imm, before, self.config.arch) + if "R_MIPS_LO16" in row: + repl = f"%lo({repl})" + elif "R_MIPS_HI16" in row: + # Ideally we'd pair up R_MIPS_LO16 and R_MIPS_HI16 to generate a + # correct addend for each, but objdump doesn't give us the order of + # the relocations, so we can't find the right LO16. :( + repl = f"%hi({repl})" + elif "R_MIPS_26" in row: + # Function calls + pass + elif "R_MIPS_PC16" in row: + # Branch to glabel. This gives confusing output, but there's not much + # we can do here. + pass + elif "R_MIPS_GPREL16" in row: + repl = f"%gp_rel({repl})" + elif "R_MIPS_GOT16" in row: + repl = f"%got({repl})" + elif "R_MIPS_CALL16" in row: + repl = f"%call16({repl})" + else: + assert False, f"unknown relocation type '{row}' for line '{prev}'" + return before + repl + after, repl + + def is_end_of_function(self, mnemonic: str, args: str) -> bool: + if self.seen_jr_ra: + return True + if mnemonic == "jr" and args == "ra": + self.seen_jr_ra = True + return False + + +class AsmProcessorPPC(AsmProcessor): + def pre_process( + self, mnemonic: str, args: str, next_row: Optional[str] + ) -> Tuple[str, str]: + + if next_row and "R_PPC_EMB_SDA21" in next_row: + # With sda21 relocs, the linker transforms `r0` into `r2`/`r13`, and + # we may encounter this in either pre-transformed or post-transformed + # versions depending on if the .o file comes from compiler output or + # from disassembly. Normalize, to make sure both forms are treated as + # equivalent. + + args = args.replace("(r2)", "(0)") + args = args.replace("(r13)", "(0)") + args = args.replace(",r2,", ",0,") + args = args.replace(",r13,", ",0,") + + # We want to convert li and lis with an sda21 reloc, + # because the r0 to r2/r13 transformation results in + # turning an li/lis into an addi/addis with r2/r13 arg + # our preprocessing normalizes all versions to addi with a 0 arg + if mnemonic in {"li", "lis"}: + mnemonic = mnemonic.replace("li", "addi") + args_parts = args.split(",") + args = args_parts[0] + ",0," + args_parts[1] + + return mnemonic, args + + def process_reloc(self, row: str, prev: str) -> Tuple[str, Optional[str]]: + arch = self.config.arch + assert any( + r in row + for r in ["R_PPC_REL24", "R_PPC_ADDR16", "R_PPC_EMB_SDA21", "R_PPC_REL14"] + ), f"unknown relocation type '{row}' for line '{prev}'" + before, imm, after = parse_relocated_line(prev) + repl = row.split()[-1] + if "R_PPC_REL24" in row: + # function calls + pass + if "R_PPC_REL14" in row: + pass + elif "R_PPC_ADDR16_HI" in row: + # absolute hi of addr + repl = f"{repl}@h" + elif "R_PPC_ADDR16_HA" in row: + # adjusted hi of addr + repl = f"{repl}@ha" + elif "R_PPC_ADDR16_LO" in row: + # lo of addr + repl = f"{repl}@l" + elif "R_PPC_ADDR16" in row: + # 16-bit absolute addr + if "+0x7" in repl: + # remove the very large addends as they are an artifact of (label-_SDA(2)_BASE_) + # computations and are unimportant in a diff setting. + if int(repl.split("+")[1], 16) > 0x70000000: + repl = repl.split("+")[0] + elif "R_PPC_EMB_SDA21" in row: + # sda21 relocations; r2/r13 --> 0 swaps are performed in pre_process + repl = f"{repl}@sda21" + + return before + repl + after, repl + + def is_end_of_function(self, mnemonic: str, args: str) -> bool: + return mnemonic == "blr" + + +class AsmProcessorARM32(AsmProcessor): + def process_reloc(self, row: str, prev: str) -> Tuple[str, Optional[str]]: + arch = self.config.arch + if "R_ARM_V4BX" in row: + # R_ARM_V4BX converts "bx <reg>" to "mov pc,<reg>" for some targets. + # Ignore for now. + return prev, None + if "R_ARM_ABS32" in row and not prev.startswith(".word"): + # Don't crash on R_ARM_ABS32 relocations incorrectly applied to code. + # (We may want to do something more fancy here that actually shows the + # related symbol, but this serves as a stop-gap.) + return prev, None + before, imm, after = parse_relocated_line(prev) + repl = row.split()[-1] + reloc_addend_from_imm(imm, before, self.config.arch) + return before + repl + after, repl + + def _normalize_arch_specific(self, mnemonic: str, row: str) -> str: + if self.config.ignore_addr_diffs: + row = self._normalize_bl(mnemonic, row) + row = self._normalize_data_pool(row) + return row + + def _normalize_bl(self, mnemonic: str, row: str) -> str: + if mnemonic != "bl": + return row + + row, _ = split_off_address(row) + return row + "<ignore>" + + def _normalize_data_pool(self, row: str) -> str: + pool_match = re.search(ARM32_LOAD_POOL_PATTERN, row) + return pool_match.group(1) if pool_match else row + + def post_process(self, lines: List["Line"]) -> None: + lines_by_line_number = {} + for line in lines: + lines_by_line_number[line.line_num] = line + for line in lines: + if line.data_pool_addr is None: + continue + + # Add data symbol and its address to the line. + line_original = lines_by_line_number[line.data_pool_addr].original + value = line_original.split()[1] + addr = "{:x}".format(line.data_pool_addr) + line.original = line.normalized_original + f"={value} ({addr})" + + +class AsmProcessorAArch64(AsmProcessor): + def __init__(self, config: Config) -> None: + super().__init__(config) + self._adrp_pair_registers: Set[str] = set() + + def _normalize_arch_specific(self, mnemonic: str, row: str) -> str: + if self.config.ignore_addr_diffs: + row = self._normalize_adrp_differences(mnemonic, row) + row = self._normalize_bl(mnemonic, row) + return row + + def _normalize_bl(self, mnemonic: str, row: str) -> str: + if mnemonic != "bl": + return row + + row, _ = split_off_address(row) + return row + "<ignore>" + + def _normalize_adrp_differences(self, mnemonic: str, row: str) -> str: + """Identifies ADRP + LDR/ADD pairs that are used to access the GOT and + suppresses any immediate differences. + + Whenever an ADRP is seen, the destination register is added to the set of registers + that are part of an ADRP + LDR/ADD pair. Registers are removed from the set as soon + as they are used for an LDR or ADD instruction which completes the pair. + + This method is somewhat crude but should manage to detect most such pairs. + """ + row_parts = row.split("\t", 1) + if mnemonic == "adrp": + self._adrp_pair_registers.add(row_parts[1].strip().split(",")[0]) + row, _ = split_off_address(row) + return row + "<ignore>" + elif mnemonic == "ldr": + for reg in self._adrp_pair_registers: + # ldr xxx, [reg] + # ldr xxx, [reg, <imm>] + if f", [{reg}" in row_parts[1]: + self._adrp_pair_registers.remove(reg) + return normalize_imms(row, AARCH64_SETTINGS) + elif mnemonic == "add": + for reg in self._adrp_pair_registers: + # add reg, reg, <imm> + if row_parts[1].startswith(f"{reg}, {reg}, "): + self._adrp_pair_registers.remove(reg) + return normalize_imms(row, AARCH64_SETTINGS) + + return row + + +class AsmProcessorI686(AsmProcessor): + def process_reloc(self, row: str, prev: str) -> Tuple[str, Optional[str]]: + repl = row.split()[-1] + mnemonic, args = prev.split(maxsplit=1) + + addr_imm = re.search(r"(?<!\$)0x[0-9a-f]+", args) + if not addr_imm: + assert False, f"failed to find address immediate for line '{prev}'" + start, end = addr_imm.span() + + if "R_386_NONE" in row: + pass + elif "R_386_32" in row: + pass + elif "R_386_PC32" in row: + pass + elif "R_386_16" in row: + pass + elif "R_386_PC16" in row: + pass + elif "R_386_8" in row: + pass + elif "R_386_PC8" in row: + pass + elif "R_386_GOT32" in row: + repl = f"%got({repl})" + elif "R_386_PLT32" in row: + repl = f"%plt({repl})" + elif "R_386_RELATIVE" in row: + repl = f"%rel({repl})" + elif "R_386_GOTOFF" in row: + repl = f"%got({repl})" + elif "R_386_GOTPC" in row: + repl = f"%got({repl})" + elif "R_386_32PLT" in row: + repl = f"%plt({repl})" + else: + assert False, f"unknown relocation type '{row}' for line '{prev}'" + + return f"{mnemonic}\t{args[:start]+repl+args[end:]}", repl + + def is_end_of_function(self, mnemonic: str, args: str) -> bool: + return mnemonic == "ret" + + +@dataclass +class ArchSettings: + name: str + re_int: Pattern[str] + re_comment: Pattern[str] + re_reg: Pattern[str] + re_sprel: Pattern[str] + re_large_imm: Pattern[str] + re_imm: Pattern[str] + re_reloc: Pattern[str] + branch_instructions: Set[str] + instructions_with_address_immediates: Set[str] + forbidden: Set[str] = field(default_factory=lambda: set(string.ascii_letters + "_")) + arch_flags: List[str] = field(default_factory=list) + branch_likely_instructions: Set[str] = field(default_factory=set) + proc: Type[AsmProcessor] = AsmProcessor + big_endian: Optional[bool] = True + delay_slot_instructions: Set[str] = field(default_factory=set) + + +MIPS_BRANCH_LIKELY_INSTRUCTIONS = { + "beql", + "bnel", + "beqzl", + "bnezl", + "bgezl", + "bgtzl", + "blezl", + "bltzl", + "bc1tl", + "bc1fl", +} +MIPS_BRANCH_INSTRUCTIONS = MIPS_BRANCH_LIKELY_INSTRUCTIONS.union( + { + "b", + "beq", + "bne", + "beqz", + "bnez", + "bgez", + "bgtz", + "blez", + "bltz", + "bc1t", + "bc1f", + } +) + +ARM32_PREFIXES = {"b", "bl"} +ARM32_CONDS = { + "", + "eq", + "ne", + "cs", + "cc", + "mi", + "pl", + "vs", + "vc", + "hi", + "ls", + "ge", + "lt", + "gt", + "le", + "al", +} +ARM32_SUFFIXES = {"", ".n", ".w"} +ARM32_BRANCH_INSTRUCTIONS = { + f"{prefix}{cond}{suffix}" + for prefix in ARM32_PREFIXES + for cond in ARM32_CONDS + for suffix in ARM32_SUFFIXES +} + +AARCH64_BRANCH_INSTRUCTIONS = { + "b", + "b.eq", + "b.ne", + "b.cs", + "b.hs", + "b.cc", + "b.lo", + "b.mi", + "b.pl", + "b.vs", + "b.vc", + "b.hi", + "b.ls", + "b.ge", + "b.lt", + "b.gt", + "b.le", + "cbz", + "cbnz", + "tbz", + "tbnz", +} + +PPC_BRANCH_INSTRUCTIONS = { + "b", + "beq", + "beq+", + "beq-", + "bne", + "bne+", + "bne-", + "blt", + "blt+", + "blt-", + "ble", + "ble+", + "ble-", + "bdnz", + "bdnz+", + "bdnz-", + "bge", + "bge+", + "bge-", + "bgt", + "bgt+", + "bgt-", +} + +I686_BRANCH_INSTRUCTIONS = { + "call", + "jmp", + "ljmp", + "ja", + "jae", + "jb", + "jbe", + "jc", + "jcxz", + "jecxz", + "jrcxz", + "je", + "jg", + "jge", + "jl", + "jle", + "jna", + "jnae", + "jnb", + "jnbe", + "jnc", + "jne", + "jng", + "jnge", + "jnl", + "jnle", + "jno", + "jnp", + "jns", + "jnz", + "jo", + "jp", + "jpe", + "jpo", + "js", + "jz", + "ja", + "jae", + "jb", + "jbe", + "jc", + "je", + "jz", + "jg", + "jge", + "jl", + "jle", + "jna", + "jnae", + "jnb", + "jnbe", + "jnc", + "jne", + "jng", + "jnge", + "jnl", + "jnle", + "jno", + "jnp", + "jns", + "jnz", + "jo", + "jp", + "jpe", + "jpo", + "js", + "jz", +} + +MIPS_SETTINGS = ArchSettings( + name="mips", + re_int=re.compile(r"[0-9]+"), + re_comment=re.compile(r"<.*>"), + # Includes: + # - General purpose registers v0..1, a0..7, t0..9, s0..8, zero, at, fp, k0..1/kt0..1 + # - Float registers f0..31, or fv0..1, fa0..7, ft0..15, fs0..8 plus odd complements + # (actually used number depends on ABI) + # sp, gp should not be in this list + re_reg=re.compile(r"\$?\b([astv][0-9]|at|f[astv]?[0-9]+f?|kt?[01]|fp|ra|zero)\b"), + re_sprel=re.compile(r"(?<=,)([0-9]+|0x[0-9a-f]+)\(sp\)"), + re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"), + re_imm=re.compile( + r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(sp)|%(lo|hi|got|gp_rel|call16)\([^)]*\)" + ), + re_reloc=re.compile(r"R_MIPS_"), + arch_flags=["-m", "mips:4300"], + branch_likely_instructions=MIPS_BRANCH_LIKELY_INSTRUCTIONS, + branch_instructions=MIPS_BRANCH_INSTRUCTIONS, + instructions_with_address_immediates=MIPS_BRANCH_INSTRUCTIONS.union({"j", "jal"}), + delay_slot_instructions=MIPS_BRANCH_INSTRUCTIONS.union({"j", "jal", "jr", "jalr"}), + proc=AsmProcessorMIPS, +) + +MIPSEL_SETTINGS = replace(MIPS_SETTINGS, name="mipsel", big_endian=False) + +MIPSEE_SETTINGS = replace( + MIPSEL_SETTINGS, name="mipsee", arch_flags=["-m", "mips:5900"] +) + +MIPS_ARCH_NAMES = {"mips", "mipsel", "mipsee"} + +ARM32_SETTINGS = ArchSettings( + name="arm32", + re_int=re.compile(r"[0-9]+"), + re_comment=re.compile(r"(<.*>|//.*$)"), + # Includes: + # - General purpose registers: r0..13 + # - Frame pointer registers: lr (r14), pc (r15) + # - VFP/NEON registers: s0..31, d0..31, q0..15, fpscr, fpexc, fpsid + # SP should not be in this list. + re_reg=re.compile( + r"\$?\b([rq][0-9]|[rq]1[0-5]|pc|lr|[ds][12]?[0-9]|[ds]3[01]|fp(scr|exc|sid))\b" + ), + re_sprel=re.compile(r"sp, #-?(0x[0-9a-fA-F]+|[0-9]+)\b"), + re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"), + re_imm=re.compile(r"(?<!sp, )#-?(0x[0-9a-fA-F]+|[0-9]+)\b"), + re_reloc=re.compile(r"R_ARM_"), + branch_instructions=ARM32_BRANCH_INSTRUCTIONS, + instructions_with_address_immediates=ARM32_BRANCH_INSTRUCTIONS.union({"adr"}), + proc=AsmProcessorARM32, +) + +ARMEL_SETTINGS = replace(ARM32_SETTINGS, name="armel", big_endian=False) + +AARCH64_SETTINGS = ArchSettings( + name="aarch64", + re_int=re.compile(r"[0-9]+"), + re_comment=re.compile(r"(<.*>|//.*$)"), + # GPRs and FP registers: X0-X30, W0-W30, [BHSDVQ]0..31 + # (FP registers may be followed by data width and number of elements, e.g. V0.4S) + # The zero registers and SP should not be in this list. + re_reg=re.compile( + r"\$?\b([bhsdvq]([12]?[0-9]|3[01])(\.\d\d?[bhsdvq])?|[xw][12]?[0-9]|[xw]30)\b" + ), + re_sprel=re.compile(r"sp, #-?(0x[0-9a-fA-F]+|[0-9]+)\b"), + re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"), + re_imm=re.compile(r"(?<!sp, )#-?(0x[0-9a-fA-F]+|[0-9]+)\b"), + re_reloc=re.compile(r"R_AARCH64_"), + branch_instructions=AARCH64_BRANCH_INSTRUCTIONS, + instructions_with_address_immediates=AARCH64_BRANCH_INSTRUCTIONS.union( + {"bl", "adrp"} + ), + proc=AsmProcessorAArch64, +) + +PPC_SETTINGS = ArchSettings( + name="ppc", + re_int=re.compile(r"[0-9]+"), + re_comment=re.compile(r"(<.*>|//.*$)"), + # r1 not included + re_reg=re.compile(r"\$?\b([rf](?:[02-9]|[1-9][0-9]+)|f1)\b"), + re_sprel=re.compile(r"(?<=,)(-?[0-9]+|-?0x[0-9a-f]+)\(r1\)"), + re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"), + re_imm=re.compile( + r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(r1)|[^ \t,]+@(l|ha|h|sda21)" + ), + re_reloc=re.compile(r"R_PPC_"), + arch_flags=["-m", "powerpc", "-M", "broadway"], + branch_instructions=PPC_BRANCH_INSTRUCTIONS, + instructions_with_address_immediates=PPC_BRANCH_INSTRUCTIONS.union({"bl"}), + proc=AsmProcessorPPC, +) + +I686_SETTINGS = ArchSettings( + name="i686", + re_int=re.compile(r"[0-9]+"), + re_comment=re.compile(r"<.*>"), + # Includes: + # - (e)a-d(x,l,h) + # - (e)s,d,b(i,p)(l) + # - cr0-7 + # - x87 st + # - MMX, SSE vector registers + # - cursed registers: eal ebl ebh edl edh... + re_reg=re.compile( + r"\%?\b(e?(([sd]i|[sb]p)l?|[abcd][xhl])|[cdesfg]s|cr[0-7]|x?mm[0-7]|st)\b" + ), + re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"), + re_sprel=re.compile(r"-?(0x[0-9a-f]+|[0-9]+)(?=\((%ebp|%esi)\))"), + re_imm=re.compile(r"-?(0x[0-9a-f]+|[0-9]+)"), + re_reloc=re.compile(r"R_386_"), + # The x86 architecture has a variable instruction length. The raw bytes of + # an instruction as displayed by objdump can line wrap if it's long enough. + # This destroys the objdump output processor logic, so we avoid this. + arch_flags=["-m", "i386", "--no-show-raw-insn"], + branch_instructions=I686_BRANCH_INSTRUCTIONS, + instructions_with_address_immediates=I686_BRANCH_INSTRUCTIONS.union({"mov"}), + proc=AsmProcessorI686, +) + +ARCH_SETTINGS = [ + MIPS_SETTINGS, + MIPSEL_SETTINGS, + MIPSEE_SETTINGS, + ARM32_SETTINGS, + ARMEL_SETTINGS, + AARCH64_SETTINGS, + PPC_SETTINGS, + I686_SETTINGS, +] + + +def hexify_int(row: str, pat: Match[str], arch: ArchSettings) -> str: + full = pat.group(0) + if len(full) <= 1: + # leave one-digit ints alone + return full + start, end = pat.span() + if start and row[start - 1] in arch.forbidden: + return full + if end < len(row) and row[end] in arch.forbidden: + return full + return hex(int(full)) + + +def parse_relocated_line(line: str) -> Tuple[str, str, str]: + # Pick out the last argument + for c in ",\t ": + if c in line: + ind2 = line.rindex(c) + break + else: + raise Exception(f"failed to parse relocated line: {line}") + before = line[: ind2 + 1] + after = line[ind2 + 1 :] + # Move an optional ($reg) part of it to 'after' + ind2 = after.find("(") + if ind2 == -1: + imm, after = after, "" + else: + imm, after = after[:ind2], after[ind2:] + return before, imm, after + + +def reloc_addend_from_imm(imm: str, before: str, arch: ArchSettings) -> str: + """For architectures like MIPS where relocations have addends embedded in + the code as immediates, convert such an immediate into an addition/ + subtraction that can occur just after the symbol.""" + # TODO this is incorrect for MIPS %lo/%hi which need to be paired up + # and combined. In practice, this means we only get symbol offsets within + # %lo, while %hi just shows the symbol. Unfortunately, objdump's output + # loses relocation order, so we cannot do this without parsing ELF relocs + # ourselves... + mnemonic = before.split()[0] + if mnemonic in arch.instructions_with_address_immediates: + addend = int(imm, 16) + else: + addend = int(imm, 0) + if addend == 0: + return "" + elif addend < 0: + return hex(addend) + else: + return "+" + hex(addend) + + +def pad_mnemonic(line: str) -> str: + if "\t" not in line: + return line + mn, args = line.split("\t", 1) + return f"{mn:<7s} {args}" + + +@dataclass +class Line: + mnemonic: str + diff_row: str + original: str + normalized_original: str + scorable_line: str + symbol: Optional[str] = None + line_num: Optional[int] = None + branch_target: Optional[int] = None + data_pool_addr: Optional[int] = None + source_filename: Optional[str] = None + source_line_num: Optional[int] = None + source_lines: List[str] = field(default_factory=list) + comment: Optional[str] = None + + +def process(dump: str, config: Config) -> List[Line]: + arch = config.arch + processor = arch.proc(config) + source_lines = [] + source_filename = None + source_line_num = None + + i = 0 + num_instr = 0 + data_refs: Dict[int, Dict[str, List[int]]] = defaultdict(lambda: defaultdict(list)) + output: List[Line] = [] + lines = dump.split("\n") + while i < len(lines): + row = lines[i] + i += 1 + + if not row: + continue + + if re.match(r"^[0-9a-f]+ <.*>:$", row): + continue + + if row.startswith("DATAREF"): + parts = row.split(" ", 3) + text_offset = int(parts[1]) + from_offset = int(parts[2]) + from_section = parts[3] + data_refs[text_offset][from_section].append(from_offset) + continue + + if config.diff_obj and num_instr >= config.max_function_size_lines: + output.append( + Line( + mnemonic="...", + diff_row="...", + original="...", + normalized_original="...", + scorable_line="...", + ) + ) + break + + if not re.match(r"^\s+[0-9a-f]+:\s+", row): + # This regex is conservative, and assumes the file path does not contain "weird" + # characters like tabs or angle brackets. + if re.match(r"^[^ \t<>][^\t<>]*:[0-9]+( \(discriminator [0-9]+\))?$", row): + source_filename, _, tail = row.rpartition(":") + source_line_num = int(tail.partition(" ")[0]) + source_lines.append(row) + continue + + # If the instructions loads a data pool symbol, extract the address of + # the symbol. + data_pool_addr = None + pool_match = re.search(ARM32_LOAD_POOL_PATTERN, row) + if pool_match: + offset = pool_match.group(3).split(" ")[0][1:] + data_pool_addr = int(offset, 16) + + m_comment = re.search(arch.re_comment, row) + comment = m_comment[0] if m_comment else None + row = re.sub(arch.re_comment, "", row) + line_num_str = row.split(":")[0] + row = row.rstrip() + tabs = row.split("\t") + line_num = eval_line_num(line_num_str.strip()) + + # TODO: use --no-show-raw-insn for all arches + if arch.name == "i686": + row = "\t".join(tabs[1:]) + else: + row = "\t".join(tabs[2:]) + + if line_num in data_refs: + refs = data_refs[line_num] + ref_str = "; ".join( + section_name + "+" + ",".join(hex(off) for off in offs) + for section_name, offs in refs.items() + ) + output.append( + Line( + mnemonic="<data-ref>", + diff_row="<data-ref>", + original=ref_str, + normalized_original=ref_str, + scorable_line="<data-ref>", + ) + ) + + if "\t" in row: + row_parts = row.split("\t", 1) + else: + # powerpc-eabi-objdump doesn't use tabs + row_parts = [part.lstrip() for part in row.split(" ", 1)] + + mnemonic = row_parts[0].strip() + args = row_parts[1].strip() if len(row_parts) >= 2 else "" + + next_line = lines[i] if i < len(lines) else None + mnemonic, args = processor.pre_process(mnemonic, args, next_line) + row = mnemonic + "\t" + args.replace("\t", " ") + + addr = "" + if mnemonic in arch.instructions_with_address_immediates: + row, addr = split_off_address(row) + # objdump prefixes addresses with 0x/-0x if they don't resolve to some + # symbol + offset. Strip that. + addr = addr.replace("0x", "") + + row = re.sub(arch.re_int, lambda m: hexify_int(row, m, arch), row) + row += addr + + # Let 'original' be 'row' with relocations applied, while we continue + # transforming 'row' into a coarser version that ignores registers and + # immediates. + original = row + + symbol = None + while i < len(lines): + reloc_row = lines[i] + if re.search(arch.re_reloc, reloc_row): + original, reloc_symbol = processor.process_reloc(reloc_row, original) + if reloc_symbol is not None: + symbol = reloc_symbol + else: + break + i += 1 + + is_text_relative_j = False + if ( + arch.name in MIPS_ARCH_NAMES + and mnemonic == "j" + and symbol is not None + and symbol.startswith(".text") + ): + symbol = None + original = row + is_text_relative_j = True + + normalized_original = processor.normalize(mnemonic, original) + + scorable_line = normalized_original + if not config.score_stack_differences: + scorable_line = re.sub(arch.re_sprel, "addr(sp)", scorable_line) + + row = re.sub(arch.re_reg, "<reg>", row) + row = re.sub(arch.re_sprel, "addr(sp)", row) + if mnemonic in arch.instructions_with_address_immediates: + row = row.strip() + row, _ = split_off_address(row) + row += "<imm>" + else: + row = normalize_imms(row, arch) + + branch_target = None + if ( + mnemonic in arch.branch_instructions or is_text_relative_j + ) and symbol is None: + x86_longjmp = re.search(r"\*(.*)\(", args) + if x86_longjmp: + capture = x86_longjmp.group(1) + if capture != "": + branch_target = int(capture, 16) + else: + branch_target = int(args.split(",")[-1], 16) + + output.append( + Line( + mnemonic=mnemonic, + diff_row=row, + original=original, + normalized_original=normalized_original, + scorable_line=scorable_line, + symbol=symbol, + line_num=line_num, + branch_target=branch_target, + data_pool_addr=data_pool_addr, + source_filename=source_filename, + source_line_num=source_line_num, + source_lines=source_lines, + comment=comment, + ) + ) + num_instr += 1 + source_lines = [] + + if config.stop_at_ret and processor.is_end_of_function(mnemonic, args): + break + + processor.post_process(output) + return output + + +def normalize_imms(row: str, arch: ArchSettings) -> str: + return re.sub(arch.re_imm, "<imm>", row) + + +def normalize_stack(row: str, arch: ArchSettings) -> str: + return re.sub(arch.re_sprel, "addr(sp)", row) + + +def check_for_symbol_mismatch( + old_line: Line, new_line: Line, symbol_map: Dict[str, str] +) -> bool: + + assert old_line.symbol is not None + assert new_line.symbol is not None + + if new_line.symbol.startswith("%hi"): + return False + + if old_line.symbol not in symbol_map: + symbol_map[old_line.symbol] = new_line.symbol + return False + elif symbol_map[old_line.symbol] == new_line.symbol: + return False + + return True + + +def field_matches_any_symbol(field: str, arch: ArchSettings) -> bool: + if arch.name == "ppc": + if "..." in field: + return True + + parts = field.rsplit("@", 1) + if len(parts) == 2 and parts[1] in {"l", "h", "ha", "sda21"}: + field = parts[0] + + return re.fullmatch((r"^@\d+$"), field) is not None + + if arch.name in MIPS_ARCH_NAMES: + return "." in field + + # Example: ".text+0x34" + if arch.name == "arm32": + return "." in field + + return False + + +def split_off_address(line: str) -> Tuple[str, str]: + """Split e.g. 'beqz $r0,1f0' into 'beqz $r0,' and '1f0'.""" + parts = line.split(",") + if len(parts) < 2: + parts = line.split(None, 1) + if len(parts) < 2: + parts.append("") + off = len(line) - len(parts[-1].strip()) + return line[:off], line[off:] + + +def diff_sequences_difflib( + seq1: List[str], seq2: List[str] +) -> List[Tuple[str, int, int, int, int]]: + differ = difflib.SequenceMatcher(a=seq1, b=seq2, autojunk=False) + return differ.get_opcodes() + + +def diff_sequences( + seq1: List[str], seq2: List[str], algorithm: str +) -> List[Tuple[str, int, int, int, int]]: + if algorithm != "levenshtein": + return diff_sequences_difflib(seq1, seq2) + + # The Levenshtein library assumes that we compare strings, not lists. Convert. + remapping: Dict[str, str] = {} + + def remap(seq: List[str]) -> str: + seq = seq[:] + for i in range(len(seq)): + val = remapping.get(seq[i]) + if val is None: + val = chr(len(remapping)) + remapping[seq[i]] = val + seq[i] = val + return "".join(seq) + + try: + rem1 = remap(seq1) + rem2 = remap(seq2) + except ValueError as e: + if len(seq1) + len(seq2) < 0x110000: + raise + # If there are too many unique elements, chr() doesn't work. + # Assume this is the case and fall back to difflib. + return diff_sequences_difflib(seq1, seq2) + + import Levenshtein + + ret: List[Tuple[str, int, int, int, int]] = Levenshtein.opcodes(rem1, rem2) + return ret + + +def diff_lines( + lines1: List[Line], + lines2: List[Line], + algorithm: str, +) -> List[Tuple[Optional[Line], Optional[Line]]]: + ret = [] + for (tag, i1, i2, j1, j2) in diff_sequences( + [line.mnemonic for line in lines1], + [line.mnemonic for line in lines2], + algorithm, + ): + for line1, line2 in itertools.zip_longest(lines1[i1:i2], lines2[j1:j2]): + if tag == "replace": + if line1 is None: + tag = "insert" + elif line2 is None: + tag = "delete" + elif tag == "insert": + assert line1 is None + elif tag == "delete": + assert line2 is None + ret.append((line1, line2)) + + return ret + + +def diff_sameline( + old_line: Line, new_line: Line, config: Config, symbol_map: Dict[str, str] +) -> Tuple[int, int, bool]: + + old = old_line.scorable_line + new = new_line.scorable_line + if old == new: + return (0, 0, False) + + num_stack_penalties = 0 + num_regalloc_penalties = 0 + has_symbol_mismatch = False + + ignore_last_field = False + if config.score_stack_differences: + oldsp = re.search(config.arch.re_sprel, old) + newsp = re.search(config.arch.re_sprel, new) + if oldsp and newsp: + oldrel = int(oldsp.group(1) or "0", 0) + newrel = int(newsp.group(1) or "0", 0) + num_stack_penalties += abs(oldrel - newrel) + ignore_last_field = True + + # Probably regalloc difference, or signed vs unsigned + + # Compare each field in order + new_parts, old_parts = new.split(None, 1), old.split(None, 1) + newfields, oldfields = new_parts[1].split(","), old_parts[1].split(",") + if ignore_last_field: + newfields = newfields[:-1] + oldfields = oldfields[:-1] + else: + # If the last field has a parenthesis suffix, e.g. "0x38(r7)" + # we split that part out to make it a separate field + # however, we don't split if it has a proceeding % macro, e.g. "%lo(.data)" + re_paren = re.compile(r"(?<!%hi)(?<!%lo)(?<!%got)(?<!%call16)(?<!%gp_rel)\(") + oldfields = oldfields[:-1] + re_paren.split(oldfields[-1]) + newfields = newfields[:-1] + re_paren.split(newfields[-1]) + + for nf, of in zip(newfields, oldfields): + if nf != of: + # If the new field is a match to any symbol case + # and the old field had a relocation, then ignore this mismatch + if ( + new_line.symbol + and old_line.symbol + and field_matches_any_symbol(nf, config.arch) + ): + if check_for_symbol_mismatch(old_line, new_line, symbol_map): + has_symbol_mismatch = True + continue + num_regalloc_penalties += 1 + + # Penalize any extra fields + num_regalloc_penalties += abs(len(newfields) - len(oldfields)) + + return (num_stack_penalties, num_regalloc_penalties, has_symbol_mismatch) + + +def score_diff_lines( + lines: List[Tuple[Optional[Line], Optional[Line]]], + config: Config, + symbol_map: Dict[str, str], +) -> int: + # This logic is copied from `scorer.py` from the decomp permuter project + # https://github.com/simonlindholm/decomp-permuter/blob/main/src/scorer.py + num_stack_penalties = 0 + num_regalloc_penalties = 0 + num_reordering_penalties = 0 + num_insertion_penalties = 0 + num_deletion_penalties = 0 + deletions = [] + insertions = [] + + def diff_insert(line: str) -> None: + # Reordering or totally different codegen. + # Defer this until later when we can tell. + insertions.append(line) + + def diff_delete(line: str) -> None: + deletions.append(line) + + # Find the end of the last long streak of matching mnemonics, if it looks + # like the objdump output was truncated. This is used to skip scoring + # misaligned lines at the end of the diff. + last_mismatch = -1 + max_index = None + lines_were_truncated = False + for index, (line1, line2) in enumerate(lines): + if (line1 and line1.original == "...") or (line2 and line2.original == "..."): + lines_were_truncated = True + if line1 and line2 and line1.mnemonic == line2.mnemonic: + if index - last_mismatch >= 50: + max_index = index + else: + last_mismatch = index + if not lines_were_truncated: + max_index = None + + for index, (line1, line2) in enumerate(lines): + if max_index is not None and index > max_index: + break + if line1 and line2 and line1.mnemonic == line2.mnemonic: + sp, rp, _ = diff_sameline(line1, line2, config, symbol_map) + num_stack_penalties += sp + num_regalloc_penalties += rp + else: + if line1: + diff_delete(line1.scorable_line) + if line2: + diff_insert(line2.scorable_line) + + insertions_co = Counter(insertions) + deletions_co = Counter(deletions) + for item in insertions_co + deletions_co: + ins = insertions_co[item] + dels = deletions_co[item] + common = min(ins, dels) + num_insertion_penalties += ins - common + num_deletion_penalties += dels - common + num_reordering_penalties += common + + return ( + num_stack_penalties * config.penalty_stackdiff + + num_regalloc_penalties * config.penalty_regalloc + + num_reordering_penalties * config.penalty_reordering + + num_insertion_penalties * config.penalty_insertion + + num_deletion_penalties * config.penalty_deletion + ) + + +@dataclass(frozen=True) +class OutputLine: + base: Optional[Text] = field(compare=False) + fmt2: Text = field(compare=False) + key2: Optional[str] + boring: bool = field(compare=False) + is_data_ref: bool = field(compare=False) + line1: Optional[Line] = field(compare=False) + line2: Optional[Line] = field(compare=False) + + +@dataclass(frozen=True) +class Diff: + lines: List[OutputLine] + score: int + max_score: int + + +def trim_nops(lines: List[Line], arch: ArchSettings) -> List[Line]: + lines = lines[:] + while ( + lines + and lines[-1].mnemonic == "nop" + and (len(lines) == 1 or lines[-2].mnemonic not in arch.delay_slot_instructions) + ): + lines.pop() + return lines + + +def do_diff(lines1: List[Line], lines2: List[Line], config: Config) -> Diff: + if config.show_source: + import cxxfilt + arch = config.arch + fmt = config.formatter + output: List[OutputLine] = [] + symbol_map: Dict[str, str] = {} + + sc1 = symbol_formatter("base-reg", 0) + sc2 = symbol_formatter("my-reg", 0) + sc3 = symbol_formatter("base-stack", 4) + sc4 = symbol_formatter("my-stack", 4) + sc5 = symbol_formatter("base-branch", 0) + sc6 = symbol_formatter("my-branch", 0) + bts1: Set[int] = set() + bts2: Set[int] = set() + + if config.show_branches: + for (lines, btset, sc) in [ + (lines1, bts1, sc5), + (lines2, bts2, sc6), + ]: + for line in lines: + bt = line.branch_target + if bt is not None: + btset.add(bt) + sc(str(bt)) + + lines1 = trim_nops(lines1, arch) + lines2 = trim_nops(lines2, arch) + + diffed_lines = diff_lines(lines1, lines2, config.algorithm) + + line_num_base = -1 + line_num_offset = 0 + line_num_2to1 = {} + for (line1, line2) in diffed_lines: + if line1 is not None and line1.line_num is not None: + line_num_base = line1.line_num + line_num_offset = 0 + else: + line_num_offset += 1 + if line2 is not None and line2.line_num is not None: + line_num_2to1[line2.line_num] = (line_num_base, line_num_offset) + + for (line1, line2) in diffed_lines: + line_color1 = line_color2 = sym_color = BasicFormat.NONE + line_prefix = " " + is_data_ref = False + out1 = Text() if not line1 else Text(pad_mnemonic(line1.original)) + out2 = Text() if not line2 else Text(pad_mnemonic(line2.original)) + if line1 and line2 and line1.diff_row == line2.diff_row: + if line1.diff_row == "<data-ref>": + if line1.normalized_original != line2.normalized_original: + line_prefix = "i" + sym_color = BasicFormat.DIFF_CHANGE + out1 = out1.reformat(sym_color) + out2 = out2.reformat(sym_color) + is_data_ref = True + elif ( + line1.normalized_original == line2.normalized_original + and line2.branch_target is None + ): + # Fast path: no coloring needed. We don't include branch instructions + # in this case because we need to check that their targets line up in + # the diff, and don't just happen to have the are the same address + # by accident. + pass + else: + mnemonic = line1.original.split()[0] + branchless1, address1 = out1.plain(), "" + branchless2, address2 = out2.plain(), "" + if mnemonic in arch.instructions_with_address_immediates: + branchless1, address1 = split_off_address(branchless1) + branchless2, address2 = split_off_address(branchless2) + + out1 = Text(branchless1) + out2 = Text(branchless2) + out1, out2 = format_fields( + arch.re_imm, out1, out2, lambda _: BasicFormat.IMMEDIATE + ) + + if line2.branch_target is not None: + target = line2.branch_target + line2_target = line_num_2to1.get(line2.branch_target) + if line2_target is None: + # If the target is outside the disassembly, extrapolate. + # This only matters near the bottom. + assert line2.line_num is not None + line2_line = line_num_2to1[line2.line_num] + line2_target = (line2_line[0] + (target - line2.line_num), 0) + + # Adjust the branch target for scoring and three-way diffing. + norm2, norm_branch2 = split_off_address(line2.normalized_original) + if norm_branch2 != "<ignore>": + retargetted = hex(line2_target[0]).replace("0x", "") + if line2_target[1] != 0: + retargetted += f"+{line2_target[1]}" + line2.normalized_original = norm2 + retargetted + sc_base, _ = split_off_address(line2.scorable_line) + line2.scorable_line = sc_base + retargetted + same_target = line2_target == (line1.branch_target, 0) + else: + # Do a naive comparison for non-branches (e.g. function calls). + same_target = address1 == address2 + + if normalize_imms(branchless1, arch) == normalize_imms( + branchless2, arch + ): + ( + stack_penalties, + regalloc_penalties, + has_symbol_mismatch, + ) = diff_sameline(line1, line2, config, symbol_map) + + if ( + regalloc_penalties == 0 + and stack_penalties == 0 + and not has_symbol_mismatch + ): + # ignore differences due to %lo(.rodata + ...) vs symbol + out1 = out1.reformat(BasicFormat.NONE) + out2 = out2.reformat(BasicFormat.NONE) + elif line2.branch_target is not None and same_target: + # same-target branch, don't color + pass + else: + # must have an imm difference (or else we would have hit the + # fast path) + sym_color = BasicFormat.IMMEDIATE + line_prefix = "i" + else: + out1, out2 = format_fields(arch.re_sprel, out1, out2, sc3, sc4) + if normalize_stack(branchless1, arch) == normalize_stack( + branchless2, arch + ): + # only stack differences (luckily stack and imm + # differences can't be combined in MIPS, so we + # don't have to think about that case) + sym_color = BasicFormat.STACK + line_prefix = "s" + else: + # reg differences and maybe imm as well + out1, out2 = format_fields(arch.re_reg, out1, out2, sc1, sc2) + cats = config.reg_categories + if cats and any( + cats.get(of.group()) != cats.get(nf.group()) + for (of, nf) in zip( + out1.finditer(arch.re_reg), out2.finditer(arch.re_reg) + ) + ): + sym_color = BasicFormat.REGISTER_CATEGORY + line_prefix = "R" + else: + sym_color = BasicFormat.REGISTER + line_prefix = "r" + line_color1 = line_color2 = sym_color + + if same_target: + address_imm_fmt = BasicFormat.NONE + else: + address_imm_fmt = BasicFormat.IMMEDIATE + out1 += Text(address1, address_imm_fmt) + out2 += Text(address2, address_imm_fmt) + elif line1 and line2: + line_prefix = "|" + line_color1 = line_color2 = sym_color = BasicFormat.DIFF_CHANGE + out1 = out1.reformat(line_color1) + out2 = out2.reformat(line_color2) + elif line1: + line_prefix = "<" + line_color1 = sym_color = BasicFormat.DIFF_REMOVE + out1 = out1.reformat(line_color1) + out2 = Text() + elif line2: + line_prefix = ">" + line_color2 = sym_color = BasicFormat.DIFF_ADD + out1 = Text() + out2 = out2.reformat(line_color2) + + if config.show_source and line2 and line2.comment: + out2 += f" {line2.comment}" + + def format_part( + out: Text, + line: Optional[Line], + line_color: Format, + btset: Set[int], + sc: FormatFunction, + ) -> Optional[Text]: + if line is None: + return None + if line.line_num is None: + return out + in_arrow = Text(" ") + out_arrow = Text() + if config.show_branches: + if line.line_num in btset: + in_arrow = Text("~>", sc(str(line.line_num))) + if line.branch_target is not None: + out_arrow = " " + Text("~>", sc(str(line.branch_target))) + formatted_line_num = Text(hex(line.line_num)[2:] + ":", line_color) + return formatted_line_num + " " + in_arrow + " " + out + out_arrow + + part1 = format_part(out1, line1, line_color1, bts1, sc5) + part2 = format_part(out2, line2, line_color2, bts2, sc6) + + if config.show_source and line2: + for source_line in line2.source_lines: + line_format = BasicFormat.SOURCE_OTHER + if config.source_old_binutils: + if source_line and re.fullmatch(".*\.c(?:pp)?:\d+", source_line): + line_format = BasicFormat.SOURCE_FILENAME + elif source_line and source_line.endswith("():"): + line_format = BasicFormat.SOURCE_FUNCTION + try: + source_line = cxxfilt.demangle( + source_line[:-3], external_only=False + ) + except: + pass + else: + # File names and function names + if source_line and source_line[0] != "│": + line_format = BasicFormat.SOURCE_FILENAME + # Function names + if source_line.endswith("():"): + line_format = BasicFormat.SOURCE_FUNCTION + try: + source_line = cxxfilt.demangle( + source_line[:-3], external_only=False + ) + except: + pass + padding = " " * 7 if config.show_line_numbers else " " * 2 + output.append( + OutputLine( + base=None, + fmt2=padding + Text(source_line, line_format), + key2=source_line, + boring=True, + is_data_ref=False, + line1=None, + line2=None, + ) + ) + + key2 = line2.normalized_original if line2 else None + boring = False + if line_prefix == " ": + boring = True + elif config.compress and config.compress.same_instr and line_prefix in "irs": + boring = True + + if config.show_line_numbers: + if line2 and line2.source_line_num is not None: + num_color = ( + BasicFormat.SOURCE_LINE_NUM + if sym_color == BasicFormat.NONE + else sym_color + ) + num2 = Text(f"{line2.source_line_num:5}", num_color) + else: + num2 = Text(" " * 5) + else: + num2 = Text() + + fmt2 = Text(line_prefix, sym_color) + num2 + " " + (part2 or Text()) + + output.append( + OutputLine( + base=part1, + fmt2=fmt2, + key2=key2, + boring=boring, + is_data_ref=is_data_ref, + line1=line1, + line2=line2, + ) + ) + + output = output[config.skip_lines :] + + score = score_diff_lines(diffed_lines, config, symbol_map) + max_score = len(lines1) * config.penalty_deletion + return Diff(lines=output, score=score, max_score=max_score) + + +def chunk_diff_lines( + diff: List[OutputLine], +) -> List[Union[List[OutputLine], OutputLine]]: + """Chunk a diff into an alternating list like A B A B ... A, where: + * A is a List[OutputLine] of insertions, + * B is a single non-insertion OutputLine, with .base != None.""" + cur_right: List[OutputLine] = [] + chunks: List[Union[List[OutputLine], OutputLine]] = [] + for output_line in diff: + if output_line.base is not None: + chunks.append(cur_right) + chunks.append(output_line) + cur_right = [] + else: + cur_right.append(output_line) + chunks.append(cur_right) + return chunks + + +def compress_matching( + li: List[Tuple[OutputLine, ...]], context: int +) -> List[Tuple[OutputLine, ...]]: + ret: List[Tuple[OutputLine, ...]] = [] + matching_streak: List[Tuple[OutputLine, ...]] = [] + context = max(context, 0) + + def flush_matching() -> None: + if len(matching_streak) <= 2 * context + 1: + ret.extend(matching_streak) + else: + ret.extend(matching_streak[:context]) + skipped = len(matching_streak) - 2 * context + filler = OutputLine( + base=Text(f"<{skipped} lines>", BasicFormat.SOURCE_OTHER), + fmt2=Text(), + key2=None, + boring=False, + is_data_ref=False, + line1=None, + line2=None, + ) + columns = len(matching_streak[0]) + ret.append(tuple([filler] * columns)) + if context > 0: + ret.extend(matching_streak[-context:]) + matching_streak.clear() + + for line in li: + if line[0].boring: + matching_streak.append(line) + else: + flush_matching() + ret.append(line) + + flush_matching() + return ret + + +def align_diffs(old_diff: Diff, new_diff: Diff, config: Config) -> TableData: + headers: Tuple[Text, ...] + diff_lines: List[Tuple[OutputLine, ...]] + padding = " " * 7 if config.show_line_numbers else " " * 2 + + if config.diff_mode in (DiffMode.THREEWAY_PREV, DiffMode.THREEWAY_BASE): + old_chunks = chunk_diff_lines(old_diff.lines) + new_chunks = chunk_diff_lines(new_diff.lines) + diff_lines = [] + empty = OutputLine(Text(), Text(), None, True, False, None, None) + assert len(old_chunks) == len(new_chunks), "same target" + for old_chunk, new_chunk in zip(old_chunks, new_chunks): + if isinstance(old_chunk, list): + assert isinstance(new_chunk, list) + if not old_chunk and not new_chunk: + # Most of the time lines sync up without insertions/deletions, + # and there's no interdiffing to be done. + continue + differ = difflib.SequenceMatcher( + a=old_chunk, b=new_chunk, autojunk=False + ) + for (tag, i1, i2, j1, j2) in differ.get_opcodes(): + if tag in ["equal", "replace"]: + for i, j in zip(range(i1, i2), range(j1, j2)): + diff_lines.append((empty, new_chunk[j], old_chunk[i])) + if tag in ["insert", "replace"]: + for j in range(j1 + i2 - i1, j2): + diff_lines.append((empty, new_chunk[j], empty)) + if tag in ["delete", "replace"]: + for i in range(i1 + j2 - j1, i2): + diff_lines.append((empty, empty, old_chunk[i])) + else: + assert isinstance(new_chunk, OutputLine) + # old_chunk.base and new_chunk.base have the same text since + # both diffs are based on the same target, but they might + # differ in color. Use the new version. + diff_lines.append((new_chunk, new_chunk, old_chunk)) + diff_lines = [ + (base, new, old if old != new else empty) for base, new, old in diff_lines + ] + headers = ( + Text("TARGET"), + Text(f"{padding}CURRENT ({new_diff.score})"), + Text(f"{padding}PREVIOUS ({old_diff.score})"), + ) + current_score = new_diff.score + max_score = new_diff.max_score + previous_score = old_diff.score + elif config.diff_mode in (DiffMode.SINGLE, DiffMode.SINGLE_BASE): + header = Text("BASE" if config.diff_mode == DiffMode.SINGLE_BASE else "CURRENT") + diff_lines = [(line,) for line in new_diff.lines] + headers = (header,) + # Scoring is disabled for view mode + current_score = 0 + max_score = 0 + previous_score = None + else: + diff_lines = [(line, line) for line in new_diff.lines] + headers = ( + Text("TARGET"), + Text(f"{padding}CURRENT ({new_diff.score})"), + ) + current_score = new_diff.score + max_score = new_diff.max_score + previous_score = None + if config.compress: + diff_lines = compress_matching(diff_lines, config.compress.context) + + def diff_line_to_table_line(line: Tuple[OutputLine, ...]) -> TableLine: + cells = [ + (line[0].base or Text(), line[0].line1) + ] + for ol in line[1:]: + cells.append((ol.fmt2, ol.line2)) + + return TableLine( + key=line[0].key2, + is_data_ref=line[0].is_data_ref, + cells=tuple(cells), + ) + + return TableData( + headers=headers, + current_score=current_score, + max_score=max_score, + previous_score=previous_score, + lines=[diff_line_to_table_line(line) for line in diff_lines], + ) + + +def debounced_fs_watch( + targets: List[str], + outq: "queue.Queue[Optional[float]]", + config: Config, + project: ProjectSettings, +) -> None: + import watchdog.events + import watchdog.observers + + class WatchEventHandler(watchdog.events.FileSystemEventHandler): + def __init__( + self, queue: "queue.Queue[float]", file_targets: List[str] + ) -> None: + self.queue = queue + self.file_targets = file_targets + + def on_modified(self, ev: object) -> None: + if isinstance(ev, watchdog.events.FileModifiedEvent): + self.changed(ev.src_path) + + def on_moved(self, ev: object) -> None: + if isinstance(ev, watchdog.events.FileMovedEvent): + self.changed(ev.dest_path) + + def should_notify(self, path: str) -> bool: + for target in self.file_targets: + if os.path.normpath(path) == target: + return True + if config.make and any( + path.endswith(suffix) for suffix in project.source_extensions + ): + return True + return False + + def changed(self, path: str) -> None: + if self.should_notify(path): + self.queue.put(time.time()) + + def debounce_thread() -> NoReturn: + listenq: "queue.Queue[float]" = queue.Queue() + file_targets: List[str] = [] + event_handler = WatchEventHandler(listenq, file_targets) + observer = watchdog.observers.Observer() + observed = set() + for target in targets: + if os.path.isdir(target): + observer.schedule(event_handler, target, recursive=True) + else: + file_targets.append(os.path.normpath(target)) + target = os.path.dirname(target) or "." + if target not in observed: + observed.add(target) + observer.schedule(event_handler, target) + observer.start() + while True: + t = listenq.get() + more = True + while more: + delay = t + DEBOUNCE_DELAY - time.time() + if delay > 0: + time.sleep(delay) + # consume entire queue + more = False + try: + while True: + t = listenq.get(block=False) + more = True + except queue.Empty: + pass + outq.put(t) + + th = threading.Thread(target=debounce_thread, daemon=True) + th.start() + + +class Display: + basedump: str + mydump: str + last_refresh_key: object + config: Config + emsg: Optional[str] + last_diff_output: Optional[Diff] + pending_update: Optional[str] + ready_queue: "queue.Queue[None]" + watch_queue: "queue.Queue[Optional[float]]" + less_proc: "Optional[subprocess.Popen[bytes]]" + + def __init__(self, basedump: str, mydump: str, config: Config) -> None: + self.config = config + self.base_lines = process(basedump, config) + self.mydump = mydump + self.emsg = None + self.last_refresh_key = None + self.last_diff_output = None + + def run_diff(self) -> Tuple[str, object]: + if self.emsg is not None: + return (self.emsg, self.emsg) + + my_lines = process(self.mydump, self.config) + + if self.config.diff_mode == DiffMode.SINGLE_BASE: + diff_output = do_diff(self.base_lines, self.base_lines, self.config) + elif self.config.diff_mode == DiffMode.SINGLE: + diff_output = do_diff(my_lines, my_lines, self.config) + else: + diff_output = do_diff(self.base_lines, my_lines, self.config) + + last_diff_output = self.last_diff_output or diff_output + if self.config.diff_mode != DiffMode.THREEWAY_BASE or not self.last_diff_output: + self.last_diff_output = diff_output + + data = align_diffs(last_diff_output, diff_output, self.config) + output = self.config.formatter.table(data) + + refresh_key = ( + [line.key2 for line in diff_output.lines], + diff_output.score, + ) + + return (output, refresh_key) + + def run_less( + self, output: str + ) -> "Tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]": + # Pipe the output through 'tail' and only then to less, to ensure the + # write call doesn't block. ('tail' has to buffer all its input before + # it starts writing.) This also means we don't have to deal with pipe + # closure errors. + buffer_proc = subprocess.Popen( + BUFFER_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + less_proc = subprocess.Popen(LESS_CMD, stdin=buffer_proc.stdout) + assert buffer_proc.stdin + assert buffer_proc.stdout + buffer_proc.stdin.write(output.encode()) + buffer_proc.stdin.close() + buffer_proc.stdout.close() + return (buffer_proc, less_proc) + + def run_sync(self) -> None: + output, _ = self.run_diff() + proca, procb = self.run_less(output) + procb.wait() + proca.wait() + + def run_async(self, watch_queue: "queue.Queue[Optional[float]]") -> None: + self.watch_queue = watch_queue + self.ready_queue = queue.Queue() + self.pending_update = None + output, refresh_key = self.run_diff() + self.last_refresh_key = refresh_key + dthread = threading.Thread(target=self.display_thread, args=(output,)) + dthread.start() + self.ready_queue.get() + + def display_thread(self, initial_output: str) -> None: + proca, procb = self.run_less(initial_output) + self.less_proc = procb + self.ready_queue.put(None) + while True: + ret = procb.wait() + proca.wait() + self.less_proc = None + if ret != 0: + # fix the terminal + os.system("tput reset") + if ret != 0 and self.pending_update is not None: + # killed by program with the intent to refresh + output = self.pending_update + self.pending_update = None + proca, procb = self.run_less(output) + self.less_proc = procb + self.ready_queue.put(None) + else: + # terminated by user, or killed + self.watch_queue.put(None) + self.ready_queue.put(None) + break + + def progress(self, msg: str) -> None: + # Write message to top-left corner + sys.stdout.write("\x1b7\x1b[1;1f{}\x1b8".format(msg + " ")) + sys.stdout.flush() + + def update(self, text: str, error: bool) -> None: + if not error and not self.emsg and text == self.mydump: + self.progress("Unchanged. ") + return + if not error: + self.mydump = text + self.emsg = None + else: + self.emsg = text + output, refresh_key = self.run_diff() + if refresh_key == self.last_refresh_key: + self.progress("Unchanged. ") + return + self.last_refresh_key = refresh_key + self.pending_update = output + if not self.less_proc: + return + self.less_proc.kill() + self.ready_queue.get() + + def terminate(self) -> None: + if not self.less_proc: + return + self.less_proc.kill() + self.ready_queue.get() + + +def main() -> None: + args = parser.parse_args() + + # Apply project-specific configuration. + settings: Dict[str, Any] = {} + diff_settings.apply(settings, args) # type: ignore + project = create_project_settings(settings) + + try: + config = create_config(args, project) + except ValueError as e: + fail(str(e)) + + if config.algorithm == "levenshtein": + try: + import Levenshtein + except ModuleNotFoundError as e: + fail(MISSING_PREREQUISITES.format(e.name)) + + if config.show_source: + try: + import cxxfilt + except ModuleNotFoundError as e: + fail(MISSING_PREREQUISITES.format(e.name)) + + if ( + config.diff_mode in (DiffMode.THREEWAY_BASE, DiffMode.THREEWAY_PREV) + and not args.watch + ): + fail("Threeway diffing requires -w.") + + if args.diff_elf_symbol: + make_target, basecmd, mycmd = dump_elf( + args.start, args.end, args.diff_elf_symbol, config, project + ) + elif config.diff_obj: + make_target, basecmd, mycmd = dump_objfile( + args.start, args.end, config, project + ) + else: + make_target, basecmd, mycmd = dump_binary(args.start, args.end, config, project) + + map_build_target_fn = getattr(diff_settings, "map_build_target", None) + if map_build_target_fn: + make_target = map_build_target_fn(make_target=make_target) + + if args.write_asm is not None: + mydump = run_objdump(mycmd, config, project) + with open(args.write_asm, "w") as f: + f.write(mydump) + print(f"Wrote assembly to {args.write_asm}.") + sys.exit(0) + + if args.base_asm is not None: + with open(args.base_asm) as f: + basedump = f.read() + elif config.diff_mode != DiffMode.SINGLE: + basedump = run_objdump(basecmd, config, project) + else: + basedump = "" + + mydump = run_objdump(mycmd, config, project) + + display = Display(basedump, mydump, config) + + if args.no_pager or args.format in ("html", "json"): + print(display.run_diff()[0]) + elif not args.watch: + display.run_sync() + else: + if not args.make: + yn = input( + "Warning: watch-mode (-w) enabled without auto-make (-m). " + "You will have to run make manually. Ok? (Y/n) " + ) + if yn.lower() == "n": + return + if args.make: + watch_sources = None + watch_sources_for_target_fn = getattr( + diff_settings, "watch_sources_for_target", None + ) + if watch_sources_for_target_fn: + watch_sources = watch_sources_for_target_fn(make_target) + watch_sources = watch_sources or project.source_directories + if not watch_sources: + fail("Missing source_directories config, don't know what to watch.") + else: + watch_sources = [make_target] + q: "queue.Queue[Optional[float]]" = queue.Queue() + debounced_fs_watch(watch_sources, q, config, project) + display.run_async(q) + last_build = 0.0 + try: + while True: + t = q.get() + if t is None: + break + if t < last_build: + continue + last_build = time.time() + if args.make: + display.progress("Building...") + ret = run_make_capture_output(make_target, project) + if ret.returncode != 0: + display.update( + ret.stderr.decode("utf-8-sig", "replace") + or ret.stdout.decode("utf-8-sig", "replace"), + error=True, + ) + continue + mydump = run_objdump(mycmd, config, project) + display.update(mydump, error=False) + except KeyboardInterrupt: + display.terminate() + + +if __name__ == "__main__": + main() diff --git a/tools/asm-differ/diff_settings.py b/tools/asm-differ/diff_settings.py new file mode 100644 index 0000000..19d67d5 --- /dev/null +++ b/tools/asm-differ/diff_settings.py @@ -0,0 +1,12 @@ +def apply(config, args): + config["baseimg"] = "target.bin" + config["myimg"] = "source.bin" + config["mapfile"] = "build.map" + config["source_directories"] = ["."] + # config["show_line_numbers_default"] = True + # config["arch"] = "mips" + # config["map_format"] = "gnu" # gnu, mw, ms + # config["build_dir"] = "build/" # only needed for mw and ms map format + # config["expected_dir"] = "expected/" # needed for -o + # config["makeflags"] = [] + # config["objdump_executable"] = "" diff --git a/tools/asm-differ/mypy.ini b/tools/asm-differ/mypy.ini new file mode 100644 index 0000000..138b939 --- /dev/null +++ b/tools/asm-differ/mypy.ini @@ -0,0 +1,17 @@ +[mypy] +check_untyped_defs = True +disallow_any_generics = True +disallow_incomplete_defs = True +disallow_untyped_calls = True +disallow_untyped_decorators = True +disallow_untyped_defs = True +no_implicit_optional = True +warn_redundant_casts = True +warn_return_any = True +warn_unused_ignores = True +ignore_missing_imports = True +python_version = 3.6 +files = diff.py + +[mypy-diff_settings] +ignore_errors = True diff --git a/tools/asm-differ/pyproject.toml b/tools/asm-differ/pyproject.toml new file mode 100644 index 0000000..7a112ae --- /dev/null +++ b/tools/asm-differ/pyproject.toml @@ -0,0 +1,21 @@ +[tool.poetry] +name = "asm-differ" +version = "0.1.0" +description = "" +authors = ["Simon Lindholm <simon.lindholm10@gmail.com>"] +license = "UNLICENSE" +readme = "README.md" +packages = [{ include = "diff.py" }] + +[tool.poetry.dependencies] +python = "^3.7" +colorama = "^0.4.6" +ansiwrap = "^0.8.4" +watchdog = "^2.2.0" +levenshtein = "^0.20.9" +cxxfilt = "^0.3.0" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/tools/asm-differ/screenshot.png b/tools/asm-differ/screenshot.png Binary files differnew file mode 100644 index 0000000..3230555 --- /dev/null +++ b/tools/asm-differ/screenshot.png diff --git a/tools/asm-processor/.gitignore b/tools/asm-processor/.gitignore new file mode 100644 index 0000000..cc5bba4 --- /dev/null +++ b/tools/asm-processor/.gitignore @@ -0,0 +1,2 @@ +*.o +*.py[cod] diff --git a/tools/asm-processor/.gitrepo b/tools/asm-processor/.gitrepo new file mode 100644 index 0000000..ef2baa1 --- /dev/null +++ b/tools/asm-processor/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme +; +[subrepo] + remote = git@github.com:simonlindholm/asm-processor.git + branch = main + commit = bbd86ea1faf84e6a7a0e101ab8068a00a3dfb2fc + parent = 75437386de04179489f4fd5fb345d122ff8a2dc0 + method = merge + cmdver = 0.4.3 diff --git a/tools/asm-processor/LICENSE b/tools/asm-processor/LICENSE new file mode 100644 index 0000000..cf1ab25 --- /dev/null +++ b/tools/asm-processor/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to <http://unlicense.org> diff --git a/tools/asm-processor/README.md b/tools/asm-processor/README.md new file mode 100644 index 0000000..009147c --- /dev/null +++ b/tools/asm-processor/README.md @@ -0,0 +1,112 @@ +# asm-processor + +Pre-process .c files and post-process .o files to enable embedding MIPS assembly into IDO-compiled C. + +## Usage + +Let's say you have a file compiled with `-g` on the IDO compiler, that looks like this: +```c +float func4(void) { + "func4"; + return 0.2f; +} +``` + +This script enables replacing it by: +```asm +GLOBAL_ASM( +.rdata +.word 0x66756e63 # func +.word 0x34000000 # 4\0\0\0 + +.late_rodata +glabel rv +.word 0x3e4ccccd # 0.2f + +.text +glabel func4 +lui $at, %hi(rv) +jr $ra +lwc1 $f0, %lo(rv)($at) +jr $ra +nop +jr $ra +nop +) +``` + +To compile the file, run `python3 build.py $CC -- $AS $ASFLAGS -- $CFLAGS -o out.o in.c`, where $CC points to an IDO binary (5.3/7.1 and recomp/qemu all supported), $AS is e.g. `mips-linux-gnu-as`, $ASFLAGS e.g. `-march=vr4300 -mabi=32` and $CFLAGS e.g. `-Wab,-r4300_mul -non_shared -G 0 -Xcpluscomm -g`. build.py may be customized as needed. + +In addition to an .o file, build.py also generates a .d file with Makefile dependencies for .s files referenced by the input .c file. +This functionality may be removed if not needed. + +Reading assembly from file is also supported, by either `GLOBAL_ASM("file.s")` or `#pragma GLOBAL_ASM("file.s")`. + +### What is supported? + +`.text`, `.data`, `.bss` and `.rodata` sections, `.word`/`.incbin`, `.ascii`/`.asciz`, and `-g`, `-g3`, `-O1`, `-O2`, `-framepointer` and `-mips1`/`-mips2` flags to the IDO compiler. + +### What is not supported? + +* complicated assembly (.ifdef, macro declarations/calls other than `glabel`, pseudo-instructions that expand to several real instructions) +* non-IDO compilers +* `-O3` (due to function reordering) + +C `#ifdef`s only work outside of `GLOBAL_ASM` calls, but is otherwise able to replace `.ifdef`. + +### What's up with "late rodata"? + +The IDO compiler emits rodata in two passes: first array/string contents, then large literals/switch jump tables. + +Data declared within `.rdata`/`.section .rodata` will end up in the first half, and `.late_rodata`/`.section .late_rodata` in the second half. + +### How does it work? + +It's a bit of a hack! +The basic idea is replace `GLOBAL_ASM` blocks with dummy C functions/global vars of the same sections sizes as the assembly. +Then the C file gets compiled, and the dummy contents overwritten with the injected assembly. + +To accomplish this, asm-processor has logic for guessing the size of assembly contents +(which assumes the assembly isn't too complicated, e.g. no macros), +and for emitting C code of exact sizes for a bunch of different IDO compiler flags. + +The assembler code is padded with nops to line it up with its correct position in the C; +this allows C and asm ELF files to be merged easily without having to fix up e.g. symbol addresses. + +The most difficult part is `late_rodata`, which is hard to create programmatically. +asm-processor does that by emitting code that uses dummy float literals/double literals/jump tables, +assembles the late_rodata at another location of the .rodata section, then overwrites the dummy rodata. +This does require some movement of symbols and relocations, and quite a bit of care in what code to +emit and how to preserve .double alignment. + +It's worth noting some alternative ways in which asm-processor would have been implemented: +- One idea to get rid of the C/asm size estimations is to emit arbitrary code, and then move code, +symbols and relocations to the correct place after the sizes are known. +Given the machinery for `late_rodata` this wouldn't have been too difficult, and it would have the upside of improved portability. +There is a big downside, however: using dummy code of incorrect size throws off alignment and can introduce unintended padding. +Fixing this would require running multiple passes of asm-processor, with one compile per `ASM_GLOBAL`. +- Another idea is to run the compiler with -S to emit assembly, modify the emitted assembly, then run the assembler +(which in IDO's case may perform additional instruction reordering etc.). +This option has not been investigated in much detail, and would perhaps be superior to the current implementation. +It does have a few unknowns to it, e.g. instruction encoding differences between GNU `as` and IDO's assembler, +how to avoid reordering the injected assembly, and how .rodata/.late_rodata are implemented. + +### Testing + +There are a few tests to ensure you don't break anything when hacking on asm-processor: `./run-tests.sh` should exit without output if they pass, or else output a diff from previous to new version. + +Tests need the environment variable `MIPS_CC` set to point to the IDO 7.1 compiler, with Pascal support enabled. + +For example if asm-processor is cloned in the same directory as [ido static recomp](https://github.com/decompals/ido-static-recomp) and the working directory is asm-processor, tests can be run using: + +```sh +MIPS_CC=../ido-static-recomp/build/7.1/out/cc ./run-tests.sh +``` + +Or using [qemu-irix](https://github.com/zeldaret/oot/releases/tag/0.1q) (don't forget `chmod u+x qemu-irix`) to emulate IDO: + +```sh +MIPS_CC='./qemu-irix -silent -L ../ido-static-recomp/ido/7.1/ ../ido-static-recomp/ido/7.1/usr/bin/cc' ./run-tests.sh +``` + +To skip running Pascal tests, remove the `tests/*.p` glob from `run-tests.sh`. diff --git a/tools/asm-processor/add-test.sh b/tools/asm-processor/add-test.sh new file mode 100755 index 0000000..708548e --- /dev/null +++ b/tools/asm-processor/add-test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +for A in "$@"; do + OBJDUMPFLAGS="-srt" + ./compile-test.sh "$A" && mips-linux-gnu-objdump $OBJDUMPFLAGS "${A%.*}.o" > "${A%.*}.objdump" +done diff --git a/tools/asm-processor/asm_processor.py b/tools/asm-processor/asm_processor.py new file mode 100644 index 0000000..0d62bc3 --- /dev/null +++ b/tools/asm-processor/asm_processor.py @@ -0,0 +1,1472 @@ +#!/usr/bin/env python3 +import argparse +import tempfile +import struct +import copy +import sys +import re +import os +from collections import namedtuple +from io import StringIO + +MAX_FN_SIZE = 100 +SLOW_CHECKS = False + +EI_NIDENT = 16 +EI_CLASS = 4 +EI_DATA = 5 +EI_VERSION = 6 +EI_OSABI = 7 +EI_ABIVERSION = 8 +STN_UNDEF = 0 + +SHN_UNDEF = 0 +SHN_ABS = 0xfff1 +SHN_COMMON = 0xfff2 +SHN_XINDEX = 0xffff +SHN_LORESERVE = 0xff00 + +STT_NOTYPE = 0 +STT_OBJECT = 1 +STT_FUNC = 2 +STT_SECTION = 3 +STT_FILE = 4 +STT_COMMON = 5 +STT_TLS = 6 + +STB_LOCAL = 0 +STB_GLOBAL = 1 +STB_WEAK = 2 + +STV_DEFAULT = 0 +STV_INTERNAL = 1 +STV_HIDDEN = 2 +STV_PROTECTED = 3 + +SHT_NULL = 0 +SHT_PROGBITS = 1 +SHT_SYMTAB = 2 +SHT_STRTAB = 3 +SHT_RELA = 4 +SHT_HASH = 5 +SHT_DYNAMIC = 6 +SHT_NOTE = 7 +SHT_NOBITS = 8 +SHT_REL = 9 +SHT_SHLIB = 10 +SHT_DYNSYM = 11 +SHT_INIT_ARRAY = 14 +SHT_FINI_ARRAY = 15 +SHT_PREINIT_ARRAY = 16 +SHT_GROUP = 17 +SHT_SYMTAB_SHNDX = 18 +SHT_MIPS_GPTAB = 0x70000003 +SHT_MIPS_DEBUG = 0x70000005 +SHT_MIPS_REGINFO = 0x70000006 +SHT_MIPS_OPTIONS = 0x7000000d + +SHF_WRITE = 0x1 +SHF_ALLOC = 0x2 +SHF_EXECINSTR = 0x4 +SHF_MERGE = 0x10 +SHF_STRINGS = 0x20 +SHF_INFO_LINK = 0x40 +SHF_LINK_ORDER = 0x80 +SHF_OS_NONCONFORMING = 0x100 +SHF_GROUP = 0x200 +SHF_TLS = 0x400 + +R_MIPS_32 = 2 +R_MIPS_26 = 4 +R_MIPS_HI16 = 5 +R_MIPS_LO16 = 6 + +MIPS_DEBUG_ST_STATIC = 2 +MIPS_DEBUG_ST_STATIC_PROC = 14 + + +class ElfFormat: + def __init__(self, is_big_endian): + self.is_big_endian = is_big_endian + self.struct_char = ">" if is_big_endian else "<" + + def pack(self, fmt, *args): + return struct.pack(self.struct_char + fmt, *args) + + def unpack(self, fmt, data): + return struct.unpack(self.struct_char + fmt, data) + + +class ElfHeader: + """ + typedef struct { + unsigned char e_ident[EI_NIDENT]; + Elf32_Half e_type; + Elf32_Half e_machine; + Elf32_Word e_version; + Elf32_Addr e_entry; + Elf32_Off e_phoff; + Elf32_Off e_shoff; + Elf32_Word e_flags; + Elf32_Half e_ehsize; + Elf32_Half e_phentsize; + Elf32_Half e_phnum; + Elf32_Half e_shentsize; + Elf32_Half e_shnum; + Elf32_Half e_shstrndx; + } Elf32_Ehdr; + """ + + def __init__(self, data): + self.e_ident = data[:EI_NIDENT] + assert self.e_ident[EI_CLASS] == 1 # 32-bit + self.fmt = ElfFormat(is_big_endian=(self.e_ident[EI_DATA] == 2)) + self.e_type, self.e_machine, self.e_version, self.e_entry, self.e_phoff, self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize, self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx = self.fmt.unpack('HHIIIIIHHHHHH', data[EI_NIDENT:]) + assert self.e_type == 1 # relocatable + assert self.e_machine == 8 # MIPS I Architecture + assert self.e_phoff == 0 # no program header + assert self.e_shoff != 0 # section header + assert self.e_shstrndx != SHN_UNDEF + + def to_bin(self): + return self.e_ident + self.fmt.pack('HHIIIIIHHHHHH', self.e_type, + self.e_machine, self.e_version, self.e_entry, self.e_phoff, + self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize, + self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx) + + +class Symbol: + """ + typedef struct { + Elf32_Word st_name; + Elf32_Addr st_value; + Elf32_Word st_size; + unsigned char st_info; + unsigned char st_other; + Elf32_Half st_shndx; + } Elf32_Sym; + """ + + def __init__(self, fmt, data, strtab, name=None): + self.fmt = fmt + self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx = fmt.unpack('IIIBBH', data) + assert self.st_shndx != SHN_XINDEX, "too many sections (SHN_XINDEX not supported)" + self.bind = st_info >> 4 + self.type = st_info & 15 + self.name = name if name is not None else strtab.lookup_str(self.st_name) + self.visibility = self.st_other & 3 + + @staticmethod + def from_parts(fmt, st_name, st_value, st_size, st_info, st_other, st_shndx, strtab, name): + header = fmt.pack('IIIBBH', st_name, st_value, st_size, st_info, st_other, st_shndx) + return Symbol(fmt, header, strtab, name) + + def to_bin(self): + st_info = (self.bind << 4) | self.type + return self.fmt.pack('IIIBBH', self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx) + + +class Relocation: + def __init__(self, fmt, data, sh_type): + self.fmt = fmt + self.sh_type = sh_type + if sh_type == SHT_REL: + self.r_offset, self.r_info = fmt.unpack('II', data) + else: + self.r_offset, self.r_info, self.r_addend = fmt.unpack('III', data) + self.sym_index = self.r_info >> 8 + self.rel_type = self.r_info & 0xff + + def to_bin(self): + self.r_info = (self.sym_index << 8) | self.rel_type + if self.sh_type == SHT_REL: + return self.fmt.pack('II', self.r_offset, self.r_info) + else: + return self.fmt.pack('III', self.r_offset, self.r_info, self.r_addend) + + +class Section: + """ + typedef struct { + Elf32_Word sh_name; + Elf32_Word sh_type; + Elf32_Word sh_flags; + Elf32_Addr sh_addr; + Elf32_Off sh_offset; + Elf32_Word sh_size; + Elf32_Word sh_link; + Elf32_Word sh_info; + Elf32_Word sh_addralign; + Elf32_Word sh_entsize; + } Elf32_Shdr; + """ + + def __init__(self, fmt, header, data, index): + self.fmt = fmt + self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize = fmt.unpack('IIIIIIIIII', header) + assert not self.sh_flags & SHF_LINK_ORDER + if self.sh_entsize != 0: + assert self.sh_size % self.sh_entsize == 0 + if self.sh_type == SHT_NOBITS: + self.data = b'' + else: + self.data = data[self.sh_offset:self.sh_offset + self.sh_size] + self.index = index + self.relocated_by = [] + + @staticmethod + def from_parts(fmt, sh_name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data, index): + header = fmt.pack('IIIIIIIIII', sh_name, sh_type, sh_flags, 0, 0, len(data), sh_link, sh_info, sh_addralign, sh_entsize) + return Section(fmt, header, data, index) + + def lookup_str(self, index): + assert self.sh_type == SHT_STRTAB + to = self.data.find(b'\0', index) + assert to != -1 + return self.data[index:to].decode('latin1') + + def add_str(self, string): + assert self.sh_type == SHT_STRTAB + ret = len(self.data) + self.data += string.encode('latin1') + b'\0' + return ret + + def is_rel(self): + return self.sh_type == SHT_REL or self.sh_type == SHT_RELA + + def header_to_bin(self): + if self.sh_type != SHT_NOBITS: + self.sh_size = len(self.data) + return self.fmt.pack('IIIIIIIIII', self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize) + + def late_init(self, sections): + if self.sh_type == SHT_SYMTAB: + self.init_symbols(sections) + elif self.is_rel(): + self.rel_target = sections[self.sh_info] + self.rel_target.relocated_by.append(self) + self.init_relocs() + + def find_symbol(self, name): + assert self.sh_type == SHT_SYMTAB + for s in self.symbol_entries: + if s.name == name: + return (s.st_shndx, s.st_value) + return None + + def find_symbol_in_section(self, name, section): + pos = self.find_symbol(name) + assert pos is not None + assert pos[0] == section.index + return pos[1] + + def init_symbols(self, sections): + assert self.sh_type == SHT_SYMTAB + assert self.sh_entsize == 16 + self.strtab = sections[self.sh_link] + entries = [] + for i in range(0, self.sh_size, self.sh_entsize): + entries.append(Symbol(self.fmt, self.data[i:i+self.sh_entsize], self.strtab)) + self.symbol_entries = entries + + def init_relocs(self): + assert self.is_rel() + entries = [] + for i in range(0, self.sh_size, self.sh_entsize): + entries.append(Relocation(self.fmt, self.data[i:i+self.sh_entsize], self.sh_type)) + self.relocations = entries + + def local_symbols(self): + assert self.sh_type == SHT_SYMTAB + return self.symbol_entries[:self.sh_info] + + def global_symbols(self): + assert self.sh_type == SHT_SYMTAB + return self.symbol_entries[self.sh_info:] + + def relocate_mdebug(self, original_offset): + assert self.sh_type == SHT_MIPS_DEBUG + new_data = bytearray(self.data) + shift_by = self.sh_offset - original_offset + + # Update the file-relative offsets in the Symbolic HDRR + hdrr_magic, hdrr_vstamp, hdrr_ilineMax, hdrr_cbLine, \ + hdrr_cbLineOffset, hdrr_idnMax, hdrr_cbDnOffset, hdrr_ipdMax, \ + hdrr_cbPdOffset, hdrr_isymMax, hdrr_cbSymOffset, hdrr_ioptMax, \ + hdrr_cbOptOffset, hdrr_iauxMax, hdrr_cbAuxOffset, hdrr_issMax, \ + hdrr_cbSsOffset, hdrr_issExtMax, hdrr_cbSsExtOffset, hdrr_ifdMax, \ + hdrr_cbFdOffset, hdrr_crfd, hdrr_cbRfdOffset, hdrr_iextMax, \ + hdrr_cbExtOffset = self.fmt.unpack("HHIIIIIIIIIIIIIIIIIIIIIII", self.data[0:0x60]) + + assert hdrr_magic == 0x7009, "Invalid magic value for .mdebug symbolic header" + + hdrr_cbLineOffset += shift_by + hdrr_cbDnOffset += shift_by + hdrr_cbPdOffset += shift_by + hdrr_cbSymOffset += shift_by + hdrr_cbOptOffset += shift_by + hdrr_cbAuxOffset += shift_by + hdrr_cbSsOffset += shift_by + hdrr_cbSsExtOffset += shift_by + hdrr_cbFdOffset += shift_by + hdrr_cbRfdOffset += shift_by + hdrr_cbExtOffset += shift_by + + new_data[0:0x60] = self.fmt.pack("HHIIIIIIIIIIIIIIIIIIIIIII", hdrr_magic, hdrr_vstamp, hdrr_ilineMax, hdrr_cbLine, \ + hdrr_cbLineOffset, hdrr_idnMax, hdrr_cbDnOffset, hdrr_ipdMax, \ + hdrr_cbPdOffset, hdrr_isymMax, hdrr_cbSymOffset, hdrr_ioptMax, \ + hdrr_cbOptOffset, hdrr_iauxMax, hdrr_cbAuxOffset, hdrr_issMax, \ + hdrr_cbSsOffset, hdrr_issExtMax, hdrr_cbSsExtOffset, hdrr_ifdMax, \ + hdrr_cbFdOffset, hdrr_crfd, hdrr_cbRfdOffset, hdrr_iextMax, \ + hdrr_cbExtOffset) + + self.data = bytes(new_data) + +class ElfFile: + def __init__(self, data): + self.data = data + assert data[:4] == b'\x7fELF', "not an ELF file" + + self.elf_header = ElfHeader(data[0:52]) + self.fmt = self.elf_header.fmt + + offset, size = self.elf_header.e_shoff, self.elf_header.e_shentsize + null_section = Section(self.fmt, data[offset:offset + size], data, 0) + num_sections = self.elf_header.e_shnum or null_section.sh_size + + self.sections = [null_section] + for i in range(1, num_sections): + ind = offset + i * size + self.sections.append(Section(self.fmt, data[ind:ind + size], data, i)) + + symtab = None + for s in self.sections: + if s.sh_type == SHT_SYMTAB: + assert not symtab + symtab = s + assert symtab is not None + self.symtab = symtab + + shstr = self.sections[self.elf_header.e_shstrndx] + for s in self.sections: + s.name = shstr.lookup_str(s.sh_name) + s.late_init(self.sections) + + def find_section(self, name): + for s in self.sections: + if s.name == name: + return s + return None + + def add_section(self, name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data): + shstr = self.sections[self.elf_header.e_shstrndx] + sh_name = shstr.add_str(name) + s = Section.from_parts(self.fmt, sh_name=sh_name, sh_type=sh_type, + sh_flags=sh_flags, sh_link=sh_link, sh_info=sh_info, + sh_addralign=sh_addralign, sh_entsize=sh_entsize, data=data, + index=len(self.sections)) + self.sections.append(s) + s.name = name + s.late_init(self.sections) + return s + + def drop_mdebug_gptab(self): + # We can only drop sections at the end, since otherwise section + # references might be wrong. Luckily, these sections typically are. + while self.sections[-1].sh_type in [SHT_MIPS_DEBUG, SHT_MIPS_GPTAB]: + self.sections.pop() + + def write(self, filename): + outfile = open(filename, 'wb') + outidx = 0 + def write_out(data): + nonlocal outidx + outfile.write(data) + outidx += len(data) + def pad_out(align): + if align and outidx % align: + write_out(b'\0' * (align - outidx % align)) + + self.elf_header.e_shnum = len(self.sections) + write_out(self.elf_header.to_bin()) + + for s in self.sections: + if s.sh_type != SHT_NOBITS and s.sh_type != SHT_NULL: + pad_out(s.sh_addralign) + old_offset = s.sh_offset + s.sh_offset = outidx + if s.sh_type == SHT_MIPS_DEBUG and s.sh_offset != old_offset: + # The .mdebug section has moved, relocate offsets + s.relocate_mdebug(old_offset) + write_out(s.data) + + pad_out(4) + self.elf_header.e_shoff = outidx + for s in self.sections: + write_out(s.header_to_bin()) + + outfile.seek(0) + outfile.write(self.elf_header.to_bin()) + outfile.close() + + +def is_temp_name(name): + return name.startswith('_asmpp_') + + +# https://stackoverflow.com/a/241506 +def re_comment_replacer(match): + s = match.group(0) + if s[0] in "/#": + return " " + else: + return s + + +re_comment_or_string = re.compile( + r'#.*|/\*.*?\*/|"(?:\\.|[^\\"])*"' +) + + +class Failure(Exception): + def __init__(self, message): + self.message = message + + def __str__(self): + return self.message + + +class GlobalState: + def __init__(self, min_instr_count, skip_instr_count, use_jtbl_for_rodata, prelude_if_late_rodata, mips1, pascal): + # A value that hopefully never appears as a 32-bit rodata constant (or we + # miscompile late rodata). Increases by 1 in each step. + self.late_rodata_hex = 0xE0123456 + self.valuectr = 0 + self.namectr = 0 + self.min_instr_count = min_instr_count + self.skip_instr_count = skip_instr_count + self.use_jtbl_for_rodata = use_jtbl_for_rodata + self.prelude_if_late_rodata = prelude_if_late_rodata + self.mips1 = mips1 + self.pascal = pascal + + def next_late_rodata_hex(self): + dummy_bytes = struct.pack('>I', self.late_rodata_hex) + if (self.late_rodata_hex & 0xffff) == 0: + # Avoid lui + self.late_rodata_hex += 1 + self.late_rodata_hex += 1 + return dummy_bytes + + def make_name(self, cat): + self.namectr += 1 + return '_asmpp_{}{}'.format(cat, self.namectr) + + def func_prologue(self, name): + if self.pascal: + return " ".join([ + "procedure {}();".format(name), + "type", + " pi = ^integer;", + " pf = ^single;", + " pd = ^double;", + "var", + " vi: pi;", + " vf: pf;", + " vd: pd;", + "begin", + " vi := vi;", + " vf := vf;", + " vd := vd;", + ]) + else: + return 'void {}(void) {{'.format(name) + + def func_epilogue(self): + if self.pascal: + return "end;" + else: + return "}" + + def pascal_assignment(self, tp, val): + self.valuectr += 1 + address = (8 * self.valuectr) & 0x7FFF + return 'v{} := p{}({}); v{}^ := {};'.format(tp, tp, address, tp, val) + +Function = namedtuple('Function', ['text_glabels', 'asm_conts', 'late_rodata_dummy_bytes', 'jtbl_rodata_size', 'late_rodata_asm_conts', 'fn_desc', 'data']) + + +class GlobalAsmBlock: + def __init__(self, fn_desc): + self.fn_desc = fn_desc + self.cur_section = '.text' + self.asm_conts = [] + self.late_rodata_asm_conts = [] + self.late_rodata_alignment = 0 + self.late_rodata_alignment_from_content = False + self.text_glabels = [] + self.fn_section_sizes = { + '.text': 0, + '.data': 0, + '.bss': 0, + '.rodata': 0, + '.late_rodata': 0, + } + self.fn_ins_inds = [] + self.glued_line = '' + self.num_lines = 0 + + def fail(self, message, line=None): + context = self.fn_desc + if line: + context += ", at line \"" + line + "\"" + raise Failure(message + "\nwithin " + context) + + def count_quoted_size(self, line, z, real_line, output_enc): + line = line.encode(output_enc).decode('latin1') + in_quote = False + has_comma = True + num_parts = 0 + ret = 0 + i = 0 + digits = "0123456789" # 0-7 would be more sane, but this matches GNU as + while i < len(line): + c = line[i] + i += 1 + if not in_quote: + if c == '"': + in_quote = True + if z and not has_comma: + self.fail(".asciiz with glued strings is not supported due to GNU as version diffs") + num_parts += 1 + elif c == ',': + has_comma = True + else: + if c == '"': + in_quote = False + has_comma = False + continue + ret += 1 + if c != '\\': + continue + if i == len(line): + self.fail("backslash at end of line not supported", real_line) + c = line[i] + i += 1 + # (if c is in "bfnrtv", we have a real escaped literal) + if c == 'x': + # hex literal, consume any number of hex chars, possibly none + while i < len(line) and line[i] in digits + "abcdefABCDEF": + i += 1 + elif c in digits: + # octal literal, consume up to two more digits + it = 0 + while i < len(line) and line[i] in digits and it < 2: + i += 1 + it += 1 + + if in_quote: + self.fail("unterminated string literal", real_line) + if num_parts == 0: + self.fail(".ascii with no string", real_line) + return ret + num_parts if z else ret + + def align2(self): + while self.fn_section_sizes[self.cur_section] % 2 != 0: + self.fn_section_sizes[self.cur_section] += 1 + + def align4(self): + while self.fn_section_sizes[self.cur_section] % 4 != 0: + self.fn_section_sizes[self.cur_section] += 1 + + def add_sized(self, size, line): + if self.cur_section in ['.text', '.late_rodata']: + if size % 4 != 0: + self.fail("size must be a multiple of 4", line) + if size < 0: + self.fail("size cannot be negative", line) + self.fn_section_sizes[self.cur_section] += size + if self.cur_section == '.text': + if not self.text_glabels: + self.fail(".text block without an initial glabel", line) + self.fn_ins_inds.append((self.num_lines - 1, size // 4)) + + def process_line(self, line, output_enc): + self.num_lines += 1 + if line.endswith('\\'): + self.glued_line += line[:-1] + return + line = self.glued_line + line + self.glued_line = '' + + real_line = line + line = re.sub(re_comment_or_string, re_comment_replacer, line) + line = line.strip() + line = re.sub(r'^[a-zA-Z0-9_]+:\s*', '', line) + changed_section = False + emitting_double = False + if line.startswith('glabel ') and self.cur_section == '.text': + self.text_glabels.append(line.split()[1]) + if not line: + pass # empty line + elif line.startswith('glabel ') or line.startswith('dlabel ') or line.startswith('endlabel ') or (' ' not in line and line.endswith(':')): + pass # label + elif line.startswith('.section') or line in ['.text', '.data', '.rdata', '.rodata', '.bss', '.late_rodata']: + # section change + self.cur_section = '.rodata' if line == '.rdata' else line.split(',')[0].split()[-1] + if self.cur_section not in ['.data', '.text', '.rodata', '.late_rodata', '.bss']: + self.fail("unrecognized .section directive", real_line) + changed_section = True + elif line.startswith('.late_rodata_alignment'): + if self.cur_section != '.late_rodata': + self.fail(".late_rodata_alignment must occur within .late_rodata section", real_line) + value = int(line.split()[1]) + if value not in [4, 8]: + self.fail(".late_rodata_alignment argument must be 4 or 8", real_line) + if self.late_rodata_alignment and self.late_rodata_alignment != value: + self.fail(".late_rodata_alignment alignment assumption conflicts with earlier .double directive. Make sure to provide explicit alignment padding.") + self.late_rodata_alignment = value + changed_section = True + elif line.startswith('.incbin'): + self.add_sized(int(line.split(',')[-1].strip(), 0), real_line) + elif line.startswith('.word') or line.startswith('.gpword') or line.startswith('.float'): + self.align4() + self.add_sized(4 * len(line.split(',')), real_line) + elif line.startswith('.double'): + self.align4() + if self.cur_section == '.late_rodata': + align8 = self.fn_section_sizes[self.cur_section] % 8 + # Automatically set late_rodata_alignment, so the generated C code uses doubles. + # This gives us correct alignment for the transferred doubles even when the + # late_rodata_alignment is wrong, e.g. for non-matching compilation. + if not self.late_rodata_alignment: + self.late_rodata_alignment = 8 - align8 + self.late_rodata_alignment_from_content = True + elif self.late_rodata_alignment != 8 - align8: + if self.late_rodata_alignment_from_content: + self.fail("found two .double directives with different start addresses mod 8. Make sure to provide explicit alignment padding.", real_line) + else: + self.fail(".double at address that is not 0 mod 8 (based on .late_rodata_alignment assumption). Make sure to provide explicit alignment padding.", real_line) + self.add_sized(8 * len(line.split(',')), real_line) + emitting_double = True + elif line.startswith('.space'): + self.add_sized(int(line.split()[1], 0), real_line) + elif line.startswith('.balign') or line.startswith('.align'): + align = int(line.split()[1]) + if align != 4: + self.fail("only .balign 4 is supported", real_line) + self.align4() + elif line.startswith('.asci'): + z = (line.startswith('.asciz') or line.startswith('.asciiz')) + self.add_sized(self.count_quoted_size(line, z, real_line, output_enc), real_line) + elif line.startswith('.byte'): + self.add_sized(len(line.split(',')), real_line) + elif line.startswith('.half'): + self.align2() + self.add_sized(2*len(line.split(',')), real_line) + elif line.startswith('.'): + # .macro, ... + self.fail("asm directive not supported", real_line) + else: + # Unfortunately, macros are hard to support for .rodata -- + # we don't know how how space they will expand to before + # running the assembler, but we need that information to + # construct the C code. So if we need that we'll either + # need to run the assembler twice (at least in some rare + # cases), or change how this program is invoked. + # Similarly, we can't currently deal with pseudo-instructions + # that expand to several real instructions. + if self.cur_section != '.text': + self.fail("instruction or macro call in non-.text section? not supported", real_line) + self.add_sized(4, real_line) + if self.cur_section == '.late_rodata': + if not changed_section: + if emitting_double: + self.late_rodata_asm_conts.append(".align 0") + self.late_rodata_asm_conts.append(real_line) + if emitting_double: + self.late_rodata_asm_conts.append(".align 2") + else: + self.asm_conts.append(real_line) + + def finish(self, state): + src = [''] * (self.num_lines + 1) + late_rodata_dummy_bytes = [] + jtbl_rodata_size = 0 + late_rodata_fn_output = [] + + num_instr = self.fn_section_sizes['.text'] // 4 + + if self.fn_section_sizes['.late_rodata'] > 0: + # Generate late rodata by emitting unique float constants. + # This requires 3 instructions for each 4 bytes of rodata. + # If we know alignment, we can use doubles, which give 3 + # instructions for 8 bytes of rodata. + size = self.fn_section_sizes['.late_rodata'] // 4 + skip_next = False + needs_double = (self.late_rodata_alignment != 0) + extra_mips1_nop = False + if state.pascal: + jtbl_size = 9 if state.mips1 else 8 + jtbl_min_rodata_size = 2 + else: + jtbl_size = 11 if state.mips1 else 9 + jtbl_min_rodata_size = 5 + for i in range(size): + if skip_next: + skip_next = False + continue + # Jump tables give 9 instructions (11 with -mips1) for >= 5 words of rodata, + # and should be emitted when: + # - -O2 or -O2 -g3 are used, which give the right codegen + # - we have emitted our first .float/.double (to ensure that we find the + # created rodata in the binary) + # - we have emitted our first .double, if any (to ensure alignment of doubles + # in shifted rodata sections) + # - we have at least 5 words of rodata left to emit (otherwise IDO does not + # generate a jump table) + # - we have at least 10 more instructions to go in this function (otherwise our + # function size computation will be wrong since the delay slot goes unused) + if (not needs_double and state.use_jtbl_for_rodata and i >= 1 and + size - i >= jtbl_min_rodata_size and + num_instr - len(late_rodata_fn_output) >= jtbl_size + 1): + if state.pascal: + cases = " ".join("{}: ;".format(case) for case in range(size - i)) + line = "case 0 of " + cases + " otherwise end;" + else: + cases = " ".join("case {}:".format(case) for case in range(size - i)) + line = "switch (*(volatile int*)0) { " + cases + " ; }" + late_rodata_fn_output.append(line) + late_rodata_fn_output.extend([""] * (jtbl_size - 1)) + jtbl_rodata_size = (size - i) * 4 + extra_mips1_nop = i != 2 + break + dummy_bytes = state.next_late_rodata_hex() + late_rodata_dummy_bytes.append(dummy_bytes) + if self.late_rodata_alignment == 4 * ((i + 1) % 2 + 1) and i + 1 < size: + dummy_bytes2 = state.next_late_rodata_hex() + late_rodata_dummy_bytes.append(dummy_bytes2) + fval, = struct.unpack('>d', dummy_bytes + dummy_bytes2) + if state.pascal: + line = state.pascal_assignment('d', fval) + else: + line = '*(volatile double*)0 = {};'.format(fval) + late_rodata_fn_output.append(line) + skip_next = True + needs_double = False + if state.mips1: + # mips1 does not have ldc1/sdc1 + late_rodata_fn_output.append('') + late_rodata_fn_output.append('') + extra_mips1_nop = False + else: + fval, = struct.unpack('>f', dummy_bytes) + if state.pascal: + line = state.pascal_assignment('f', fval) + else: + line = '*(volatile float*)0 = {}f;'.format(fval) + late_rodata_fn_output.append(line) + extra_mips1_nop = True + late_rodata_fn_output.append('') + late_rodata_fn_output.append('') + if state.mips1 and extra_mips1_nop: + late_rodata_fn_output.append('') + + text_name = None + if self.fn_section_sizes['.text'] > 0 or late_rodata_fn_output: + text_name = state.make_name('func') + src[0] = state.func_prologue(text_name) + src[self.num_lines] = state.func_epilogue() + instr_count = self.fn_section_sizes['.text'] // 4 + if instr_count < state.min_instr_count: + self.fail("too short .text block") + tot_emitted = 0 + tot_skipped = 0 + fn_emitted = 0 + fn_skipped = 0 + skipping = True + rodata_stack = late_rodata_fn_output[::-1] + for (line, count) in self.fn_ins_inds: + for _ in range(count): + if (fn_emitted > MAX_FN_SIZE and instr_count - tot_emitted > state.min_instr_count and + (not rodata_stack or rodata_stack[-1])): + # Don't let functions become too large. When a function reaches 284 + # instructions, and -O2 -framepointer flags are passed, the IRIX + # compiler decides it is a great idea to start optimizing more. + # Also, Pascal cannot handle too large functions before it runs out + # of unique statements to write. + fn_emitted = 0 + fn_skipped = 0 + skipping = True + src[line] += (' ' + state.func_epilogue() + ' ' + + state.func_prologue(state.make_name('large_func')) + ' ') + if ( + skipping and + fn_skipped < state.skip_instr_count + + (state.prelude_if_late_rodata if rodata_stack else 0) + ): + fn_skipped += 1 + tot_skipped += 1 + else: + skipping = False + if rodata_stack: + src[line] += rodata_stack.pop() + elif state.pascal: + src[line] += state.pascal_assignment('i', '0') + else: + src[line] += '*(volatile int*)0 = 0;' + tot_emitted += 1 + fn_emitted += 1 + if rodata_stack: + size = len(late_rodata_fn_output) // 3 + available = instr_count - tot_skipped + self.fail( + "late rodata to text ratio is too high: {} / {} must be <= 1/3\n" + "add .late_rodata_alignment (4|8) to the .late_rodata " + "block to double the allowed ratio." + .format(size, available)) + + rodata_name = None + if self.fn_section_sizes['.rodata'] > 0: + if state.pascal: + self.fail(".rodata isn't supported with Pascal for now") + rodata_name = state.make_name('rodata') + src[self.num_lines] += ' const char {}[{}] = {{1}};'.format(rodata_name, self.fn_section_sizes['.rodata']) + + data_name = None + if self.fn_section_sizes['.data'] > 0: + data_name = state.make_name('data') + if state.pascal: + line = ' var {}: packed array[1..{}] of char := [otherwise: 0];'.format(data_name, self.fn_section_sizes['.data']) + else: + line = ' char {}[{}] = {{1}};'.format(data_name, self.fn_section_sizes['.data']) + src[self.num_lines] += line + + bss_name = None + if self.fn_section_sizes['.bss'] > 0: + if state.pascal: + self.fail(".bss isn't supported with Pascal") + bss_name = state.make_name('bss') + src[self.num_lines] += ' char {}[{}];'.format(bss_name, self.fn_section_sizes['.bss']) + + fn = Function( + text_glabels=self.text_glabels, + asm_conts=self.asm_conts, + late_rodata_dummy_bytes=late_rodata_dummy_bytes, + jtbl_rodata_size=jtbl_rodata_size, + late_rodata_asm_conts=self.late_rodata_asm_conts, + fn_desc=self.fn_desc, + data={ + '.text': (text_name, self.fn_section_sizes['.text']), + '.data': (data_name, self.fn_section_sizes['.data']), + '.rodata': (rodata_name, self.fn_section_sizes['.rodata']), + '.bss': (bss_name, self.fn_section_sizes['.bss']), + }) + return src, fn + +cutscene_data_regexpr = re.compile(r"CutsceneData (.|\n)*\[\] = {") +float_regexpr = re.compile(r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?f") + +def repl_float_hex(m): + return str(struct.unpack(">I", struct.pack(">f", float(m.group(0).strip().rstrip("f"))))[0]) + +Opts = namedtuple('Opts', ['opt', 'framepointer', 'mips1', 'kpic', 'pascal', 'input_enc', 'output_enc']) + +def parse_source(f, opts, out_dependencies, print_source=None): + if opts.opt in ['O1', 'O2']: + if opts.framepointer: + min_instr_count = 6 + skip_instr_count = 5 + else: + min_instr_count = 2 + skip_instr_count = 1 + elif opts.opt == 'O0': + if opts.framepointer: + min_instr_count = 8 + skip_instr_count = 8 + else: + min_instr_count = 4 + skip_instr_count = 4 + elif opts.opt == 'g': + if opts.framepointer: + min_instr_count = 7 + skip_instr_count = 7 + else: + min_instr_count = 4 + skip_instr_count = 4 + elif opts.opt == 'g3': + if opts.framepointer: + min_instr_count = 4 + skip_instr_count = 4 + else: + min_instr_count = 2 + skip_instr_count = 2 + else: + raise Failure("must pass one of -g, -O0, -O1, -O2, -O2 -g3") + prelude_if_late_rodata = 0 + if opts.kpic: + # Without optimizations, the PIC prelude always takes up 3 instructions. + # With optimizations, the prelude is optimized out if there's no late rodata. + if opts.opt in ('g3', 'O2'): + prelude_if_late_rodata = 3 + else: + min_instr_count += 3 + skip_instr_count += 3 + + use_jtbl_for_rodata = False + if opts.opt in ['O2', 'g3'] and not opts.framepointer and not opts.kpic: + use_jtbl_for_rodata = True + + state = GlobalState(min_instr_count, skip_instr_count, use_jtbl_for_rodata, prelude_if_late_rodata, opts.mips1, opts.pascal) + output_enc = opts.output_enc + + global_asm = None + asm_functions = [] + output_lines = [ + '#line 1 "' + f.name + '"' + ] + + is_cutscene_data = False + is_early_include = False + + for line_no, raw_line in enumerate(f, 1): + raw_line = raw_line.rstrip() + line = raw_line.lstrip() + + # Print exactly one output line per source line, to make compiler + # errors have correct line numbers. These will be overridden with + # reasonable content further down. + output_lines.append('') + + if global_asm is not None: + if line.startswith(')'): + src, fn = global_asm.finish(state) + for i, line2 in enumerate(src): + output_lines[start_index + i] = line2 + asm_functions.append(fn) + global_asm = None + else: + global_asm.process_line(raw_line, output_enc) + elif line in ['GLOBAL_ASM(', '#pragma GLOBAL_ASM(']: + global_asm = GlobalAsmBlock("GLOBAL_ASM block at line " + str(line_no)) + start_index = len(output_lines) + elif ((line.startswith('GLOBAL_ASM("') or line.startswith('#pragma GLOBAL_ASM("')) + and line.endswith('")')): + fname = line[line.index('(') + 2 : -2] + out_dependencies.append(fname) + global_asm = GlobalAsmBlock(fname) + with open(fname, encoding=opts.input_enc) as f: + for line2 in f: + global_asm.process_line(line2.rstrip(), output_enc) + src, fn = global_asm.finish(state) + output_lines[-1] = ''.join(src) + asm_functions.append(fn) + global_asm = None + elif line == '#pragma asmproc recurse': + # C includes qualified as + # #pragma asmproc recurse + # #include "file.c" + # will be processed recursively when encountered + is_early_include = True + elif is_early_include: + # Previous line was a #pragma asmproc recurse + is_early_include = False + if not line.startswith("#include "): + raise Failure("#pragma asmproc recurse must be followed by an #include ") + fpath = os.path.dirname(f.name) + fname = os.path.join(fpath, line[line.index(' ') + 2 : -1]) + out_dependencies.append(fname) + include_src = StringIO() + with open(fname, encoding=opts.input_enc) as include_file: + parse_source(include_file, opts, out_dependencies, include_src) + include_src.write('#line ' + str(line_no + 1) + ' "' + f.name + '"') + output_lines[-1] = include_src.getvalue() + include_src.close() + else: + # This is a hack to replace all floating-point numbers in an array of a particular type + # (in this case CutsceneData) with their corresponding IEEE-754 hexadecimal representation + if cutscene_data_regexpr.search(line) is not None: + is_cutscene_data = True + elif line.endswith("};"): + is_cutscene_data = False + if is_cutscene_data: + raw_line = re.sub(float_regexpr, repl_float_hex, raw_line) + output_lines[-1] = raw_line + + if print_source: + if isinstance(print_source, StringIO): + for line in output_lines: + print_source.write(line + '\n') + else: + newline_encoded = "\n".encode(output_enc) + for line in output_lines: + try: + line_encoded = line.encode(output_enc) + except UnicodeEncodeError: + print("Failed to encode a line to", output_enc) + print("The line:", line) + print("The line, utf-8-encoded:", line.encode("utf-8")) + raise + print_source.write(line_encoded) + print_source.write(newline_encoded) + print_source.flush() + + return asm_functions + +def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, drop_mdebug_gptab, convert_statics): + SECTIONS = ['.data', '.text', '.rodata', '.bss'] + + with open(objfile_name, 'rb') as f: + objfile = ElfFile(f.read()) + fmt = objfile.fmt + + prev_locs = { + '.text': 0, + '.data': 0, + '.rodata': 0, + '.bss': 0, + } + to_copy = { + '.text': [], + '.data': [], + '.rodata': [], + '.bss': [], + } + asm = [] + all_late_rodata_dummy_bytes = [] + all_jtbl_rodata_size = [] + late_rodata_asm = [] + late_rodata_source_name_start = None + late_rodata_source_name_end = None + + # Generate an assembly file with all the assembly we need to fill in. For + # simplicity we pad with nops/.space so that addresses match exactly, so we + # don't have to fix up relocations/symbol references. + all_text_glabels = set() + func_sizes = {} + for function in functions: + ifdefed = False + for sectype, (temp_name, size) in function.data.items(): + if temp_name is None: + continue + assert size > 0 + loc = objfile.symtab.find_symbol(temp_name) + if loc is None: + ifdefed = True + break + loc = loc[1] + prev_loc = prev_locs[sectype] + if loc < prev_loc: + # If the dummy C generates too little asm, and we have two + # consecutive GLOBAL_ASM blocks, we detect that error here. + # On the other hand, if it generates too much, we don't have + # a good way of discovering that error: it's indistinguishable + # from a static symbol occurring after the GLOBAL_ASM block. + raise Failure("Wrongly computed size for section {} (diff {}). This is an asm-processor bug!".format(sectype, prev_loc- loc)) + if loc != prev_loc: + asm.append('.section ' + sectype) + if sectype == '.text': + for i in range((loc - prev_loc) // 4): + asm.append('nop') + else: + asm.append('.space {}'.format(loc - prev_loc)) + to_copy[sectype].append((loc, size, temp_name, function.fn_desc)) + if function.text_glabels and sectype == '.text': + func_sizes[function.text_glabels[0]] = size + prev_locs[sectype] = loc + size + if not ifdefed: + all_text_glabels.update(function.text_glabels) + all_late_rodata_dummy_bytes.append(function.late_rodata_dummy_bytes) + all_jtbl_rodata_size.append(function.jtbl_rodata_size) + late_rodata_asm.append(function.late_rodata_asm_conts) + for sectype, (temp_name, size) in function.data.items(): + if temp_name is not None: + asm.append('.section ' + sectype) + asm.append('glabel ' + temp_name + '_asm_start') + asm.append('.text') + for line in function.asm_conts: + asm.append(line) + for sectype, (temp_name, size) in function.data.items(): + if temp_name is not None: + asm.append('.section ' + sectype) + asm.append('glabel ' + temp_name + '_asm_end') + if any(late_rodata_asm): + late_rodata_source_name_start = '_asmpp_late_rodata_start' + late_rodata_source_name_end = '_asmpp_late_rodata_end' + asm.append('.section .late_rodata') + # Put some padding at the start to avoid conflating symbols with + # references to the whole section. + asm.append('.word 0, 0') + asm.append('glabel {}'.format(late_rodata_source_name_start)) + for conts in late_rodata_asm: + asm.extend(conts) + asm.append('glabel {}'.format(late_rodata_source_name_end)) + + o_file = tempfile.NamedTemporaryFile(prefix='asm-processor', suffix='.o', delete=False) + o_name = o_file.name + o_file.close() + s_file = tempfile.NamedTemporaryFile(prefix='asm-processor', suffix='.s', delete=False) + s_name = s_file.name + try: + s_file.write(asm_prelude + b'\n') + for line in asm: + s_file.write(line.encode(output_enc) + b'\n') + s_file.close() + ret = os.system(assembler + " " + s_name + " -o " + o_name) + if ret != 0: + raise Failure("failed to assemble") + with open(o_name, 'rb') as f: + asm_objfile = ElfFile(f.read()) + + # Remove clutter from objdump output for tests, and make the tests + # portable by avoiding absolute paths. Outside of tests .mdebug is + # useful for showing source together with asm, though. + mdebug_section = objfile.find_section('.mdebug') + if drop_mdebug_gptab: + objfile.drop_mdebug_gptab() + + # Unify reginfo sections + target_reginfo = objfile.find_section('.reginfo') + if target_reginfo is not None: + source_reginfo_data = list(asm_objfile.find_section('.reginfo').data) + data = list(target_reginfo.data) + for i in range(20): + data[i] |= source_reginfo_data[i] + target_reginfo.data = bytes(data) + + # Move over section contents + modified_text_positions = set() + jtbl_rodata_positions = set() + last_rodata_pos = 0 + for sectype in SECTIONS: + if not to_copy[sectype]: + continue + source = asm_objfile.find_section(sectype) + assert source is not None, "didn't find source section: " + sectype + for (pos, count, temp_name, fn_desc) in to_copy[sectype]: + loc1 = asm_objfile.symtab.find_symbol_in_section(temp_name + '_asm_start', source) + loc2 = asm_objfile.symtab.find_symbol_in_section(temp_name + '_asm_end', source) + assert loc1 == pos, "assembly and C files don't line up for section " + sectype + ", " + fn_desc + if loc2 - loc1 != count: + raise Failure("incorrectly computed size for section " + sectype + ", " + fn_desc + ". If using .double, make sure to provide explicit alignment padding.") + if sectype == '.bss': + continue + target = objfile.find_section(sectype) + assert target is not None, "missing target section of type " + sectype + data = list(target.data) + for (pos, count, _, _) in to_copy[sectype]: + data[pos:pos + count] = source.data[pos:pos + count] + if sectype == '.text': + assert count % 4 == 0 + assert pos % 4 == 0 + for i in range(count // 4): + modified_text_positions.add(pos + 4 * i) + elif sectype == '.rodata': + last_rodata_pos = pos + count + target.data = bytes(data) + + # Move over late rodata. This is heuristic, sadly, since I can't think + # of another way of doing it. + moved_late_rodata = {} + if any(all_late_rodata_dummy_bytes) or any(all_jtbl_rodata_size): + source = asm_objfile.find_section('.late_rodata') + target = objfile.find_section('.rodata') + source_pos = asm_objfile.symtab.find_symbol_in_section(late_rodata_source_name_start, source) + source_end = asm_objfile.symtab.find_symbol_in_section(late_rodata_source_name_end, source) + if source_end - source_pos != sum(map(len, all_late_rodata_dummy_bytes)) * 4 + sum(all_jtbl_rodata_size): + raise Failure("computed wrong size of .late_rodata") + new_data = list(target.data) + for dummy_bytes_list, jtbl_rodata_size in zip(all_late_rodata_dummy_bytes, all_jtbl_rodata_size): + for index, dummy_bytes in enumerate(dummy_bytes_list): + if not fmt.is_big_endian: + dummy_bytes = dummy_bytes[::-1] + pos = target.data.index(dummy_bytes, last_rodata_pos) + # This check is nice, but makes time complexity worse for large files: + if SLOW_CHECKS and target.data.find(dummy_bytes, pos + 4) != -1: + raise Failure("multiple occurrences of late_rodata hex magic. Change asm-processor to use something better than 0xE0123456!") + if index == 0 and len(dummy_bytes_list) > 1 and target.data[pos+4:pos+8] == b'\0\0\0\0': + # Ugly hack to handle double alignment for non-matching builds. + # We were told by .late_rodata_alignment (or deduced from a .double) + # that a function's late_rodata started out 4 (mod 8), and emitted + # a float and then a double. But it was actually 0 (mod 8), so our + # double was moved by 4 bytes. To make them adjacent to keep jump + # tables correct, move the float by 4 bytes as well. + new_data[pos:pos+4] = b'\0\0\0\0' + pos += 4 + new_data[pos:pos+4] = source.data[source_pos:source_pos+4] + moved_late_rodata[source_pos] = pos + last_rodata_pos = pos + 4 + source_pos += 4 + if jtbl_rodata_size > 0: + assert dummy_bytes_list, "should always have dummy bytes before jtbl data" + pos = last_rodata_pos + new_data[pos : pos + jtbl_rodata_size] = \ + source.data[source_pos : source_pos + jtbl_rodata_size] + for i in range(0, jtbl_rodata_size, 4): + moved_late_rodata[source_pos + i] = pos + i + jtbl_rodata_positions.add(pos + i) + last_rodata_pos += jtbl_rodata_size + source_pos += jtbl_rodata_size + target.data = bytes(new_data) + + # Merge strtab data. + strtab_adj = len(objfile.symtab.strtab.data) + objfile.symtab.strtab.data += asm_objfile.symtab.strtab.data + + # Find relocated symbols + relocated_symbols = set() + for sectype in SECTIONS + ['.late_rodata']: + for obj in [asm_objfile, objfile]: + sec = obj.find_section(sectype) + if sec is None: + continue + for reltab in sec.relocated_by: + for rel in reltab.relocations: + relocated_symbols.add(obj.symtab.symbol_entries[rel.sym_index]) + + # Move over symbols, deleting the temporary function labels. + # Skip over new local symbols that aren't relocated against, to + # avoid conflicts. + empty_symbol = objfile.symtab.symbol_entries[0] + new_syms = [s for s in objfile.symtab.symbol_entries[1:] if not is_temp_name(s.name)] + + for i, s in enumerate(asm_objfile.symtab.symbol_entries): + is_local = (i < asm_objfile.symtab.sh_info) + if is_local and s not in relocated_symbols: + continue + if is_temp_name(s.name): + assert s not in relocated_symbols + continue + if s.st_shndx not in [SHN_UNDEF, SHN_ABS]: + section_name = asm_objfile.sections[s.st_shndx].name + target_section_name = section_name + if section_name == ".late_rodata": + target_section_name = ".rodata" + elif section_name not in SECTIONS: + raise Failure("generated assembly .o must only have symbols for .text, .data, .rodata, .late_rodata, ABS and UNDEF, but found " + section_name) + objfile_section = objfile.find_section(target_section_name) + if objfile_section is None: + raise Failure("generated assembly .o has section that real objfile lacks: " + target_section_name) + s.st_shndx = objfile_section.index + # glabel's aren't marked as functions, making objdump output confusing. Fix that. + if s.name in all_text_glabels: + s.type = STT_FUNC + if s.name in func_sizes: + s.st_size = func_sizes[s.name] + if section_name == '.late_rodata': + if s.st_value == 0: + # This must be a symbol corresponding to the whole .late_rodata + # section, being referred to from a relocation. + # Moving local symbols is tricky, because it requires fixing up + # lo16/hi16 relocation references to .late_rodata+<offset>. + # Just disallow it for now. + raise Failure("local symbols in .late_rodata are not allowed") + s.st_value = moved_late_rodata[s.st_value] + s.st_name += strtab_adj + new_syms.append(s) + make_statics_global = convert_statics in ("global", "global-with-filename") + + # Add static symbols from .mdebug, so they can be referred to from GLOBAL_ASM + if mdebug_section and convert_statics != "no": + strtab_index = len(objfile.symtab.strtab.data) + new_strtab_data = [] + ifd_max, cb_fd_offset = fmt.unpack('II', mdebug_section.data[18*4 : 20*4]) + cb_sym_offset, = fmt.unpack('I', mdebug_section.data[9*4 : 10*4]) + cb_ss_offset, = fmt.unpack('I', mdebug_section.data[15*4 : 16*4]) + for i in range(ifd_max): + offset = cb_fd_offset + 18*4*i + iss_base, _, isym_base, csym = fmt.unpack('IIII', objfile.data[offset + 2*4 : offset + 6*4]) + for j in range(csym): + offset2 = cb_sym_offset + 12 * (isym_base + j) + iss, value, st_sc_index = fmt.unpack('III', objfile.data[offset2 : offset2 + 12]) + st = (st_sc_index >> 26) + sc = (st_sc_index >> 21) & 0x1f + if st in [MIPS_DEBUG_ST_STATIC, MIPS_DEBUG_ST_STATIC_PROC]: + symbol_name_offset = cb_ss_offset + iss_base + iss + symbol_name_offset_end = objfile.data.find(b'\0', symbol_name_offset) + assert symbol_name_offset_end != -1 + symbol_name = objfile.data[symbol_name_offset : symbol_name_offset_end + 1] + emitted_symbol_name = symbol_name + if convert_statics == "global-with-filename": + # Change the emitted symbol name to include the filename, + # but don't let that affect deduplication logic. + emitted_symbol_name = objfile_name.encode("utf-8") + b":" + symbol_name + section_name = {1: '.text', 2: '.data', 3: '.bss', 15: '.rodata'}[sc] + section = objfile.find_section(section_name) + symtype = STT_FUNC if sc == 1 else STT_OBJECT + binding = STB_GLOBAL if make_statics_global else STB_LOCAL + sym = Symbol.from_parts( + fmt, + st_name=strtab_index, + st_value=value, + st_size=0, + st_info=(binding << 4 | symtype), + st_other=STV_DEFAULT, + st_shndx=section.index, + strtab=objfile.symtab.strtab, + name=symbol_name[:-1].decode('latin1')) + strtab_index += len(emitted_symbol_name) + new_strtab_data.append(emitted_symbol_name) + new_syms.append(sym) + objfile.symtab.strtab.data += b''.join(new_strtab_data) + + # Get rid of duplicate symbols, favoring ones that are not UNDEF. + # Skip this for unnamed local symbols though. + new_syms.sort(key=lambda s: 0 if s.st_shndx != SHN_UNDEF else 1) + old_syms = [] + newer_syms = [] + name_to_sym = {} + for s in new_syms: + if s.name == "_gp_disp": + s.type = STT_OBJECT + if s.bind == STB_LOCAL and s.st_shndx == SHN_UNDEF: + raise Failure("local symbol \"" + s.name + "\" is undefined") + if not s.name: + if s.bind != STB_LOCAL: + raise Failure("global symbol with no name") + newer_syms.append(s) + else: + existing = name_to_sym.get(s.name) + if not existing: + name_to_sym[s.name] = s + newer_syms.append(s) + elif s.st_shndx != SHN_UNDEF: + raise Failure("symbol \"" + s.name + "\" defined twice") + else: + s.replace_by = existing + old_syms.append(s) + new_syms = newer_syms + + # Put local symbols in front, with the initial dummy entry first, and + # _gp_disp at the end if it exists. + new_syms.insert(0, empty_symbol) + new_syms.sort(key=lambda s: (s.bind != STB_LOCAL, s.name == "_gp_disp")) + num_local_syms = sum(1 for s in new_syms if s.bind == STB_LOCAL) + + for i, s in enumerate(new_syms): + s.new_index = i + for s in old_syms: + s.new_index = s.replace_by.new_index + objfile.symtab.data = b''.join(s.to_bin() for s in new_syms) + objfile.symtab.sh_info = num_local_syms + + # Fix up relocation symbol references + for sectype in SECTIONS: + target = objfile.find_section(sectype) + + if target is not None: + # fixup relocation symbol indices, since we butchered them above + for reltab in target.relocated_by: + nrels = [] + for rel in reltab.relocations: + if (sectype == '.text' and rel.r_offset in modified_text_positions or + sectype == '.rodata' and rel.r_offset in jtbl_rodata_positions): + # don't include relocations for late_rodata dummy code + continue + rel.sym_index = objfile.symtab.symbol_entries[rel.sym_index].new_index + nrels.append(rel) + reltab.relocations = nrels + reltab.data = b''.join(rel.to_bin() for rel in nrels) + + # Move over relocations + for sectype in SECTIONS + ['.late_rodata']: + source = asm_objfile.find_section(sectype) + if source is None or not source.data: + continue + + target_sectype = '.rodata' if sectype == '.late_rodata' else sectype + target = objfile.find_section(target_sectype) + assert target is not None, target_sectype + target_reltab = objfile.find_section('.rel' + target_sectype) + target_reltaba = objfile.find_section('.rela' + target_sectype) + for reltab in source.relocated_by: + for rel in reltab.relocations: + rel.sym_index = asm_objfile.symtab.symbol_entries[rel.sym_index].new_index + if sectype == '.late_rodata': + rel.r_offset = moved_late_rodata[rel.r_offset] + new_data = b''.join(rel.to_bin() for rel in reltab.relocations) + if reltab.sh_type == SHT_REL: + if not target_reltab: + target_reltab = objfile.add_section('.rel' + target_sectype, + sh_type=SHT_REL, sh_flags=0, + sh_link=objfile.symtab.index, sh_info=target.index, + sh_addralign=4, sh_entsize=8, data=b'') + target_reltab.data += new_data + else: + if not target_reltaba: + target_reltaba = objfile.add_section('.rela' + target_sectype, + sh_type=SHT_RELA, sh_flags=0, + sh_link=objfile.symtab.index, sh_info=target.index, + sh_addralign=4, sh_entsize=12, data=b'') + target_reltaba.data += new_data + + objfile.write(objfile_name) + finally: + s_file.close() + os.remove(s_name) + try: + os.remove(o_name) + except: + pass + +def run_wrapped(argv, outfile, functions): + parser = argparse.ArgumentParser(description="Pre-process .c files and post-process .o files to enable embedding assembly into C.") + parser.add_argument('filename', help="path to .c code") + parser.add_argument('--post-process', dest='objfile', help="path to .o file to post-process") + parser.add_argument('--assembler', dest='assembler', help="assembler command (e.g. \"mips-linux-gnu-as -march=vr4300 -mabi=32\")") + parser.add_argument('--asm-prelude', dest='asm_prelude', help="path to a file containing a prelude to the assembly file (with .set and .macro directives, e.g.)") + parser.add_argument('--input-enc', default='latin1', help="input encoding (default: %(default)s)") + parser.add_argument('--output-enc', default='latin1', help="output encoding (default: %(default)s)") + parser.add_argument('--drop-mdebug-gptab', dest='drop_mdebug_gptab', action='store_true', help="drop mdebug and gptab sections") + parser.add_argument('--convert-statics', dest='convert_statics', choices=["no", "local", "global", "global-with-filename"], default="local", help="change static symbol visibility (default: %(default)s)") + parser.add_argument('--force', dest='force', action='store_true', help="force processing of files without GLOBAL_ASM blocks") + parser.add_argument('-framepointer', dest='framepointer', action='store_true') + parser.add_argument('-mips1', dest='mips1', action='store_true') + parser.add_argument('-g3', dest='g3', action='store_true') + parser.add_argument('-KPIC', dest='kpic', action='store_true') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-O0', dest='opt', action='store_const', const='O0') + group.add_argument('-O1', dest='opt', action='store_const', const='O1') + group.add_argument('-O2', dest='opt', action='store_const', const='O2') + group.add_argument('-g', dest='opt', action='store_const', const='g') + args = parser.parse_args(argv) + opt = args.opt + pascal = any(args.filename.endswith(ext) for ext in (".p", ".pas", ".pp")) + if args.g3: + if opt != 'O2': + raise Failure("-g3 is only supported together with -O2") + opt = 'g3' + if args.mips1 and (opt not in ('O1', 'O2') or args.framepointer): + raise Failure("-mips1 is only supported together with -O1 or -O2") + if pascal and opt not in ('O1', 'O2', 'g3'): + raise Failure("Pascal is only supported together with -O1, -O2 or -O2 -g3") + opts = Opts(opt, args.framepointer, args.mips1, args.kpic, pascal, args.input_enc, args.output_enc) + + if args.objfile is None: + with open(args.filename, encoding=args.input_enc) as f: + deps = [] + functions = parse_source(f, opts, out_dependencies=deps, print_source=outfile) + return functions, deps + else: + if args.assembler is None: + raise Failure("must pass assembler command") + if functions is None: + with open(args.filename, encoding=args.input_enc) as f: + functions = parse_source(f, opts, out_dependencies=[]) + if not functions and not args.force: + return + asm_prelude = b'' + if args.asm_prelude: + with open(args.asm_prelude, 'rb') as f: + asm_prelude = f.read() + fixup_objfile(args.objfile, functions, asm_prelude, args.assembler, args.output_enc, args.drop_mdebug_gptab, args.convert_statics) + +def run(argv, outfile=sys.stdout.buffer, functions=None): + try: + return run_wrapped(argv, outfile, functions) + except Failure as e: + print("Error:", e, file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + run(sys.argv[1:]) diff --git a/tools/asm-processor/build.py b/tools/asm-processor/build.py new file mode 100644 index 0000000..efbaade --- /dev/null +++ b/tools/asm-processor/build.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path +import shlex +import subprocess +import tempfile +import uuid +import asm_processor + +# Boolean for debugging purposes +# Preprocessed files are temporary, set to True to keep a copy +keep_preprocessed_files = False + +dir_path = Path(__file__).resolve().parent +asm_prelude_path = dir_path / "prelude.inc" + +all_args = sys.argv[1:] +sep0 = next(index for index, arg in enumerate(all_args) if not arg.startswith("-")) +sep1 = all_args.index("--") +sep2 = all_args.index("--", sep1 + 1) + +asmproc_flags = all_args[:sep0] +compiler = all_args[sep0:sep1] + +assembler_args = all_args[sep1 + 1 : sep2] +assembler_sh = " ".join(shlex.quote(x) for x in assembler_args) + + +compile_args = all_args[sep2 + 1 :] + +in_file = Path(compile_args[-1]) +del compile_args[-1] + +out_ind = compile_args.index("-o") +out_file = Path(compile_args[out_ind + 1]) +del compile_args[out_ind + 1] +del compile_args[out_ind] + + +in_dir = in_file.resolve().parent +opt_flags = [ + x for x in compile_args if x in {"-g3", "-g", "-O0", "-O1", "-O2", "-framepointer", "-KPIC"} +] +if "-mips2" not in compile_args: + opt_flags.append("-mips1") + +asmproc_flags += opt_flags + [str(in_file)] + +# Drop .mdebug and .gptab sections from resulting binaries. This makes +# resulting .o files much smaller and speeds up builds, but loses line +# number debug data. +# asmproc_flags += ["--drop-mdebug-gptab"] + +# Convert encoding before compiling. +# asmproc_flags += ["--input-enc", "utf-8", "--output-enc", "euc-jp"] + +with tempfile.TemporaryDirectory(prefix="asm_processor") as tmpdirname: + tmpdir_path = Path(tmpdirname) + preprocessed_filename = "preprocessed_" + uuid.uuid4().hex + in_file.suffix + preprocessed_path = tmpdir_path / preprocessed_filename + + with preprocessed_path.open("wb") as f: + functions, deps = asm_processor.run(asmproc_flags, outfile=f) + + if keep_preprocessed_files: + import shutil + + keep_output_dir = Path("./asm_processor_preprocessed") + keep_output_dir.mkdir(parents=True, exist_ok=True) + + shutil.copy( + preprocessed_path, + keep_output_dir / (in_file.stem + "_" + preprocessed_filename), + ) + + compile_cmdline = ( + compiler + + compile_args + + ["-I", str(in_dir), "-o", str(out_file), str(preprocessed_path)] + ) + + try: + subprocess.check_call(compile_cmdline) + except subprocess.CalledProcessError as e: + print("Failed to compile file " + str(in_file) + ". Command line:") + print() + print(" ".join(shlex.quote(x) for x in compile_cmdline)) + print() + sys.exit(55) + + asm_processor.run( + asmproc_flags + + [ + "--post-process", + str(out_file), + "--assembler", + assembler_sh, + "--asm-prelude", + str(asm_prelude_path), + ], + functions=functions, + ) + + deps_file = out_file.with_suffix(".asmproc.d") + if deps: + with deps_file.open("w") as f: + f.write(str(out_file) + ": " + " \\\n ".join(deps) + "\n") + for dep in deps: + f.write("\n" + dep + ":\n") + else: + try: + deps_file.unlink() + except OSError: + pass diff --git a/tools/asm-processor/compile-test.sh b/tools/asm-processor/compile-test.sh new file mode 100755 index 0000000..6551662 --- /dev/null +++ b/tools/asm-processor/compile-test.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -o pipefail +INPUT="$1" +OUTPUT="${INPUT%.*}.o" + +rm -f "$OUTPUT" + +CC="$MIPS_CC" # ido 7.1 via recomp or qemu-irix +AS="mips-linux-gnu-as" +ASFLAGS="-march=vr4300 -mabi=32" +OPTFLAGS=$(grep 'COMPILE-FLAGS: ' $INPUT | sed 's#^.*COMPILE-FLAGS: ##' | sed 's#}$##') +ASMPFLAGS=$(grep 'ASMP-FLAGS: ' $INPUT | sed 's#^.*ASMP-FLAGS: ##' | sed 's#}$##') +ISET=$(grep 'COMPILE-ISET: ' $INPUT | sed 's#^.*COMPILE-ISET: ##' | sed 's#}$##') +if [[ -z "$OPTFLAGS" ]]; then + OPTFLAGS="-g" +fi +CFLAGS="-Wab,-r4300_mul -G 0 -Xcpluscomm -fullwarn -wlint -woff 819,820,852,821 -signed -c" +if [[ -z "$ISET" ]]; then + CFLAGS="$CFLAGS -mips2" +fi +if [[ "$OPTFLAGS" != *-KPIC* ]]; then + CFLAGS="$CFLAGS -non_shared" +fi + +set -e +python3 build.py --drop-mdebug-gptab $ASMPFLAGS $CC -- $AS $ASFLAGS -- $CFLAGS $OPTFLAGS $ISET -o "$OUTPUT" "$INPUT" diff --git a/tools/asm-processor/prelude.inc b/tools/asm-processor/prelude.inc new file mode 100644 index 0000000..3e58ff1 --- /dev/null +++ b/tools/asm-processor/prelude.inc @@ -0,0 +1,43 @@ +.set noat +.set noreorder +.set gp=64 +.macro glabel label + .global \label + \label: +.endm + + +# Float register aliases (o32 ABI, odd ones are rarely used) + +.set $fv0, $f0 +.set $fv0f, $f1 +.set $fv1, $f2 +.set $fv1f, $f3 +.set $ft0, $f4 +.set $ft0f, $f5 +.set $ft1, $f6 +.set $ft1f, $f7 +.set $ft2, $f8 +.set $ft2f, $f9 +.set $ft3, $f10 +.set $ft3f, $f11 +.set $fa0, $f12 +.set $fa0f, $f13 +.set $fa1, $f14 +.set $fa1f, $f15 +.set $ft4, $f16 +.set $ft4f, $f17 +.set $ft5, $f18 +.set $ft5f, $f19 +.set $fs0, $f20 +.set $fs0f, $f21 +.set $fs1, $f22 +.set $fs1f, $f23 +.set $fs2, $f24 +.set $fs2f, $f25 +.set $fs3, $f26 +.set $fs3f, $f27 +.set $fs4, $f28 +.set $fs4f, $f29 +.set $fs5, $f30 +.set $fs5f, $f31 diff --git a/tools/asm-processor/run-tests.sh b/tools/asm-processor/run-tests.sh new file mode 100755 index 0000000..5cae0a7 --- /dev/null +++ b/tools/asm-processor/run-tests.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +for A in tests/*.c tests/*.p; do + OBJDUMPFLAGS=-srt + echo $A + ./compile-test.sh "$A" && mips-linux-gnu-objdump $OBJDUMPFLAGS "${A%.*}.o" | diff - "${A%.*}.objdump" || echo FAIL "$A" +done diff --git a/tools/asm-processor/tests/ascii.c b/tools/asm-processor/tests/ascii.c new file mode 100644 index 0000000..bb27d25 --- /dev/null +++ b/tools/asm-processor/tests/ascii.c @@ -0,0 +1,19 @@ +void foo(void) { "abcdef"; } + +GLOBAL_ASM( +.rdata + .ascii "AB" + .ascii "CD", "EF" + .ascii "GH\n\n\n\0\11\222\3333\44444\x1234567\n\nIJK" +) + +void bar(void) { "hello"; } + +GLOBAL_ASM( +.rdata + .asciiz "12" + .asciiz "34", "56" + .asciiz "78\n\n\n\0\11\222\3333\44444\x1234567\n\n9A" +) + +void baz(void) { "ghijkl"; } diff --git a/tools/asm-processor/tests/ascii.objdump b/tools/asm-processor/tests/ascii.objdump new file mode 100644 index 0000000..bf42638 --- /dev/null +++ b/tools/asm-processor/tests/ascii.objdump @@ -0,0 +1,29 @@ + +tests/ascii.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000030 .text +00000000 l d .rodata 00000050 .rodata +00000000 g F .text 00000010 foo +00000010 g F .text 00000010 bar +00000020 g F .text 00000010 baz + + +Contents of section .text: + 0000 03e00008 00000000 03e00008 00000000 ................ + 0010 03e00008 00000000 03e00008 00000000 ................ + 0020 03e00008 00000000 03e00008 00000000 ................ +Contents of section .rodata: + 0000 61626364 65660000 41424344 45464748 abcdef..ABCDEFGH + 0010 0a0a0a00 0992db33 24343467 0a0a494a .......3$44g..IJ + 0020 4b000000 68656c6c 6f000000 31320033 K...hello...12.3 + 0030 34003536 0037380a 0a0a0009 92db3324 4.56.78.......3$ + 0040 3434670a 0a394100 6768696a 6b6c0000 44g..9A.ghijkl.. +Contents of section .options: + 0000 01200000 00000000 80000000 00000000 . .............. + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000000 00000000 00000000 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/comments.c b/tools/asm-processor/tests/comments.c new file mode 100644 index 0000000..4c93669 --- /dev/null +++ b/tools/asm-processor/tests/comments.c @@ -0,0 +1,6 @@ +const char before[] = "^"; +GLOBAL_ASM( +.rdata +.asciz "aaaa /* bbbb */ # cccc", /**//**//**//**/ /*/ "xxxx" /*/ /* dddd " eeee */ "# ffff" # gggg "hhhh" /* iiii */ +) +const char after[] = "$"; diff --git a/tools/asm-processor/tests/comments.objdump b/tools/asm-processor/tests/comments.objdump new file mode 100644 index 0000000..246df20 --- /dev/null +++ b/tools/asm-processor/tests/comments.objdump @@ -0,0 +1,21 @@ + +tests/comments.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .rodata 00000030 .rodata +00000000 g O .rodata 00000002 before +00000024 g O .rodata 00000002 after + + +Contents of section .rodata: + 0000 5e000000 61616161 202f2a20 62626262 ^...aaaa /* bbbb + 0010 202a2f20 23206363 63630023 20666666 */ # cccc.# fff + 0020 66000000 24000000 00000000 00000000 f...$........... +Contents of section .options: + 0000 01200000 00000000 00000000 00000000 . .............. + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/force.c b/tools/asm-processor/tests/force.c new file mode 100644 index 0000000..03b630c --- /dev/null +++ b/tools/asm-processor/tests/force.c @@ -0,0 +1,17 @@ +// COMPILE-FLAGS: -O2 +// ASMP-FLAGS: --convert-statics=global-with-filename --force +static int xtext(int a, int b, int c); +const int rodata1[] = {1}; +static const int rodata2[] = {2}; +int data1[] = {3}; +static int data2[] = {4}; +int bss1; +static int bss2; + +static int xtext(int a, int b, int c) { + return 1; +} + +void baz(void) { + xtext(bss2, rodata2[0], data2[0]); +} diff --git a/tools/asm-processor/tests/force.objdump b/tools/asm-processor/tests/force.objdump new file mode 100644 index 0000000..7ef7024 --- /dev/null +++ b/tools/asm-processor/tests/force.objdump @@ -0,0 +1,47 @@ + +tests/force.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000050 .text +00000000 l d .rodata 00000010 .rodata +00000000 l d .data 00000010 .data +00000000 l d .bss 00000010 .bss +00000000 g O .rodata 00000004 rodata1 +00000000 g O .data 00000004 data1 +00000000 g O .bss 00000004 bss1 +00000014 g F .text 00000034 baz +00000004 g O .rodata 00000000 tests/force.o:rodata2 +00000004 g O .data 00000000 tests/force.o:data2 +00000004 g O .bss 00000000 tests/force.o:bss2 +00000000 g F .text 00000000 tests/force.o:xtext + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +0000001c R_MIPS_HI16 .bss +00000034 R_MIPS_LO16 .bss +00000020 R_MIPS_HI16 .rodata +0000002c R_MIPS_LO16 .rodata +00000024 R_MIPS_HI16 .data +00000028 R_MIPS_LO16 .data +00000030 R_MIPS_26 .text + + +Contents of section .text: + 0000 afa40000 afa50004 afa60008 03e00008 ................ + 0010 24020001 27bdffe8 afbf0014 3c040000 $...'.......<... + 0020 3c050000 3c060000 8cc60004 8ca50004 <...<........... + 0030 0c000000 8c840004 8fbf0014 27bd0018 ............'... + 0040 03e00008 00000000 00000000 00000000 ................ +Contents of section .rodata: + 0000 00000001 00000002 00000000 00000000 ................ +Contents of section .data: + 0000 00000003 00000004 00000000 00000000 ................ +Contents of section .options: + 0000 01200000 00000000 a0000074 00000000 . .........t.... + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 a0000074 00000000 00000000 00000000 ...t............ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/kpic-o1.c b/tools/asm-processor/tests/kpic-o1.c new file mode 100644 index 0000000..4d3346d --- /dev/null +++ b/tools/asm-processor/tests/kpic-o1.c @@ -0,0 +1,93 @@ +// COMPILE-FLAGS: -O1 -KPIC +GLOBAL_ASM( +glabel foo +addiu $a0, $a0, 1 +addiu $a0, $a0, 2 +addiu $a0, $a0, 3 +addiu $a0, $a0, 4 +addiu $a0, $a0, 5 +addiu $a0, $a0, 6 +addiu $a0, $a0, 7 +addiu $a0, $a0, 8 +addiu $a0, $a0, 9 +addiu $a0, $a0, 10 +addiu $a0, $a0, 11 +addiu $a0, $a0, 12 +) +GLOBAL_ASM( +.late_rodata +.float 1 +.text +glabel float_fn +addiu $a0, $a0, 13 +addiu $a0, $a0, 14 +addiu $a0, $a0, 15 +addiu $a0, $a0, 16 +addiu $a0, $a0, 17 +addiu $a0, $a0, 18 +addiu $a0, $a0, 19 +addiu $a0, $a0, 20 +addiu $a0, $a0, 21 +addiu $a0, $a0, 22 +addiu $a0, $a0, 23 +addiu $a0, $a0, 24 +addiu $a0, $a0, 25 +addiu $a0, $a0, 26 +addiu $a0, $a0, 27 +addiu $a0, $a0, 28 +addiu $a0, $a0, 29 +addiu $a0, $a0, 30 +) +GLOBAL_ASM( +.late_rodata +.late_rodata_alignment 4 +.float 2 +.double 1 +.double 2 +.double 3 +.double 4 +.double 5 +.double 6 +.double 7 +.double 8 +.text +glabel doubles +addiu $a0, $a0, 31 +addiu $a0, $a0, 32 +addiu $a0, $a0, 33 +addiu $a0, $a0, 34 +addiu $a0, $a0, 35 +addiu $a0, $a0, 36 +addiu $a0, $a0, 37 +addiu $a0, $a0, 38 +addiu $a0, $a0, 39 +addiu $a0, $a0, 40 +addiu $a0, $a0, 41 +addiu $a0, $a0, 42 +addiu $a0, $a0, 43 +addiu $a0, $a0, 44 +addiu $a0, $a0, 45 +addiu $a0, $a0, 46 +addiu $a0, $a0, 47 +addiu $a0, $a0, 48 +addiu $a0, $a0, 49 +addiu $a0, $a0, 50 +addiu $a0, $a0, 51 +addiu $a0, $a0, 52 +addiu $a0, $a0, 53 +addiu $a0, $a0, 54 +addiu $a0, $a0, 55 +addiu $a0, $a0, 56 +addiu $a0, $a0, 57 +addiu $a0, $a0, 58 +addiu $a0, $a0, 59 +addiu $a0, $a0, 60 +addiu $a0, $a0, 61 +addiu $a0, $a0, 62 +addiu $a0, $a0, 63 +addiu $a0, $a0, 64 +addiu $a0, $a0, 65 +addiu $a0, $a0, 66 +addiu $a0, $a0, 67 +addiu $a0, $a0, 68 +) diff --git a/tools/asm-processor/tests/kpic-o1.objdump b/tools/asm-processor/tests/kpic-o1.objdump new file mode 100644 index 0000000..6e33675 --- /dev/null +++ b/tools/asm-processor/tests/kpic-o1.objdump @@ -0,0 +1,46 @@ + +tests/kpic-o1.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000110 .text +00000000 l d .rodata 00000050 .rodata +00000000 g F .text 00000030 foo +00000030 g F .text 00000048 float_fn +00000078 g F .text 00000098 doubles +00000000 O *UND* 00000000 _gp_disp + + +RELOCATION RECORDS FOR [.text]: (none) + +Contents of section .text: + 0000 24840001 24840002 24840003 24840004 $...$...$...$... + 0010 24840005 24840006 24840007 24840008 $...$...$...$... + 0020 24840009 2484000a 2484000b 2484000c $...$...$...$... + 0030 2484000d 2484000e 2484000f 24840010 $...$...$...$... + 0040 24840011 24840012 24840013 24840014 $...$...$...$... + 0050 24840015 24840016 24840017 24840018 $...$...$...$... + 0060 24840019 2484001a 2484001b 2484001c $...$...$...$... + 0070 2484001d 2484001e 2484001f 24840020 $...$...$...$.. + 0080 24840021 24840022 24840023 24840024 $..!$.."$..#$..$ + 0090 24840025 24840026 24840027 24840028 $..%$..&$..'$..( + 00a0 24840029 2484002a 2484002b 2484002c $..)$..*$..+$.., + 00b0 2484002d 2484002e 2484002f 24840030 $..-$...$../$..0 + 00c0 24840031 24840032 24840033 24840034 $..1$..2$..3$..4 + 00d0 24840035 24840036 24840037 24840038 $..5$..6$..7$..8 + 00e0 24840039 2484003a 2484003b 2484003c $..9$..:$..;$..< + 00f0 2484003d 2484003e 2484003f 24840040 $..=$..>$..?$..@ + 0100 24840041 24840042 24840043 24840044 $..A$..B$..C$..D +Contents of section .rodata: + 0000 3f800000 40000000 3ff00000 00000000 ?...@...?....... + 0010 40000000 00000000 40080000 00000000 @.......@....... + 0020 40100000 00000000 40140000 00000000 @.......@....... + 0030 40180000 00000000 401c0000 00000000 @.......@....... + 0040 40200000 00000000 00000000 00000000 @ .............. +Contents of section .options: + 0000 01200000 00000000 92000002 00000000 . .............. + 0010 000f0ff0 00000000 00000000 00000000 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 92000012 00000000 000f0ff0 00000000 ................ + 0010 00000000 00000000 ........ diff --git a/tools/asm-processor/tests/kpic-o2.c b/tools/asm-processor/tests/kpic-o2.c new file mode 100644 index 0000000..f037341 --- /dev/null +++ b/tools/asm-processor/tests/kpic-o2.c @@ -0,0 +1,92 @@ +// COMPILE-FLAGS: -O2 -KPIC +GLOBAL_ASM( +glabel foo +addiu $a0, $a0, 1 +addiu $a0, $a0, 2 +addiu $a0, $a0, 3 +addiu $a0, $a0, 4 +addiu $a0, $a0, 5 +addiu $a0, $a0, 6 +addiu $a0, $a0, 7 +addiu $a0, $a0, 8 +addiu $a0, $a0, 9 +addiu $a0, $a0, 10 +addiu $a0, $a0, 11 +addiu $a0, $a0, 12 +) +GLOBAL_ASM( +.late_rodata +.float 1 +.text +glabel float_fn +addiu $a0, $a0, 13 +addiu $a0, $a0, 14 +addiu $a0, $a0, 15 +addiu $a0, $a0, 16 +addiu $a0, $a0, 17 +addiu $a0, $a0, 18 +addiu $a0, $a0, 19 +addiu $a0, $a0, 20 +addiu $a0, $a0, 21 +addiu $a0, $a0, 22 +addiu $a0, $a0, 23 +addiu $a0, $a0, 24 +addiu $a0, $a0, 25 +addiu $a0, $a0, 26 +addiu $a0, $a0, 27 +addiu $a0, $a0, 28 +addiu $a0, $a0, 29 +addiu $a0, $a0, 30 +) +GLOBAL_ASM( +.late_rodata +.float 2 +.double 1 +.double 2 +.double 3 +.double 4 +.double 5 +.double 6 +.double 7 +.double 8 +.text +glabel doubles +addiu $a0, $a0, 31 +addiu $a0, $a0, 32 +addiu $a0, $a0, 33 +addiu $a0, $a0, 34 +addiu $a0, $a0, 35 +addiu $a0, $a0, 36 +addiu $a0, $a0, 37 +addiu $a0, $a0, 38 +addiu $a0, $a0, 39 +addiu $a0, $a0, 40 +addiu $a0, $a0, 41 +addiu $a0, $a0, 42 +addiu $a0, $a0, 43 +addiu $a0, $a0, 44 +addiu $a0, $a0, 45 +addiu $a0, $a0, 46 +addiu $a0, $a0, 47 +addiu $a0, $a0, 48 +addiu $a0, $a0, 49 +addiu $a0, $a0, 50 +addiu $a0, $a0, 51 +addiu $a0, $a0, 52 +addiu $a0, $a0, 53 +addiu $a0, $a0, 54 +addiu $a0, $a0, 55 +addiu $a0, $a0, 56 +addiu $a0, $a0, 57 +addiu $a0, $a0, 58 +addiu $a0, $a0, 59 +addiu $a0, $a0, 60 +addiu $a0, $a0, 61 +addiu $a0, $a0, 62 +addiu $a0, $a0, 63 +addiu $a0, $a0, 64 +addiu $a0, $a0, 65 +addiu $a0, $a0, 66 +addiu $a0, $a0, 67 +addiu $a0, $a0, 68 +) diff --git a/tools/asm-processor/tests/kpic-o2.objdump b/tools/asm-processor/tests/kpic-o2.objdump new file mode 100644 index 0000000..abf87ae --- /dev/null +++ b/tools/asm-processor/tests/kpic-o2.objdump @@ -0,0 +1,46 @@ + +tests/kpic-o2.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000110 .text +00000000 l d .rodata 00000050 .rodata +00000000 g F .text 00000030 foo +00000030 g F .text 00000048 float_fn +00000078 g F .text 00000098 doubles +00000000 O *UND* 00000000 _gp_disp + + +RELOCATION RECORDS FOR [.text]: (none) + +Contents of section .text: + 0000 24840001 24840002 24840003 24840004 $...$...$...$... + 0010 24840005 24840006 24840007 24840008 $...$...$...$... + 0020 24840009 2484000a 2484000b 2484000c $...$...$...$... + 0030 2484000d 2484000e 2484000f 24840010 $...$...$...$... + 0040 24840011 24840012 24840013 24840014 $...$...$...$... + 0050 24840015 24840016 24840017 24840018 $...$...$...$... + 0060 24840019 2484001a 2484001b 2484001c $...$...$...$... + 0070 2484001d 2484001e 2484001f 24840020 $...$...$...$.. + 0080 24840021 24840022 24840023 24840024 $..!$.."$..#$..$ + 0090 24840025 24840026 24840027 24840028 $..%$..&$..'$..( + 00a0 24840029 2484002a 2484002b 2484002c $..)$..*$..+$.., + 00b0 2484002d 2484002e 2484002f 24840030 $..-$...$../$..0 + 00c0 24840031 24840032 24840033 24840034 $..1$..2$..3$..4 + 00d0 24840035 24840036 24840037 24840038 $..5$..6$..7$..8 + 00e0 24840039 2484003a 2484003b 2484003c $..9$..:$..;$..< + 00f0 2484003d 2484003e 2484003f 24840040 $..=$..>$..?$..@ + 0100 24840041 24840042 24840043 24840044 $..A$..B$..C$..D +Contents of section .rodata: + 0000 3f800000 40000000 3ff00000 00000000 ?...@...?....... + 0010 40000000 00000000 40080000 00000000 @.......@....... + 0020 40100000 00000000 40140000 00000000 @.......@....... + 0030 40180000 00000000 401c0000 00000000 @.......@....... + 0040 40200000 00000000 00000000 00000000 @ .............. +Contents of section .options: + 0000 01200000 00000000 92000002 00000000 . .............. + 0010 000f0ff0 00000000 00000000 00000000 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 92000012 00000000 000f0ff0 00000000 ................ + 0010 00000000 00000000 ........ diff --git a/tools/asm-processor/tests/label-sameline.c b/tools/asm-processor/tests/label-sameline.c new file mode 100644 index 0000000..a35b43d --- /dev/null +++ b/tools/asm-processor/tests/label-sameline.c @@ -0,0 +1,7 @@ +GLOBAL_ASM( +.rdata +.word 0x12345678 +glabel blah +.word blah2 + /*a*/ blah2: /*b*/ .word blah /*c*/ +) diff --git a/tools/asm-processor/tests/label-sameline.objdump b/tools/asm-processor/tests/label-sameline.objdump new file mode 100644 index 0000000..1f1aacf --- /dev/null +++ b/tools/asm-processor/tests/label-sameline.objdump @@ -0,0 +1,25 @@ + +tests/label-sameline.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .rodata 00000010 .rodata +00000000 l d .rodata 00000000 +00000004 g .rodata 00000000 blah + + +RELOCATION RECORDS FOR [.rodata]: +OFFSET TYPE VALUE +00000004 R_MIPS_32 +00000008 R_MIPS_32 blah + + +Contents of section .rodata: + 0000 12345678 00000008 00000000 00000000 .4Vx............ +Contents of section .options: + 0000 01200000 00000000 00000000 00000000 . .............. + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/large.c b/tools/asm-processor/tests/large.c new file mode 100644 index 0000000..4ffb4ac --- /dev/null +++ b/tools/asm-processor/tests/large.c @@ -0,0 +1,164 @@ + +GLOBAL_ASM( +glabel test + +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 +addiu $sp, $sp, -24 + sw $zero, 4($sp) +lw $t6, 4($sp) +addu $t7, $a0, $t6 +sb $zero, ($t7) +lw $t8, 4($sp) +addiu $t9, $t8, 1 +slt $at, $t9, $a1 + sw $t9, 4($sp) + nop +jr $ra + addiu $sp, $sp, 24 + +) + +void foo(void) {} diff --git a/tools/asm-processor/tests/large.objdump b/tools/asm-processor/tests/large.objdump new file mode 100644 index 0000000..17409a7 --- /dev/null +++ b/tools/asm-processor/tests/large.objdump @@ -0,0 +1,58 @@ + +tests/large.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000280 .text +00000270 g F .text 00000010 foo +00000000 g F .text 00000270 test + + +Contents of section .text: + 0000 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0010 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0020 afb90004 00000000 03e00008 27bd0018 ............'... + 0030 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0040 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0050 afb90004 00000000 03e00008 27bd0018 ............'... + 0060 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0070 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0080 afb90004 00000000 03e00008 27bd0018 ............'... + 0090 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 00a0 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 00b0 afb90004 00000000 03e00008 27bd0018 ............'... + 00c0 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 00d0 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 00e0 afb90004 00000000 03e00008 27bd0018 ............'... + 00f0 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0100 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0110 afb90004 00000000 03e00008 27bd0018 ............'... + 0120 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0130 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0140 afb90004 00000000 03e00008 27bd0018 ............'... + 0150 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0160 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0170 afb90004 00000000 03e00008 27bd0018 ............'... + 0180 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0190 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 01a0 afb90004 00000000 03e00008 27bd0018 ............'... + 01b0 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 01c0 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 01d0 afb90004 00000000 03e00008 27bd0018 ............'... + 01e0 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 01f0 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0200 afb90004 00000000 03e00008 27bd0018 ............'... + 0210 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0220 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0230 afb90004 00000000 03e00008 27bd0018 ............'... + 0240 27bdffe8 afa00004 8fae0004 008e7821 '.............x! + 0250 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0260 afb90004 00000000 03e00008 27bd0018 ............'... + 0270 03e00008 00000000 03e00008 00000000 ................ +Contents of section .options: + 0000 01200000 00000000 80000000 00000000 . .............. + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 a300c032 00000000 00000000 00000000 ...2............ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/late_rodata_align.c b/tools/asm-processor/tests/late_rodata_align.c new file mode 100644 index 0000000..7367c79 --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_align.c @@ -0,0 +1,80 @@ +GLOBAL_ASM( +.late_rodata + .float 4.1 + .float 4.2 + .float 4.3 + .float 4.4 +.text +glabel a + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +float foo(void) { "foo"; return 1.1f; } + +GLOBAL_ASM( +.late_rodata +.late_rodata_alignment 4 + .float 5.1 + .float 5.2 + .float 5.3 + .float 5.4 +.text +glabel b + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +float bar(void) { "bar"; return 1.2f; } + +GLOBAL_ASM( +.late_rodata +.late_rodata_alignment 8 + .float 6.1 + .float 6.2 + .float 6.3 + .float 6.4 + .float 6.5 +.text +glabel c + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + diff --git a/tools/asm-processor/tests/late_rodata_align.objdump b/tools/asm-processor/tests/late_rodata_align.objdump new file mode 100644 index 0000000..87c05e8 --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_align.objdump @@ -0,0 +1,51 @@ + +tests/late_rodata_align.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 000000f0 .text +00000000 l d .rodata 00000050 .rodata +00000040 g F .text 0000001c foo +00000090 g F .text 0000001c bar +00000000 g F .text 00000040 a +0000005c g F .text 00000034 b +000000ac g F .text 00000038 c + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +00000040 R_MIPS_HI16 .rodata +00000048 R_MIPS_LO16 .rodata +00000090 R_MIPS_HI16 .rodata +00000098 R_MIPS_LO16 .rodata + + +Contents of section .text: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00000000 00000000 00000000 ................ + 0020 00000000 00000000 00000000 00000000 ................ + 0030 00000000 00000000 00000000 00000000 ................ + 0040 3c010000 03e00008 c4200018 03e00008 <........ ...... + 0050 00000000 03e00008 00000000 00000000 ................ + 0060 00000000 00000000 00000000 00000000 ................ + 0070 00000000 00000000 00000000 00000000 ................ + 0080 00000000 00000000 00000000 00000000 ................ + 0090 3c010000 03e00008 c420002c 03e00008 <........ .,.... + 00a0 00000000 03e00008 00000000 00000000 ................ + 00b0 00000000 00000000 00000000 00000000 ................ + 00c0 00000000 00000000 00000000 00000000 ................ + 00d0 00000000 00000000 00000000 00000000 ................ + 00e0 00000000 00000000 00000000 00000000 ................ +Contents of section .rodata: + 0000 666f6f00 62617200 40833333 40866666 foo.bar.@.33@.ff + 0010 4089999a 408ccccd 3f8ccccd 40a33333 @...@...?...@.33 + 0020 40a66666 40a9999a 40accccd 3f99999a @.ff@...@...?... + 0030 40c33333 40c66666 40c9999a 40cccccd @.33@.ff@...@... + 0040 40d00000 00000000 00000000 00000000 @............... +Contents of section .options: + 0000 01200000 00000000 80000002 00000000 . .............. + 0010 000005f1 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000002 00000000 000005f1 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/late_rodata_doubles.c b/tools/asm-processor/tests/late_rodata_doubles.c new file mode 100644 index 0000000..0a27b41 --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_doubles.c @@ -0,0 +1,83 @@ +GLOBAL_ASM( +.late_rodata + .float 4.1 +.text +glabel a + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +float foo(void) { + return 4.15f; +} + +GLOBAL_ASM( +.late_rodata + .float 4.2 + .word 0 + .double 4.3 +.text +glabel b + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +float bar(void) { + return 4.4f; +} + +GLOBAL_ASM( +.late_rodata + .float 4.55 + .double 4.6 +.text +glabel c + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +float baz(void) { + return 4.6f; +} diff --git a/tools/asm-processor/tests/late_rodata_doubles.objdump b/tools/asm-processor/tests/late_rodata_doubles.objdump new file mode 100644 index 0000000..84d7e67 --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_doubles.objdump @@ -0,0 +1,55 @@ + +tests/late_rodata_doubles.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000120 .text +00000000 l d .rodata 00000030 .rodata +00000040 g F .text 0000001c foo +0000009c g F .text 0000001c bar +000000f8 g F .text 0000001c baz +00000000 g F .text 00000040 a +0000005c g F .text 00000040 b +000000b8 g F .text 00000040 c + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +00000040 R_MIPS_HI16 .rodata +00000048 R_MIPS_LO16 .rodata +0000009c R_MIPS_HI16 .rodata +000000a4 R_MIPS_LO16 .rodata +000000f8 R_MIPS_HI16 .rodata +00000100 R_MIPS_LO16 .rodata + + +Contents of section .text: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00000000 00000000 00000000 ................ + 0020 00000000 00000000 00000000 00000000 ................ + 0030 00000000 00000000 00000000 00000000 ................ + 0040 3c010000 03e00008 c4200004 03e00008 <........ ...... + 0050 00000000 03e00008 00000000 00000000 ................ + 0060 00000000 00000000 00000000 00000000 ................ + 0070 00000000 00000000 00000000 00000000 ................ + 0080 00000000 00000000 00000000 00000000 ................ + 0090 00000000 00000000 00000000 3c010000 ............<... + 00a0 03e00008 c4200018 03e00008 00000000 ..... .......... + 00b0 03e00008 00000000 00000000 00000000 ................ + 00c0 00000000 00000000 00000000 00000000 ................ + 00d0 00000000 00000000 00000000 00000000 ................ + 00e0 00000000 00000000 00000000 00000000 ................ + 00f0 00000000 00000000 3c010000 03e00008 ........<....... + 0100 c4200028 03e00008 00000000 03e00008 . .(............ + 0110 00000000 00000000 00000000 00000000 ................ +Contents of section .rodata: + 0000 40833333 4084cccd 40866666 00000000 @.33@...@.ff.... + 0010 40113333 33333333 408ccccd 4091999a @.333333@...@... + 0020 40126666 66666666 40933333 00000000 @.ffffff@.33.... +Contents of section .options: + 0000 01200000 00000000 80000002 00000000 . .............. + 0010 000000f1 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000002 00000000 000000f1 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/late_rodata_doubles_mips1.c b/tools/asm-processor/tests/late_rodata_doubles_mips1.c new file mode 100644 index 0000000..b97261f --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_doubles_mips1.c @@ -0,0 +1,86 @@ +// COMPILE-FLAGS: -O2 +// COMPILE-ISET: -mips1 +// exact copy of late_rodata_doubles.c except for the -mips1 -O2 additions +GLOBAL_ASM( +.late_rodata + .float 4.1 +.text +glabel a + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +float foo(void) { + return 4.15f; +} + +GLOBAL_ASM( +.late_rodata + .float 4.2 + .word 0 + .double 4.3 +.text +glabel b + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +float bar(void) { + return 4.4f; +} + +GLOBAL_ASM( +.late_rodata + .float 4.55 + .double 4.6 +.text +glabel c + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +float baz(void) { + return 4.6f; +} diff --git a/tools/asm-processor/tests/late_rodata_doubles_mips1.objdump b/tools/asm-processor/tests/late_rodata_doubles_mips1.objdump new file mode 100644 index 0000000..8fac85f --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_doubles_mips1.objdump @@ -0,0 +1,52 @@ + +tests/late_rodata_doubles_mips1.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 000000f0 .text +00000000 l d .rodata 00000030 .rodata +00000040 g F .text 00000010 foo +00000090 g F .text 00000010 bar +000000e0 g F .text 00000010 baz +00000000 g F .text 00000040 a +00000050 g F .text 00000040 b +000000a0 g F .text 00000040 c + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +00000040 R_MIPS_HI16 .rodata +00000044 R_MIPS_LO16 .rodata +00000090 R_MIPS_HI16 .rodata +00000094 R_MIPS_LO16 .rodata +000000e0 R_MIPS_HI16 .rodata +000000e4 R_MIPS_LO16 .rodata + + +Contents of section .text: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00000000 00000000 00000000 ................ + 0020 00000000 00000000 00000000 00000000 ................ + 0030 00000000 00000000 00000000 00000000 ................ + 0040 3c010000 c4200004 03e00008 00000000 <.... .......... + 0050 00000000 00000000 00000000 00000000 ................ + 0060 00000000 00000000 00000000 00000000 ................ + 0070 00000000 00000000 00000000 00000000 ................ + 0080 00000000 00000000 00000000 00000000 ................ + 0090 3c010000 c4200018 03e00008 00000000 <.... .......... + 00a0 00000000 00000000 00000000 00000000 ................ + 00b0 00000000 00000000 00000000 00000000 ................ + 00c0 00000000 00000000 00000000 00000000 ................ + 00d0 00000000 00000000 00000000 00000000 ................ + 00e0 3c010000 c4200028 03e00008 00000000 <.... .(........ +Contents of section .rodata: + 0000 40833333 4084cccd 40866666 00000000 @.33@...@.ff.... + 0010 40113333 33333333 408ccccd 4091999a @.333333@...@... + 0020 40126666 66666666 40933333 00000000 @.ffffff@.33.... +Contents of section .options: + 0000 01200000 00000000 80000002 00000000 . .............. + 0010 000c0011 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000002 00000000 000c0011 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/late_rodata_jtbl.c b/tools/asm-processor/tests/late_rodata_jtbl.c new file mode 100644 index 0000000..3c2a5bb --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_jtbl.c @@ -0,0 +1,153 @@ +// COMPILE-FLAGS: -O2 +GLOBAL_ASM( +.late_rodata +.double 1 +.double 2 +.double 3 +.double 4 +.double 5 +.double 6 +.double 7 +.double 8 +.text +glabel doubles1 +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +) + +float a(void) { return 1.1f; } + +GLOBAL_ASM( +.late_rodata +.float 1 +.double 2 +.double 3 +.double 4 +.double 5 +.double 6 +.double 7 +.double 8 +.double 9 +.float 10 +.text +glabel doubles2 +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +) + +GLOBAL_ASM( +glabel a2 +move $a0, $a0 +nop +nop +nop +jr $ra +move $a0, $a0 +) + +GLOBAL_ASM( +.late_rodata + +glabel jtbl +.word case0, case1, case2, case3, case4, case5, case6, case7, case8, case9, case10 +.word case11, case12, case13, case14, case15, case16, case17, case18, case19, case20 +.word case21, case22, case23, case24, case25, case26 + +.text +glabel foo +sltiu $at, $a0, 0xa +beqz $at, .L756E659B + sll $t7, $a0, 2 +lui $at, %hi(jtbl) +addu $at, $at, $t7 +lw $t7, %lo(jtbl)($at) +jr $t7 + nop +case0: addiu $a0, $a0, 1 +case1: addiu $a0, $a0, 1 +case2: addiu $a0, $a0, 1 +case3: addiu $a0, $a0, 1 +case4: addiu $a0, $a0, 1 +case5: addiu $a0, $a0, 1 +case6: addiu $a0, $a0, 1 +case7: addiu $a0, $a0, 1 +case8: addiu $a0, $a0, 1 +case9: addiu $a0, $a0, 1 +case10: addiu $a0, $a0, 1 +case11: addiu $a0, $a0, 1 +case12: addiu $a0, $a0, 1 +case13: addiu $a0, $a0, 1 +case14: addiu $a0, $a0, 1 +case15: addiu $a0, $a0, 1 +case16: addiu $a0, $a0, 1 +case17: addiu $a0, $a0, 1 +case18: addiu $a0, $a0, 1 +case19: addiu $a0, $a0, 1 +case20: addiu $a0, $a0, 1 +case21: addiu $a0, $a0, 1 +case22: addiu $a0, $a0, 1 +case23: addiu $a0, $a0, 1 +case24: addiu $a0, $a0, 1 +case25: addiu $a0, $a0, 1 +case26: +jr $ra + addiu $v0, $a0, 1 + +.L756E659B: +addiu $v0, $zero, 2 +jr $ra + nop +) + +GLOBAL_ASM( +glabel b2 +move $a0, $a0 +nop +nop +jr $ra +move $a0, $a0 +) + +float b(void) { return 1.2f; } diff --git a/tools/asm-processor/tests/late_rodata_jtbl.objdump b/tools/asm-processor/tests/late_rodata_jtbl.objdump new file mode 100644 index 0000000..44bb6e8 --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_jtbl.objdump @@ -0,0 +1,110 @@ + +tests/late_rodata_jtbl.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 000001a0 .text +00000000 l d .rodata 00000100 .rodata +00000000 l d .text 00000000 +0000005c g F .text 0000000c a +0000018c g F .text 0000000c b +00000000 g F .text 0000005c doubles1 +00000068 g F .text 0000005c doubles2 +000000c4 g F .text 00000018 a2 +000000dc g F .text 0000009c foo +0000008c g .rodata 00000000 jtbl +00000178 g F .text 00000014 b2 + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +0000005c R_MIPS_HI16 .rodata +00000064 R_MIPS_LO16 .rodata +0000018c R_MIPS_HI16 .rodata +00000194 R_MIPS_LO16 .rodata +000000e8 R_MIPS_HI16 jtbl +000000f0 R_MIPS_LO16 jtbl + + +RELOCATION RECORDS FOR [.rodata]: +OFFSET TYPE VALUE +0000008c R_MIPS_32 +00000090 R_MIPS_32 +00000094 R_MIPS_32 +00000098 R_MIPS_32 +0000009c R_MIPS_32 +000000a0 R_MIPS_32 +000000a4 R_MIPS_32 +000000a8 R_MIPS_32 +000000ac R_MIPS_32 +000000b0 R_MIPS_32 +000000b4 R_MIPS_32 +000000b8 R_MIPS_32 +000000bc R_MIPS_32 +000000c0 R_MIPS_32 +000000c4 R_MIPS_32 +000000c8 R_MIPS_32 +000000cc R_MIPS_32 +000000d0 R_MIPS_32 +000000d4 R_MIPS_32 +000000d8 R_MIPS_32 +000000dc R_MIPS_32 +000000e0 R_MIPS_32 +000000e4 R_MIPS_32 +000000e8 R_MIPS_32 +000000ec R_MIPS_32 +000000f0 R_MIPS_32 +000000f4 R_MIPS_32 + + +Contents of section .text: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00000000 00000000 00000000 ................ + 0020 00000000 00000000 00000000 00000000 ................ + 0030 00000000 00000000 00000000 00000000 ................ + 0040 00000000 00000000 00000000 00000000 ................ + 0050 00000000 00000000 00000000 3c010000 ............<... + 0060 03e00008 c4200040 00000000 00000000 ..... .@........ + 0070 00000000 00000000 00000000 00000000 ................ + 0080 00000000 00000000 00000000 00000000 ................ + 0090 00000000 00000000 00000000 00000000 ................ + 00a0 00000000 00000000 00000000 00000000 ................ + 00b0 00000000 00000000 00000000 00000000 ................ + 00c0 00000000 00802025 00000000 00000000 ...... %........ + 00d0 00000000 03e00008 00802025 2c81000a .......... %,... + 00e0 10200022 00047880 3c010000 002f0821 . ."..x.<..../.! + 00f0 8c2f0000 01e00008 00000000 24840001 ./..........$... + 0100 24840001 24840001 24840001 24840001 $...$...$...$... + 0110 24840001 24840001 24840001 24840001 $...$...$...$... + 0120 24840001 24840001 24840001 24840001 $...$...$...$... + 0130 24840001 24840001 24840001 24840001 $...$...$...$... + 0140 24840001 24840001 24840001 24840001 $...$...$...$... + 0150 24840001 24840001 24840001 24840001 $...$...$...$... + 0160 24840001 03e00008 24820001 24020002 $.......$...$... + 0170 03e00008 00000000 00802025 00000000 .......... %.... + 0180 00000000 03e00008 00802025 3c010000 .......... %<... + 0190 03e00008 c42000f8 00000000 00000000 ..... .......... +Contents of section .rodata: + 0000 3ff00000 00000000 40000000 00000000 ?.......@....... + 0010 40080000 00000000 40100000 00000000 @.......@....... + 0020 40140000 00000000 40180000 00000000 @.......@....... + 0030 401c0000 00000000 40200000 00000000 @.......@ ...... + 0040 3f8ccccd 3f800000 40000000 00000000 ?...?...@....... + 0050 40080000 00000000 40100000 00000000 @.......@....... + 0060 40140000 00000000 40180000 00000000 @.......@....... + 0070 401c0000 00000000 40200000 00000000 @.......@ ...... + 0080 40220000 00000000 41200000 000000fc @"......A ...... + 0090 00000100 00000104 00000108 0000010c ................ + 00a0 00000110 00000114 00000118 0000011c ................ + 00b0 00000120 00000124 00000128 0000012c ... ...$...(..., + 00c0 00000130 00000134 00000138 0000013c ...0...4...8...< + 00d0 00000140 00000144 00000148 0000014c ...@...D...H...L + 00e0 00000150 00000154 00000158 0000015c ...P...T...X...\ + 00f0 00000160 00000164 3f99999a 00000000 ...`...d?....... +Contents of section .options: + 0000 01200000 00000000 80004002 00000000 . ........@..... + 0010 000000f1 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 8000c016 00000000 000000f1 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/late_rodata_jtbl_mips1.c b/tools/asm-processor/tests/late_rodata_jtbl_mips1.c new file mode 100644 index 0000000..505f4c1 --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_jtbl_mips1.c @@ -0,0 +1,155 @@ +// COMPILE-FLAGS: -O2 +// COMPILE-ISET: -mips1 +// exact copy of late_rodata_jtbl.c except for the -mips1 addition +GLOBAL_ASM( +.late_rodata +.double 1 +.double 2 +.double 3 +.double 4 +.double 5 +.double 6 +.double 7 +.double 8 +.text +glabel doubles1 +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +) + +float a(void) { return 1.1f; } + +GLOBAL_ASM( +.late_rodata +.float 1 +.double 2 +.double 3 +.double 4 +.double 5 +.double 6 +.double 7 +.double 8 +.double 9 +.float 10 +.text +glabel doubles2 +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +) + +GLOBAL_ASM( +glabel a2 +move $a0, $a0 +nop +nop +nop +jr $ra +move $a0, $a0 +) + +GLOBAL_ASM( +.late_rodata + +glabel jtbl +.word case0, case1, case2, case3, case4, case5, case6, case7, case8, case9, case10 +.word case11, case12, case13, case14, case15, case16, case17, case18, case19, case20 +.word case21, case22, case23, case24, case25, case26 + +.text +glabel foo +sltiu $at, $a0, 0xa +beqz $at, .L756E659B + sll $t7, $a0, 2 +lui $at, %hi(jtbl) +addu $at, $at, $t7 +lw $t7, %lo(jtbl)($at) +jr $t7 + nop +case0: addiu $a0, $a0, 1 +case1: addiu $a0, $a0, 1 +case2: addiu $a0, $a0, 1 +case3: addiu $a0, $a0, 1 +case4: addiu $a0, $a0, 1 +case5: addiu $a0, $a0, 1 +case6: addiu $a0, $a0, 1 +case7: addiu $a0, $a0, 1 +case8: addiu $a0, $a0, 1 +case9: addiu $a0, $a0, 1 +case10: addiu $a0, $a0, 1 +case11: addiu $a0, $a0, 1 +case12: addiu $a0, $a0, 1 +case13: addiu $a0, $a0, 1 +case14: addiu $a0, $a0, 1 +case15: addiu $a0, $a0, 1 +case16: addiu $a0, $a0, 1 +case17: addiu $a0, $a0, 1 +case18: addiu $a0, $a0, 1 +case19: addiu $a0, $a0, 1 +case20: addiu $a0, $a0, 1 +case21: addiu $a0, $a0, 1 +case22: addiu $a0, $a0, 1 +case23: addiu $a0, $a0, 1 +case24: addiu $a0, $a0, 1 +case25: addiu $a0, $a0, 1 +case26: +jr $ra + addiu $v0, $a0, 1 + +.L756E659B: +addiu $v0, $zero, 2 +jr $ra + nop +) + +GLOBAL_ASM( +glabel b2 +move $a0, $a0 +nop +nop +jr $ra +move $a0, $a0 +) + +float b(void) { return 1.2f; } diff --git a/tools/asm-processor/tests/late_rodata_jtbl_mips1.objdump b/tools/asm-processor/tests/late_rodata_jtbl_mips1.objdump new file mode 100644 index 0000000..d37c8bb --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_jtbl_mips1.objdump @@ -0,0 +1,110 @@ + +tests/late_rodata_jtbl_mips1.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 000001a0 .text +00000000 l d .rodata 00000100 .rodata +00000000 l d .text 00000000 +0000005c g F .text 00000010 a +00000190 g F .text 00000010 b +00000000 g F .text 0000005c doubles1 +0000006c g F .text 0000005c doubles2 +000000c8 g F .text 00000018 a2 +000000e0 g F .text 0000009c foo +0000008c g .rodata 00000000 jtbl +0000017c g F .text 00000014 b2 + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +0000005c R_MIPS_HI16 .rodata +00000060 R_MIPS_LO16 .rodata +00000190 R_MIPS_HI16 .rodata +00000194 R_MIPS_LO16 .rodata +000000ec R_MIPS_HI16 jtbl +000000f4 R_MIPS_LO16 jtbl + + +RELOCATION RECORDS FOR [.rodata]: +OFFSET TYPE VALUE +0000008c R_MIPS_32 +00000090 R_MIPS_32 +00000094 R_MIPS_32 +00000098 R_MIPS_32 +0000009c R_MIPS_32 +000000a0 R_MIPS_32 +000000a4 R_MIPS_32 +000000a8 R_MIPS_32 +000000ac R_MIPS_32 +000000b0 R_MIPS_32 +000000b4 R_MIPS_32 +000000b8 R_MIPS_32 +000000bc R_MIPS_32 +000000c0 R_MIPS_32 +000000c4 R_MIPS_32 +000000c8 R_MIPS_32 +000000cc R_MIPS_32 +000000d0 R_MIPS_32 +000000d4 R_MIPS_32 +000000d8 R_MIPS_32 +000000dc R_MIPS_32 +000000e0 R_MIPS_32 +000000e4 R_MIPS_32 +000000e8 R_MIPS_32 +000000ec R_MIPS_32 +000000f0 R_MIPS_32 +000000f4 R_MIPS_32 + + +Contents of section .text: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00000000 00000000 00000000 ................ + 0020 00000000 00000000 00000000 00000000 ................ + 0030 00000000 00000000 00000000 00000000 ................ + 0040 00000000 00000000 00000000 00000000 ................ + 0050 00000000 00000000 00000000 3c010000 ............<... + 0060 c4200040 03e00008 00000000 00000000 . .@............ + 0070 00000000 00000000 00000000 00000000 ................ + 0080 00000000 00000000 00000000 00000000 ................ + 0090 00000000 00000000 00000000 00000000 ................ + 00a0 00000000 00000000 00000000 00000000 ................ + 00b0 00000000 00000000 00000000 00000000 ................ + 00c0 00000000 00000000 00802025 00000000 .......... %.... + 00d0 00000000 00000000 03e00008 00802025 .............. % + 00e0 2c81000a 10200022 00047880 3c010000 ,.... ."..x.<... + 00f0 002f0821 8c2f0000 01e00008 00000000 ./.!./.......... + 0100 24840001 24840001 24840001 24840001 $...$...$...$... + 0110 24840001 24840001 24840001 24840001 $...$...$...$... + 0120 24840001 24840001 24840001 24840001 $...$...$...$... + 0130 24840001 24840001 24840001 24840001 $...$...$...$... + 0140 24840001 24840001 24840001 24840001 $...$...$...$... + 0150 24840001 24840001 24840001 24840001 $...$...$...$... + 0160 24840001 24840001 03e00008 24820001 $...$.......$... + 0170 24020002 03e00008 00000000 00802025 $............. % + 0180 00000000 00000000 03e00008 00802025 .............. % + 0190 3c010000 c42000f8 03e00008 00000000 <.... .......... +Contents of section .rodata: + 0000 3ff00000 00000000 40000000 00000000 ?.......@....... + 0010 40080000 00000000 40100000 00000000 @.......@....... + 0020 40140000 00000000 40180000 00000000 @.......@....... + 0030 401c0000 00000000 40200000 00000000 @.......@ ...... + 0040 3f8ccccd 3f800000 40000000 00000000 ?...?...@....... + 0050 40080000 00000000 40100000 00000000 @.......@....... + 0060 40140000 00000000 40180000 00000000 @.......@....... + 0070 401c0000 00000000 40200000 00000000 @.......@ ...... + 0080 40220000 00000000 41200000 00000100 @"......A ...... + 0090 00000104 00000108 0000010c 00000110 ................ + 00a0 00000114 00000118 0000011c 00000120 ............... + 00b0 00000124 00000128 0000012c 00000130 ...$...(...,...0 + 00c0 00000134 00000138 0000013c 00000140 ...4...8...<...@ + 00d0 00000144 00000148 0000014c 00000150 ...D...H...L...P + 00e0 00000154 00000158 0000015c 00000160 ...T...X...\...` + 00f0 00000164 00000168 3f99999a 00000000 ...d...h?....... +Contents of section .options: + 0000 01200000 00000000 80004002 00000000 . ........@..... + 0010 000000f1 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 8000c016 00000000 000000f1 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/late_rodata_misaligned_doubles.c b/tools/asm-processor/tests/late_rodata_misaligned_doubles.c new file mode 100644 index 0000000..a453dbd --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_misaligned_doubles.c @@ -0,0 +1,77 @@ +GLOBAL_ASM( +.late_rodata + .float 4.01 + .word 0 + .double 4.02 +.text +glabel a + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +double foo(void) { return 4.03; } + +GLOBAL_ASM( +.late_rodata + .float 4.04 + .double 4.05 +.text +glabel b + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + +double bar(void) { return 4.06; } +float baz(void) { return 4.07f; } + +GLOBAL_ASM( +.late_rodata + .double 4.08 +.text +glabel c + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop +) + diff --git a/tools/asm-processor/tests/late_rodata_misaligned_doubles.objdump b/tools/asm-processor/tests/late_rodata_misaligned_doubles.objdump new file mode 100644 index 0000000..cdba84d --- /dev/null +++ b/tools/asm-processor/tests/late_rodata_misaligned_doubles.objdump @@ -0,0 +1,56 @@ + +tests/late_rodata_misaligned_doubles.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000120 .text +00000000 l d .rodata 00000040 .rodata +00000040 g F .text 0000001c foo +0000009c g F .text 0000001c bar +000000b8 g F .text 0000001c baz +00000000 g F .text 00000040 a +0000005c g F .text 00000040 b +000000d4 g F .text 00000040 c + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +00000040 R_MIPS_HI16 .rodata +00000048 R_MIPS_LO16 .rodata +0000009c R_MIPS_HI16 .rodata +000000a4 R_MIPS_LO16 .rodata +000000b8 R_MIPS_HI16 .rodata +000000c0 R_MIPS_LO16 .rodata + + +Contents of section .text: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00000000 00000000 00000000 ................ + 0020 00000000 00000000 00000000 00000000 ................ + 0030 00000000 00000000 00000000 00000000 ................ + 0040 3c010000 03e00008 d4200010 03e00008 <........ ...... + 0050 00000000 03e00008 00000000 00000000 ................ + 0060 00000000 00000000 00000000 00000000 ................ + 0070 00000000 00000000 00000000 00000000 ................ + 0080 00000000 00000000 00000000 00000000 ................ + 0090 00000000 00000000 00000000 3c010000 ............<... + 00a0 03e00008 d4200028 03e00008 00000000 ..... .(........ + 00b0 03e00008 00000000 3c010000 03e00008 ........<....... + 00c0 c4200030 03e00008 00000000 03e00008 . .0............ + 00d0 00000000 00000000 00000000 00000000 ................ + 00e0 00000000 00000000 00000000 00000000 ................ + 00f0 00000000 00000000 00000000 00000000 ................ + 0100 00000000 00000000 00000000 00000000 ................ + 0110 00000000 00000000 00000000 00000000 ................ +Contents of section .rodata: + 0000 408051ec 00000000 4010147a e147ae14 @.Q.....@..z.G.. + 0010 40101eb8 51eb851f 00000000 408147ae @...Q.......@.G. + 0020 40103333 33333333 40103d70 a3d70a3d @.333333@.=p...= + 0030 40823d71 00000000 401051eb 851eb852 @.=q....@.Q....R +Contents of section .options: + 0000 01200000 00000000 80000002 00000000 . .............. + 0010 000000f3 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000002 00000000 000000f3 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/line-continuation-separate-file.s b/tools/asm-processor/tests/line-continuation-separate-file.s new file mode 100644 index 0000000..e130180 --- /dev/null +++ b/tools/asm-processor/tests/line-continuation-separate-file.s @@ -0,0 +1,4 @@ +.rdata + label: .asciiz "1\n\ +2", \ + "34", "56" diff --git a/tools/asm-processor/tests/line-continuation.asmproc.d b/tools/asm-processor/tests/line-continuation.asmproc.d new file mode 100644 index 0000000..687dd1a --- /dev/null +++ b/tools/asm-processor/tests/line-continuation.asmproc.d @@ -0,0 +1,3 @@ +tests/line-continuation.o: tests/line-continuation-separate-file.s + +tests/line-continuation-separate-file.s: diff --git a/tools/asm-processor/tests/line-continuation.c b/tools/asm-processor/tests/line-continuation.c new file mode 100644 index 0000000..d5ab093 --- /dev/null +++ b/tools/asm-processor/tests/line-continuation.c @@ -0,0 +1,22 @@ +void foo(void) { "abcdef"; } + +GLOBAL_ASM( +.rdata + .ascii "AB" \ + "CD", "EF" + .ascii "GH\n\n\n\0\11\222\3333\44444\x1234567\n\nIJK" +) + +void bar(void) { "hello"; } + +GLOBAL_ASM( +.rdata + .asciiz "1\ +2" + .asciiz "34", "56" + .asciiz "78\n\n\n\0\11\222\3333\44444\x1234567\n\n9A" +) + +void baz(void) { "ghijkl"; } + +GLOBAL_ASM("tests/line-continuation-separate-file.s") diff --git a/tools/asm-processor/tests/line-continuation.objdump b/tools/asm-processor/tests/line-continuation.objdump new file mode 100644 index 0000000..038766f --- /dev/null +++ b/tools/asm-processor/tests/line-continuation.objdump @@ -0,0 +1,30 @@ + +tests/line-continuation.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000030 .text +00000000 l d .rodata 00000060 .rodata +00000000 g F .text 00000010 foo +00000010 g F .text 00000010 bar +00000020 g F .text 00000010 baz + + +Contents of section .text: + 0000 03e00008 00000000 03e00008 00000000 ................ + 0010 03e00008 00000000 03e00008 00000000 ................ + 0020 03e00008 00000000 03e00008 00000000 ................ +Contents of section .rodata: + 0000 61626364 65660000 41424344 45464748 abcdef..ABCDEFGH + 0010 0a0a0a00 0992db33 24343467 0a0a494a .......3$44g..IJ + 0020 4b000000 68656c6c 6f000000 31320033 K...hello...12.3 + 0030 34003536 0037380a 0a0a0009 92db3324 4.56.78.......3$ + 0040 3434670a 0a394100 6768696a 6b6c0000 44g..9A.ghijkl.. + 0050 310a3200 33340035 36000000 00000000 1.2.34.56....... +Contents of section .options: + 0000 01200000 00000000 80000000 00000000 . .............. + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000000 00000000 00000000 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/o0.c b/tools/asm-processor/tests/o0.c new file mode 100644 index 0000000..6f8e5e4 --- /dev/null +++ b/tools/asm-processor/tests/o0.c @@ -0,0 +1,28 @@ +// COMPILE-FLAGS: -O0 + +int a(void) { return 1; } +GLOBAL_ASM( +glabel foo +addiu $a0, $a0, 1 +addiu $a0, $a0, 2 +addiu $a0, $a0, 3 +jr $ra +addiu $a0, $a0, 4 +) +float b(void) { return 1.2f; } +GLOBAL_ASM( +.late_rodata +glabel float1 +.float 12.34 + +.text +glabel bar +addiu $a0, $a0, 5 +addiu $a0, $a0, 6 +addiu $a0, $a0, 7 +addiu $a0, $a0, 8 +lui $v0, %hi(float1 + 1) +jr $ra +addiu $v0, $v0, %lo(float1 + 1) +) +float c(void) { return 1.3f; } diff --git a/tools/asm-processor/tests/o0.objdump b/tools/asm-processor/tests/o0.objdump new file mode 100644 index 0000000..f94ec7a --- /dev/null +++ b/tools/asm-processor/tests/o0.objdump @@ -0,0 +1,44 @@ + +tests/o0.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000090 .text +00000000 l d .rodata 00000010 .rodata +00000000 g F .text 0000001c a +00000030 g F .text 00000020 b +0000006c g F .text 00000020 c +0000001c g F .text 00000014 foo +00000050 g F .text 0000001c bar +00000004 g .rodata 00000000 float1 + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +00000030 R_MIPS_HI16 .rodata +00000034 R_MIPS_LO16 .rodata +0000006c R_MIPS_HI16 .rodata +00000070 R_MIPS_LO16 .rodata +00000060 R_MIPS_HI16 float1 +00000068 R_MIPS_LO16 float1 + + +Contents of section .text: + 0000 24020001 03e00008 00000000 03e00008 $............... + 0010 00000000 03e00008 00000000 24840001 ............$... + 0020 24840002 24840003 03e00008 24840004 $...$.......$... + 0030 3c010000 c4200000 03e00008 00000000 <.... .......... + 0040 03e00008 00000000 03e00008 00000000 ................ + 0050 24840005 24840006 24840007 24840008 $...$...$...$... + 0060 3c020000 03e00008 24420001 3c010000 <.......$B..<... + 0070 c4200008 03e00008 00000000 03e00008 . .............. + 0080 00000000 03e00008 00000000 00000000 ................ +Contents of section .rodata: + 0000 3f99999a 414570a4 3fa66666 00000000 ?...AEp.?.ff.... +Contents of section .options: + 0000 01200000 00000000 80000006 00000000 . .............. + 0010 00000011 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000016 00000000 00000011 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/o2.c b/tools/asm-processor/tests/o2.c new file mode 100644 index 0000000..144a604 --- /dev/null +++ b/tools/asm-processor/tests/o2.c @@ -0,0 +1,26 @@ +// COMPILE-FLAGS: -O2 + +int a(void) { return 1; } +GLOBAL_ASM( +glabel foo +addiu $a0, $a0, 1 +addiu $a0, $a0, 2 +addiu $a0, $a0, 3 +jr $ra +addiu $a0, $a0, 4 +) +float b(void) { return 1.2f; } +GLOBAL_ASM( +.late_rodata +glabel float1 +.float 12.34 + +.text +glabel bar +addiu $a0, $a0, 5 +addiu $a0, $a0, 6 +lui $v0, %hi(float1 + 1) +jr $ra +addiu $v0, $v0, %lo(float1 + 1) +) +float c(void) { return 1.3f; } diff --git a/tools/asm-processor/tests/o2.objdump b/tools/asm-processor/tests/o2.objdump new file mode 100644 index 0000000..61e3fb0 --- /dev/null +++ b/tools/asm-processor/tests/o2.objdump @@ -0,0 +1,40 @@ + +tests/o2.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000050 .text +00000000 l d .rodata 00000010 .rodata +00000000 g F .text 00000008 a +0000001c g F .text 0000000c b +0000003c g F .text 0000000c c +00000008 g F .text 00000014 foo +00000028 g F .text 00000014 bar +00000004 g .rodata 00000000 float1 + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +0000001c R_MIPS_HI16 .rodata +00000024 R_MIPS_LO16 .rodata +0000003c R_MIPS_HI16 .rodata +00000044 R_MIPS_LO16 .rodata +00000030 R_MIPS_HI16 float1 +00000038 R_MIPS_LO16 float1 + + +Contents of section .text: + 0000 03e00008 24020001 24840001 24840002 ....$...$...$... + 0010 24840003 03e00008 24840004 3c010000 $.......$...<... + 0020 03e00008 c4200000 24840005 24840006 ..... ..$...$... + 0030 3c020000 03e00008 24420001 3c010000 <.......$B..<... + 0040 03e00008 c4200008 00000000 00000000 ..... .......... +Contents of section .rodata: + 0000 3f99999a 414570a4 3fa66666 00000000 ?...AEp.?.ff.... +Contents of section .options: + 0000 01200000 00000000 80000006 00000000 . .............. + 0010 00000011 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000016 00000000 00000011 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/pascal.objdump b/tools/asm-processor/tests/pascal.objdump new file mode 100644 index 0000000..cb3740f --- /dev/null +++ b/tools/asm-processor/tests/pascal.objdump @@ -0,0 +1,137 @@ + +tests/pascal.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 000000e0 .text +00000000 l d .rodata 00000030 .rodata +00000000 l d .data 00000010 .data +00000000 l d .bss 00000010 .bss +00000000 l O .bss 00000000 $dat +00000000 g F .text 0000000c foo +000000d0 g F .text 0000000c bar +0000000c g F .text 00000004 test +00000048 g F .text 00000004 test2 +0000008c g F .text 00000044 test3 +00000000 *UND* 00000000 get +00000000 *UND* 00000000 put +00000000 *UND* 00000000 pascal_close +00000000 *UND* 00000000 fflush +00000000 *UND* 00000000 filesize +00000000 *UND* 00000000 curpos +00000000 *UND* 00000000 seek +00000000 *UND* 00000000 eof +00000000 *UND* 00000000 eoln +00000000 *UND* 00000000 page +00000000 *UND* 00000000 reset +00000000 *UND* 00000000 rewrite +00000000 *UND* 00000000 cos +00000000 *UND* 00000000 exp +00000000 *UND* 00000000 sqrt +00000000 *UND* 00000000 log +00000000 *UND* 00000000 atan +00000000 *UND* 00000000 sin +00000000 *UND* 00000000 __random_float +00000000 *UND* 00000000 clock +00000000 *UND* 00000000 exit +00000000 *UND* 00000000 __date +00000000 *UND* 00000000 __time +00000000 *UND* 00000000 get_arg +00000000 *UND* 00000000 new +00000000 *UND* 00000000 dispose +00000000 *UND* 00000000 initfile +00000000 *UND* 00000000 peek_char +00000000 *UND* 00000000 next_char +00000000 *UND* 00000000 readln +00000000 *UND* 00000000 read_int64 +00000000 *UND* 00000000 read_card64 +00000000 *UND* 00000000 read_integer +00000000 *UND* 00000000 read_cardinal +00000000 *UND* 00000000 read_integer_range +00000000 *UND* 00000000 read_real +00000000 *UND* 00000000 read_double +00000000 *UND* 00000000 read_extended +00000000 *UND* 00000000 read_string +00000000 *UND* 00000000 read_enum +00000000 *UND* 00000000 read_char +00000000 *UND* 00000000 read_char_range +00000000 *UND* 00000000 read_boolean +00000000 *UND* 00000000 read_set +00000000 *UND* 00000000 writeln +00000000 *UND* 00000000 write_int64 +00000000 *UND* 00000000 write_card64 +00000000 *UND* 00000000 write_integer +00000000 *UND* 00000000 write_cardinal +00000000 *UND* 00000000 write_boolean +00000000 *UND* 00000000 write_char +00000000 *UND* 00000000 write_real +00000000 *UND* 00000000 write_double +00000000 *UND* 00000000 write_extended +00000000 *UND* 00000000 write_string +00000000 *UND* 00000000 write_enum +00000000 *UND* 00000000 write_set +00000000 *UND* 00000000 caseerror +00000000 *UND* 00000000 __pc_nloc_goto +00000000 *UND* 00000000 memcpy +00000000 *UND* 00000000 __in_range +00000000 *UND* 00000000 __ll_mul +00000000 *UND* 00000000 __ll_div +00000000 *UND* 00000000 __ull_div +00000000 *UND* 00000000 __ll_mod +00000000 *UND* 00000000 __ll_rem +00000000 *UND* 00000000 __ull_rem +00000000 *UND* 00000000 __ll_lshift +00000000 *UND* 00000000 __ll_rshift +00000000 *UND* 00000000 __ll_to_f +00000000 *UND* 00000000 __ull_to_f +00000000 *UND* 00000000 __ll_to_d +00000000 *UND* 00000000 __ull_to_d +00000000 *UND* 00000000 __f_ll_ll +00000000 *UND* 00000000 __f_to_ull +00000000 *UND* 00000000 __d_to_ll +00000000 *UND* 00000000 __d_to_ull +00000000 *UND* 00000000 round64 +00000000 *UND* 00000000 trunc64 +00000000 *UND* 00000000 max64 +00000000 *UND* 00000000 min64 +00000000 *UND* 00000000 abs64 +00000000 *UND* 00000000 odd64 +00000000 *UND* 00000000 trapNaN +00000000 *UND* 00000008 input +00000000 *UND* 00000008 output +00000000 *UND* 00000008 err +00000000 *UND* 00000008 __Argc + + +RELOCATION RECORDS FOR [.text]: (none) + +RELOCATION RECORDS FOR [.rodata]: (none) + +Contents of section .text: + 0000 00041080 03e00008 00441023 27bdffe8 .........D.#'... + 0010 18a00009 afa00004 8fae0004 008e7821 ..............x! + 0020 a1e00000 8fb80004 27190001 0325082a ........'....%.* + 0030 1420fff9 afb90004 10000001 00000000 . .............. + 0040 03e00008 27bd0018 00000000 00000000 ....'........... + 0050 00000000 00000000 00000000 00000000 ................ + 0060 00000000 00000000 00000000 00000000 ................ + 0070 00000000 00000000 00000000 00000000 ................ + 0080 00000000 00000000 00000000 00000000 ................ + 0090 00000000 00000000 00000000 00000000 ................ + 00a0 00000000 00000000 00000000 00000000 ................ + 00b0 00000000 00000000 00000000 00000000 ................ + 00c0 00000000 00000000 00000000 00000000 ................ + 00d0 00041080 03e00008 00441023 00000000 .........D.#.... +Contents of section .rodata: + 0000 00123123 00456456 00789789 00000001 ..1#.EdV.x...... + 0010 3ff19999 9999999a 00000002 00000003 ?............... + 0020 4000cccc cccccccd 00000000 00000000 @............... +Contents of section .data: + 0000 00002323 00003434 00000000 00000000 ..##..44........ +Contents of section .options: + 0000 01200000 00000000 80004016 00000000 . ........@..... + 0010 000000f0 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 a300c036 00000000 000000f0 00000000 ...6............ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/pascal.p b/tools/asm-processor/tests/pascal.p new file mode 100644 index 0000000..0886126 --- /dev/null +++ b/tools/asm-processor/tests/pascal.p @@ -0,0 +1,95 @@ +{ COMPILE-FLAGS: -O2 } + +function foo(x: integer): integer; +begin + foo := x * 3 +end; + +GLOBAL_ASM( +.section .data +.word 0x2323 + +.late_rodata +.word 0x123123 +.word 0x456456 +.word 0x789789 +.text +glabel test +/* 000090 00400090 27BDFFF8 */ addiu $sp, $sp, -24 +/* 000094 00400094 18A00009 */ blez $a1, .L004000BC +/* 000098 00400098 AFA00004 */ sw $zero, 4($sp) +.L0040009C: +/* 00009C 0040009C 8FAE0004 */ lw $t6, 4($sp) +/* 0000A0 004000A0 008E7821 */ addu $t7, $a0, $t6 +/* 0000A4 004000A4 A1E00000 */ sb $zero, ($t7) +/* 0000A8 004000A8 8FB80004 */ lw $t8, 4($sp) +/* 0000AC 004000AC 27190001 */ addiu $t9, $t8, 1 +/* 0000B0 004000B0 0325082A */ slt $at, $t9, $a1 +/* 0000B4 004000B4 1420FFF9 */ bnez $at, .L0040009C +/* 0000B8 004000B8 AFB90004 */ sw $t9, 4($sp) +.L004000BC: +/* 0000BC 004000BC 10000001 */ b .L004000C4 +/* 0000C0 004000C0 00000000 */ nop +.L004000C4: +/* 0000C4 004000C4 03E00008 */ jr $ra +/* 0000C8 004000C8 27BD0008 */ addiu $sp, $sp, 24 +) + +GLOBAL_ASM( +.section .data +.word 0x3434 + +.late_rodata +.word 0x1 +.double 1.1 +.word 0x2, 0x3 +.text +glabel test2 +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +) + +GLOBAL_ASM( +.late_rodata +.double 2.1 +.text +glabel test3 +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +) + +function bar(x: integer): integer; +begin + return x * 3 +end; + diff --git a/tools/asm-processor/tests/static-global.c b/tools/asm-processor/tests/static-global.c new file mode 100644 index 0000000..a7f6169 --- /dev/null +++ b/tools/asm-processor/tests/static-global.c @@ -0,0 +1,33 @@ +// COMPILE-FLAGS: -O2 +// ASMP-FLAGS: --convert-statics=global +static int xtext(int a, int b, int c); +const int rodata1[] = {1}; +static const int rodata2[] = {2}; +int data1[] = {3}; +static int data2[] = {4}; +int bss1; +static int bss2; + +GLOBAL_ASM( +glabel bar +lui $a0, %hi(rodata2) +lw $a0, %lo(rodata2)($a0) +lui $a1, %hi(data2) +lw $a1, %lo(data2)($a0) +lui $a2, %hi(bss2) +lw $a2, %lo(bss2)($a0) +jal xtext +nop +jr $ra +nop +nop +nop +) + +static int xtext(int a, int b, int c) { + return 1; +} + +void baz(void) { + xtext(bss2, rodata2[0], data2[0]); +} diff --git a/tools/asm-processor/tests/static-global.objdump b/tools/asm-processor/tests/static-global.objdump new file mode 100644 index 0000000..a519e4f --- /dev/null +++ b/tools/asm-processor/tests/static-global.objdump @@ -0,0 +1,58 @@ + +tests/static-global.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000080 .text +00000000 l d .rodata 00000010 .rodata +00000000 l d .data 00000010 .data +00000000 l d .bss 00000010 .bss +00000000 g O .rodata 00000004 rodata1 +00000000 g O .data 00000004 data1 +00000000 g O .bss 00000004 bss1 +00000044 g F .text 00000034 baz +00000000 g F .text 00000030 bar +00000004 g O .rodata 00000000 rodata2 +00000004 g O .data 00000000 data2 +00000004 g O .bss 00000000 bss2 +00000030 g F .text 00000000 xtext + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +0000004c R_MIPS_HI16 .bss +00000064 R_MIPS_LO16 .bss +00000050 R_MIPS_HI16 .rodata +0000005c R_MIPS_LO16 .rodata +00000054 R_MIPS_HI16 .data +00000058 R_MIPS_LO16 .data +00000060 R_MIPS_26 .text +00000000 R_MIPS_HI16 rodata2 +00000004 R_MIPS_LO16 rodata2 +00000008 R_MIPS_HI16 data2 +0000000c R_MIPS_LO16 data2 +00000010 R_MIPS_HI16 bss2 +00000014 R_MIPS_LO16 bss2 +00000018 R_MIPS_26 xtext + + +Contents of section .text: + 0000 3c040000 8c840000 3c050000 8c850000 <.......<....... + 0010 3c060000 8c860000 0c000000 00000000 <............... + 0020 03e00008 00000000 00000000 00000000 ................ + 0030 afa40000 afa50004 afa60008 03e00008 ................ + 0040 24020001 27bdffe8 afbf0014 3c040000 $...'.......<... + 0050 3c050000 3c060000 8cc60004 8ca50004 <...<........... + 0060 0c00000c 8c840004 8fbf0014 27bd0018 ............'... + 0070 03e00008 00000000 00000000 00000000 ................ +Contents of section .rodata: + 0000 00000001 00000002 00000000 00000000 ................ +Contents of section .data: + 0000 00000003 00000004 00000000 00000000 ................ +Contents of section .options: + 0000 01200000 00000000 a0000074 00000000 . .........t.... + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 a0000074 00000000 00000000 00000000 ...t............ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/static.c b/tools/asm-processor/tests/static.c new file mode 100644 index 0000000..c265021 --- /dev/null +++ b/tools/asm-processor/tests/static.c @@ -0,0 +1,32 @@ +// COMPILE-FLAGS: -O2 +static int xtext(int a, int b, int c); +const int rodata1[] = {1}; +static const int rodata2[] = {2}; +int data1[] = {3}; +static int data2[] = {4}; +int bss1; +static int bss2; + +GLOBAL_ASM( +glabel bar +lui $a0, %hi(rodata2) +lw $a0, %lo(rodata2)($a0) +lui $a1, %hi(data2) +lw $a1, %lo(data2)($a0) +lui $a2, %hi(bss2) +lw $a2, %lo(bss2)($a0) +jal xtext +nop +jr $ra +nop +nop +nop +) + +static int xtext(int a, int b, int c) { + return 1; +} + +void baz(void) { + xtext(bss2, rodata2[0], data2[0]); +} diff --git a/tools/asm-processor/tests/static.objdump b/tools/asm-processor/tests/static.objdump new file mode 100644 index 0000000..dd62c94 --- /dev/null +++ b/tools/asm-processor/tests/static.objdump @@ -0,0 +1,58 @@ + +tests/static.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000080 .text +00000000 l d .rodata 00000010 .rodata +00000000 l d .data 00000010 .data +00000000 l d .bss 00000010 .bss +00000004 l O .rodata 00000000 rodata2 +00000004 l O .data 00000000 data2 +00000004 l O .bss 00000000 bss2 +00000030 l F .text 00000000 xtext +00000000 g O .rodata 00000004 rodata1 +00000000 g O .data 00000004 data1 +00000000 g O .bss 00000004 bss1 +00000044 g F .text 00000034 baz +00000000 g F .text 00000030 bar + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +0000004c R_MIPS_HI16 .bss +00000064 R_MIPS_LO16 .bss +00000050 R_MIPS_HI16 .rodata +0000005c R_MIPS_LO16 .rodata +00000054 R_MIPS_HI16 .data +00000058 R_MIPS_LO16 .data +00000060 R_MIPS_26 .text +00000000 R_MIPS_HI16 rodata2 +00000004 R_MIPS_LO16 rodata2 +00000008 R_MIPS_HI16 data2 +0000000c R_MIPS_LO16 data2 +00000010 R_MIPS_HI16 bss2 +00000014 R_MIPS_LO16 bss2 +00000018 R_MIPS_26 xtext + + +Contents of section .text: + 0000 3c040000 8c840000 3c050000 8c850000 <.......<....... + 0010 3c060000 8c860000 0c000000 00000000 <............... + 0020 03e00008 00000000 00000000 00000000 ................ + 0030 afa40000 afa50004 afa60008 03e00008 ................ + 0040 24020001 27bdffe8 afbf0014 3c040000 $...'.......<... + 0050 3c050000 3c060000 8cc60004 8ca50004 <...<........... + 0060 0c00000c 8c840004 8fbf0014 27bd0018 ............'... + 0070 03e00008 00000000 00000000 00000000 ................ +Contents of section .rodata: + 0000 00000001 00000002 00000000 00000000 ................ +Contents of section .data: + 0000 00000003 00000004 00000000 00000000 ................ +Contents of section .options: + 0000 01200000 00000000 a0000074 00000000 . .........t.... + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 a0000074 00000000 00000000 00000000 ...t............ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/test1.c b/tools/asm-processor/tests/test1.c new file mode 100644 index 0000000..475555a --- /dev/null +++ b/tools/asm-processor/tests/test1.c @@ -0,0 +1,71 @@ + +GLOBAL_ASM( +.rdata +.word 0x1212 +) + +GLOBAL_ASM( +.late_rodata +.word 0x123123 +.text +glabel test +/* 000090 00400090 27BDFFF8 */ addiu $sp, $sp, -24 +/* 000094 00400094 18A00009 */ blez $a1, .L004000BC +/* 000098 00400098 AFA00004 */ sw $zero, 4($sp) +.L0040009C: +/* 00009C 0040009C 8FAE0004 */ lw $t6, 4($sp) +/* 0000A0 004000A0 008E7821 */ addu $t7, $a0, $t6 +/* 0000A4 004000A4 A1E00000 */ sb $zero, ($t7) +/* 0000A8 004000A8 8FB80004 */ lw $t8, 4($sp) +/* 0000AC 004000AC 27190001 */ addiu $t9, $t8, 1 +/* 0000B0 004000B0 0325082A */ slt $at, $t9, $a1 +/* 0000B4 004000B4 1420FFF9 */ bnez $at, .L0040009C +/* 0000B8 004000B8 AFB90004 */ sw $t9, 4($sp) +.L004000BC: +/* 0000BC 004000BC 10000001 */ b .L004000C4 +/* 0000C0 004000C0 00000000 */ nop +.L004000C4: +/* 0000C4 004000C4 03E00008 */ jr $ra +/* 0000C8 004000C8 27BD0008 */ addiu $sp, $sp, 24 +) + +char bss1[3]; +GLOBAL_ASM( +.bss +bss2: +.space 3 +) +char bss3[3]; +char bss4[3]; +const int rodata1[2] = {1}; +extern int some_rodata; + +unsigned g(float, int); +unsigned f(void) { + return g(1.1f, some_rodata); +} + +GLOBAL_ASM( +.rdata +glabel some_rodata +.word 0x1313 +.text +.late_rodata +.word 0x321321 +.text +glabel g +/* 0000C0 004000C0 27BDFFE8 */ addiu $sp, $sp, -0x18 +/* 0000C4 004000C4 AFBF0014 */ sw $ra, 0x14($sp) +/* 0000C8 004000C8 240E0004 */ addiu $t6, $zero, 4 +/* 0000CC 004000CC 3C010041 */ lui $at, %hi(D_410100) +/* 0000D0 004000D0 AC2E0100 */ sw $t6, %lo(D_410100)($at) +/* 0000D4 004000D4 0C10002C */ jal func_004000B0 +/* 0000D8 004000D8 00000000 */ nop +/* 0000DC 004000DC 10000001 */ b .L004000E4 +/* 0000E0 004000E0 00000000 */ nop +.L004000E4: +/* 0000E4 004000E4 8FBF0014 */ lw $ra, 0x14($sp) +/* 0000E8 004000E8 27BD0018 */ addiu $sp, $sp, 0x18 +/* 0000EC 004000EC 03E00008 */ jr $ra +/* 0000F0 004000F0 00000000 */ nop +) diff --git a/tools/asm-processor/tests/test1.objdump b/tools/asm-processor/tests/test1.objdump new file mode 100644 index 0000000..b97ae46 --- /dev/null +++ b/tools/asm-processor/tests/test1.objdump @@ -0,0 +1,54 @@ + +tests/test1.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 000000b0 .text +00000000 l d .rodata 00000020 .rodata +00000000 l d .bss 00000010 .bss +00000000 g O .bss 00000003 bss1 +00000008 g O .bss 00000003 bss3 +0000000c g O .bss 00000003 bss4 +00000004 g O .rodata 00000008 rodata1 +0000003c g F .text 0000003c f +00000000 g F .text 0000003c test +0000000c g .rodata 00000000 some_rodata +00000078 g F .text 00000004 g +00000000 *UND* 00000000 D_410100 +00000000 *UND* 00000000 func_004000B0 + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +00000044 R_MIPS_HI16 .rodata +00000054 R_MIPS_LO16 .rodata +00000048 R_MIPS_HI16 some_rodata +0000004c R_MIPS_LO16 some_rodata +00000050 R_MIPS_26 g +00000084 R_MIPS_HI16 D_410100 +00000088 R_MIPS_LO16 D_410100 +0000008c R_MIPS_26 func_004000B0 + + +Contents of section .text: + 0000 27bdffe8 18a00009 afa00004 8fae0004 '............... + 0010 008e7821 a1e00000 8fb80004 27190001 ..x!........'... + 0020 0325082a 1420fff9 afb90004 10000001 .%.*. .......... + 0030 00000000 03e00008 27bd0018 27bdffe8 ........'...'... + 0040 afbf0014 3c010000 3c050000 8ca50000 ....<...<....... + 0050 0c000000 c42c0014 10000003 00000000 .....,.......... + 0060 10000001 00000000 8fbf0014 27bd0018 ............'... + 0070 03e00008 00000000 27bdffe8 afbf0014 ........'....... + 0080 240e0004 3c010000 ac2e0000 0c000000 $...<........... + 0090 00000000 10000001 00000000 8fbf0014 ................ + 00a0 27bd0018 03e00008 00000000 00000000 '............... +Contents of section .rodata: + 0000 00001212 00000001 00000000 00001313 ................ + 0010 00123123 3f8ccccd 00321321 00000000 ..1#?....2.!.... +Contents of section .options: + 0000 01200000 00000000 a0000022 00000000 . .........".... + 0010 00001010 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 a300c032 00000000 00001010 00000000 ...2............ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/test2.c b/tools/asm-processor/tests/test2.c new file mode 100644 index 0000000..15969fe --- /dev/null +++ b/tools/asm-processor/tests/test2.c @@ -0,0 +1,69 @@ +const char buf1[1] = {1}; +float func1(void) { + "func1"; + return 0.1f; +} +const char buf2[1] = {2}; +void func2(void) { + *(volatile float*)0 = -3.5792360305786133f; + *(volatile float*)0 = -3.5792362689971924f; + // "func2"; + // return 0.2f; +} +const char buf3[1] = {3}; +int func3(int x) { + switch(x) { + case 0: + return 1; + case 1: + return 2; + case 2: + return 3; + case 3: + return 4; + case 4: + return 5; + case 5: + return 4; + case 6: + return 4; + case 7: + return 4; + default: + return 3; + } +} + +#if 1 +GLOBAL_ASM( +.rdata +.word 0x66756e63 # func +.word 0x34000000 # 4\0\0\0 +.word jumptarget + 4 + +.late_rodata +glabel rv +.word 0x3e4ccccd # 0.2f +.word jumptarget + 8 + +.text +glabel func4 +lui $at, %hi(rv) +glabel jumptarget +jr $ra +lwc1 $f0, %lo(rv)($at) +jr $ra +nop +jr $ra +nop +jr $ra +nop +jr $ra +nop +) +#else +float func4(void) { + "func4"; + return 0.2f; +} +#endif diff --git a/tools/asm-processor/tests/test2.objdump b/tools/asm-processor/tests/test2.objdump new file mode 100644 index 0000000..276ed9f --- /dev/null +++ b/tools/asm-processor/tests/test2.objdump @@ -0,0 +1,76 @@ + +tests/test2.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 000000f0 .text +00000000 l d .rodata 00000060 .rodata +00000000 g O .rodata 00000001 buf1 +00000000 g F .text 0000001c func1 +0000000c g O .rodata 00000001 buf2 +0000001c g F .text 00000028 func2 +00000010 g O .rodata 00000001 buf3 +00000044 g F .text 0000007c func3 +000000c4 g F .text 00000000 jumptarget +000000c0 g F .text 0000000c func4 +0000004c g .rodata 00000000 rv + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +00000000 R_MIPS_HI16 .rodata +00000008 R_MIPS_LO16 .rodata +0000001c R_MIPS_HI16 .rodata +00000020 R_MIPS_LO16 .rodata +00000028 R_MIPS_HI16 .rodata +0000002c R_MIPS_LO16 .rodata +00000054 R_MIPS_HI16 .rodata +0000005c R_MIPS_LO16 .rodata +000000c0 R_MIPS_HI16 rv +000000c8 R_MIPS_LO16 rv + + +RELOCATION RECORDS FOR [.rodata]: +OFFSET TYPE VALUE +0000002c R_MIPS_32 .text +00000030 R_MIPS_32 .text +00000034 R_MIPS_32 .text +00000038 R_MIPS_32 .text +0000003c R_MIPS_32 .text +00000040 R_MIPS_32 .text +00000044 R_MIPS_32 .text +00000048 R_MIPS_32 .text +0000001c R_MIPS_32 jumptarget +00000050 R_MIPS_32 jumptarget + + +Contents of section .text: + 0000 3c010000 03e00008 c4200020 03e00008 <........ . .... + 0010 00000000 03e00008 00000000 3c010000 ............<... + 0020 c4240024 e4040000 3c010000 c4260028 .$.$....<....&.( + 0030 e4060000 03e00008 00000000 03e00008 ................ + 0040 00000000 2c810008 10200017 00000000 ....,.... ...... + 0050 00047080 3c010000 002e0821 8c2e002c ..p.<......!..., + 0060 01c00008 00000000 03e00008 24020001 ............$... + 0070 03e00008 24020002 03e00008 24020003 ....$.......$... + 0080 03e00008 24020004 03e00008 24020005 ....$.......$... + 0090 03e00008 24020004 03e00008 24020004 ....$.......$... + 00a0 03e00008 24020004 03e00008 24020003 ....$.......$... + 00b0 03e00008 00000000 03e00008 00000000 ................ + 00c0 3c010000 03e00008 c4200000 03e00008 <........ ...... + 00d0 00000000 03e00008 00000000 03e00008 ................ + 00e0 00000000 03e00008 00000000 00000000 ................ +Contents of section .rodata: + 0000 01000000 66756e63 31000000 02000000 ....func1....... + 0010 03000000 66756e63 34000000 00000004 ....func4....... + 0020 3dcccccd c0651234 c0651235 00000068 =....e.4.e.5...h + 0030 00000070 00000078 00000080 00000088 ...p...x........ + 0040 00000090 00000098 000000a0 3e4ccccd ............>L.. + 0050 00000008 00000000 00000000 00000000 ................ +Contents of section .options: + 0000 01200000 00000000 80004016 00000000 . ........@..... + 0010 00000051 00000000 00000000 00007ff0 ...Q............ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80004016 00000000 00000051 00000000 ..@........Q.... + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/test3.c b/tools/asm-processor/tests/test3.c new file mode 100644 index 0000000..d74bb4c --- /dev/null +++ b/tools/asm-processor/tests/test3.c @@ -0,0 +1,70 @@ + +GLOBAL_ASM( +.rdata +.word 321321 +.text +glabel test +/* 000090 00400090 27BDFFF8 */ addiu $sp, $sp, -24 +/* 000094 00400094 18A00009 */ blez $a1, .L004000BC +/* 000098 00400098 AFA00004 */ sw $zero, 4($sp) +.L0040009C: +/* 00009C 0040009C 8FAE0004 */ lw $t6, 4($sp) +/* 0000A0 004000A0 008E7821 */ addu $t7, $a0, $t6 +/* 0000A4 004000A4 A1E00000 */ sb $zero, ($t7) +/* 0000A8 004000A8 8FB80004 */ lw $t8, 4($sp) +/* 0000AC 004000AC 27190001 */ addiu $t9, $t8, 1 +/* 0000B0 004000B0 0325082A */ slt $at, $t9, $a1 +/* 0000B4 004000B4 1420FFF9 */ bnez $at, .L0040009C +/* 0000B8 004000B8 AFB90004 */ sw $t9, 4($sp) +.L004000BC: +/* 0000BC 004000BC 10000001 */ b .L004000C4 +/* 0000C0 004000C0 00000000 */ nop +.L004000C4: +/* 0000C4 004000C4 03E00008 */ jr $ra +/* 0000C8 004000C8 27BD0008 */ addiu $sp, $sp, 24 +) + +// static -> no symbols +// bss +char globalBuf[4]; +const char constBuf[4]; + +// data +char globalBufInit[4] = {1}; + +// rodata +const char constBufInit[4] = {1}; +const char constBufInit2[1] = {2}; +const char constBufInit3[1] = {3}; + +unsigned g(void); +unsigned f(void) { + // aligns to 4 or 8 byte boundary (char -> 4, double -> 8) + double x = 5.1; + float y = 5.2f; + float z = 5.3f; + "Hello "; + "World"; + return g(); +} + +GLOBAL_ASM( +.rdata +.word 123123 +.text +glabel g +/* 0000C0 004000C0 27BDFFE8 */ addiu $sp, $sp, -0x18 +/* 0000C4 004000C4 AFBF0014 */ sw $ra, 0x14($sp) +/* 0000C8 004000C8 240E0004 */ addiu $t6, $zero, 4 +/* 0000CC 004000CC 3C010041 */ lui $at, %hi(D_410100) +/* 0000D0 004000D0 AC2E0100 */ sw $t6, %lo(D_410100)($at) +/* 0000D4 004000D4 0C10002C */ jal func_004000B0 +/* 0000D8 004000D8 00000000 */ nop +/* 0000DC 004000DC 10000001 */ b .L004000E4 +/* 0000E0 004000E0 00000000 */ nop +.L004000E4: +/* 0000E4 004000E4 8FBF0014 */ lw $ra, 0x14($sp) +/* 0000E8 004000E8 27BD0018 */ addiu $sp, $sp, 0x18 +/* 0000EC 004000EC 03E00008 */ jr $ra +/* 0000F0 004000F0 00000000 */ nop +) diff --git a/tools/asm-processor/tests/test3.objdump b/tools/asm-processor/tests/test3.objdump new file mode 100644 index 0000000..bfa7f6e --- /dev/null +++ b/tools/asm-processor/tests/test3.objdump @@ -0,0 +1,64 @@ + +tests/test3.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 000000d0 .text +00000000 l d .rodata 00000040 .rodata +00000000 l d .data 00000010 .data +00000000 l d .bss 00000010 .bss +00000000 g O .bss 00000004 globalBuf +00000004 g O .bss 00000004 constBuf +00000000 g O .data 00000004 globalBufInit +00000004 g O .rodata 00000004 constBufInit +00000008 g O .rodata 00000001 constBufInit2 +0000000c g O .rodata 00000001 constBufInit3 +0000003c g F .text 00000054 f +00000000 g F .text 00000004 test +00000090 g F .text 00000004 g +00000000 *UND* 00000000 D_410100 +00000000 *UND* 00000000 func_004000B0 + + +RELOCATION RECORDS FOR [.text]: +OFFSET TYPE VALUE +00000044 R_MIPS_HI16 .rodata +00000048 R_MIPS_LO16 .rodata +00000050 R_MIPS_HI16 .rodata +00000054 R_MIPS_LO16 .rodata +0000005c R_MIPS_HI16 .rodata +00000060 R_MIPS_LO16 .rodata +00000068 R_MIPS_26 g +0000009c R_MIPS_HI16 D_410100 +000000a0 R_MIPS_LO16 D_410100 +000000a4 R_MIPS_26 func_004000B0 + + +Contents of section .text: + 0000 27bdffe8 18a00009 afa00004 8fae0004 '............... + 0010 008e7821 a1e00000 8fb80004 27190001 ..x!........'... + 0020 0325082a 1420fff9 afb90004 10000001 .%.*. .......... + 0030 00000000 03e00008 27bd0018 27bdffd8 ........'...'... + 0040 afbf0014 3c010000 d4240028 f7a40020 ....<....$.(... + 0050 3c010000 c4260030 e7a6001c 3c010000 <....&.0....<... + 0060 c4280034 e7a80018 0c000000 00000000 .(.4............ + 0070 10000003 00000000 10000001 00000000 ................ + 0080 8fbf0014 27bd0028 03e00008 00000000 ....'..(........ + 0090 27bdffe8 afbf0014 240e0004 3c010000 '.......$...<... + 00a0 ac2e0000 0c000000 00000000 10000001 ................ + 00b0 00000000 8fbf0014 27bd0018 03e00008 ........'....... + 00c0 00000000 00000000 00000000 00000000 ................ +Contents of section .rodata: + 0000 0004e729 01000000 02000000 03000000 ...)............ + 0010 48656c6c 6f202000 576f726c 64000000 Hello .World... + 0020 0001e0f3 00000000 40146666 66666666 ........@.ffffff + 0030 40a66666 40a9999a 00000000 00000000 @.ff@........... +Contents of section .data: + 0000 01000000 00000000 00000000 00000000 ................ +Contents of section .options: + 0000 01200000 00000000 a0000002 00000000 . .............. + 0010 00000170 00000000 00000000 00007ff0 ...p............ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 a300c032 00000000 00000170 00000000 ...2.......p.... + 0010 00000000 00007ff0 ........ diff --git a/tools/decompress_baserom.py b/tools/decompress_baserom.py new file mode 100755 index 0000000..bf910b9 --- /dev/null +++ b/tools/decompress_baserom.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 + +import hashlib, io, struct, sys +from pathlib import Path +import argparse + +from libyaz0 import decompress + +FILE_TABLE_OFFSET = { + "jp": 0x19D40, + "cn": 0x21D80, +} + +VERSIONS_MD5S = { + "jp": "d7ae64f2f47a9fa3f87686a3c5ce09af", + "cn": "af83e0cf36298e62e9eb2eb8c89aa710", +} + +description = "Convert a rom that uses dmadata to an uncompressed one." + +parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawTextHelpFormatter) +parser.add_argument("-v", "--version", help="Version of the game to extract.", default="jp") + +args = parser.parse_args() + +BASEROM_PATH = Path(f"baserom.{args.version}.z64") +UNCOMPRESSED_PATH = Path(f"baserom_uncompressed.{args.version}.z64") + + +Version = args.version + +file_table_offset = FILE_TABLE_OFFSET[Version] +correct_str_hash = VERSIONS_MD5S[Version] + + + +def round_up(n,shift): + mod = 1 << shift + return (n + mod - 1) >> shift << shift + +def as_word(b, off=0): + return struct.unpack(">I", b[off:off+4])[0] + +def as_word_list(b): + return [i[0] for i in struct.iter_unpack(">I", b)] + +def calc_crc(rom_data, cic_type): + start = 0x1000 + end = 0x101000 + + unsigned_long = lambda i: i & 0xFFFFFFFF + rol = lambda i, b: unsigned_long(i << b) | (i >> (-b & 0x1F)) + + if cic_type == 6101 or cic_type == 6102: + seed = 0xF8CA4DDC + elif cic_type == 6103: + seed = 0xA3886759 + elif cic_type == 6105: + seed = 0xDF26F436 + elif cic_type == 6106: + seed = 0x1FEA617A + else: + assert False, f"Unknown cic type: {cic_type}" + + t1 = t2 = t3 = t4 = t5 = t6 = seed + + for pos in range(start, end, 4): + d = as_word(rom_data, pos) + r = rol(d, d & 0x1F) + + t6d = unsigned_long(t6 + d) + if t6d < t6: + t4 = unsigned_long(t4 + 1) + t6 = t6d + t3 ^= d + t5 = unsigned_long(t5 + r) + + if t2 > d: + t2 ^= r + else: + t2 ^= t6 ^ d + + if cic_type == 6105: + t1 = unsigned_long(t1 + (as_word(rom_data, 0x0750 + (pos & 0xFF)) ^ d)) + else: + t1 = unsigned_long(t1 + (t5 ^ d)) + + chksum = [0,0] + + if cic_type == 6103: + chksum[0] = unsigned_long((t6 ^ t4) + t3) + chksum[1] = unsigned_long((t5 ^ t2) + t1) + elif cic_type == 6106: + chksum[0] = unsigned_long((t6 * t4) + t3) + chksum[1] = unsigned_long((t5 * t2) + t1) + else: + chksum[0] = t6 ^ t4 ^ t3 + chksum[1] = t5 ^ t2 ^ t1 + + return struct.pack(">II", chksum[0], chksum[1]) + +def read_dmadata_entry(addr): + return as_word_list(fileContent[addr:addr+0x10]) + +def read_dmadata(start): + dmadata = [] + addr = start + entry = read_dmadata_entry(addr) + i = 0 + while any([e != 0 for e in entry]): + # print(f"0x{addr:08X} " + str([f"{e:08X}" for e in entry])) + dmadata.append(entry) + addr += 0x10 + i += 1 + entry = read_dmadata_entry(addr) + # print(f"0x{addr:08X} " + str([f"{e:08X}" for e in entry])) + return dmadata + +def update_crc(decompressed): + print("Recalculating crc...") + new_crc = calc_crc(decompressed.getbuffer(), 6105) + + decompressed.seek(0x10) + decompressed.write(new_crc) + return decompressed + +def decompress_rom(dmadata_addr, dmadata): + rom_segments = {} # vrom start : data s.t. len(data) == vrom_end - vrom_start + new_dmadata = bytearray() # new dmadata: {vrom start , vrom end , vrom start , 0} + + decompressed = io.BytesIO(b"") + + for v_start, v_end, p_start, p_end in dmadata: + if p_start == 0xFFFFFFFF and p_end == 0xFFFFFFFF: + new_dmadata.extend(struct.pack(">IIII", v_start, v_end, p_start, p_end)) + continue + if p_end == 0: # uncompressed + rom_segments.update({v_start : fileContent[p_start:p_start + v_end - v_start]}) + else: # compressed + rom_segments.update({v_start : decompress(fileContent[p_start:p_end])}) + new_dmadata.extend(struct.pack(">IIII", v_start, v_end, v_start, 0)) + + # write rom segments to vaddrs + for vrom_st,data in rom_segments.items(): + decompressed.seek(vrom_st) + decompressed.write(data) + # write new dmadata + decompressed.seek(dmadata_addr) + decompressed.write(new_dmadata) + # pad to size + padding_end = round_up(dmadata[-1][1], 14) + decompressed.seek(padding_end-1) + decompressed.write(bytearray([0])) + # re-calculate crc + return update_crc(decompressed) + + +def get_str_hash(byte_array): + return str(hashlib.md5(byte_array).hexdigest()) + +# If the baserom exists and is correct, we don't need to change anything +if UNCOMPRESSED_PATH.exists(): + with UNCOMPRESSED_PATH.open(mode="rb") as file: + fileContent = bytearray(file.read()) + if get_str_hash(fileContent) == correct_str_hash: + print("Found valid baserom - exiting early") + sys.exit(0) + +# Determine if we have a ROM file +romFileName = BASEROM_PATH +if BASEROM_PATH.with_suffix(".z64").exists(): + romFileName = BASEROM_PATH.with_suffix(".z64") +elif BASEROM_PATH.with_suffix(".n64").exists(): + romFileName = BASEROM_PATH.with_suffix(".n64") +elif BASEROM_PATH.with_suffix(".v64").exists(): + romFileName = BASEROM_PATH.with_suffix(".v64") +else: + print(f"Error: Could not find {BASEROM_PATH}/.n64/.v64") + sys.exit(1) + +# Read in the original ROM +print(f"File '{str(romFileName)}' found.") +with romFileName.open(mode="rb") as file: + fileContent = bytearray(file.read()) + +fileContentLen = len(fileContent) + +# Check if ROM needs to be byte/word swapped +# Little-endian +if fileContent[0] == 0x40: + # Word Swap ROM + print("ROM needs to be word swapped...") + words = str(int(fileContentLen/4)) + little_byte_format = "<" + words + "I" + big_byte_format = ">" + words + "I" + tmp = struct.unpack_from(little_byte_format, fileContent, 0) + struct.pack_into(big_byte_format, fileContent, 0, *tmp) + + print("Word swapping done.") + +# Byte-swapped +elif fileContent[0] == 0x37: + # Byte Swap ROM + print("ROM needs to be byte swapped...") + halfwords = str(int(fileContentLen/2)) + little_byte_format = "<" + halfwords + "H" + big_byte_format = ">" + halfwords + "H" + tmp = struct.unpack_from(little_byte_format, fileContent, 0) + struct.pack_into(big_byte_format, fileContent, 0, *tmp) + + print("Byte swapping done.") + +dmadata = read_dmadata(file_table_offset) +# Decompress +if any([b != 0 for b in fileContent[file_table_offset + 0xAC:file_table_offset + 0xAC + 0x4]]): + print("Decompressing rom...") + fileContent = decompress_rom(file_table_offset, dmadata).getbuffer() + print(f"{len(fileContent):X}") + +padding_start = round_up(dmadata[-1][1], 12) +padding_end = round_up(dmadata[-1][1], 14) +print(f"Padding from {padding_start:X} to {padding_end:X}...") +for i in range(padding_start,padding_end): + fileContent[i] = 0xFF + +# Check to see if the ROM is a "vanilla" ROM +str_hash = get_str_hash(bytearray(fileContent)) +if str_hash != correct_str_hash: + print("Error: Expected a hash of " + correct_str_hash + " but got " + str_hash + ". " + + "The baserom has probably been tampered, find a new one") + sys.exit(1) + +# Write out our new ROM +print(f"Writing new ROM {UNCOMPRESSED_PATH}.") +with UNCOMPRESSED_PATH.open("wb") as file: + file.write(bytes(fileContent)) + +print("Done!") diff --git a/tools/fado/.clang-format b/tools/fado/.clang-format new file mode 100644 index 0000000..c7b900f --- /dev/null +++ b/tools/fado/.clang-format @@ -0,0 +1,23 @@ +IndentWidth: 4 +Language: Cpp +UseTab: Never +ColumnLimit: 120 +PointerAlignment: Left +BreakBeforeBraces: Attach +SpaceAfterCStyleCast: false +Cpp11BracedListStyle: false +IndentCaseLabels: true +BinPackArguments: true +BinPackParameters: true +AlignAfterOpenBracket: Align +AlignOperands: true +BreakBeforeTernaryOperators: true +BreakBeforeBinaryOperators: None +AllowShortBlocksOnASingleLine: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: false +AlignEscapedNewlines: Left +AlignTrailingComments: true +SortIncludes: false diff --git a/tools/fado/.gitignore b/tools/fado/.gitignore new file mode 100644 index 0000000..83b9486 --- /dev/null +++ b/tools/fado/.gitignore @@ -0,0 +1,56 @@ +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +# Custom +build/ +.vscode diff --git a/tools/fado/.gitrepo b/tools/fado/.gitrepo new file mode 100644 index 0000000..2e5a361 --- /dev/null +++ b/tools/fado/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme +; +[subrepo] + remote = git@github.com:EllipticEllipsis/fado.git + branch = master + commit = 8d896ee97d565508755584803c409fc33bb0c953 + parent = f07748a7a0efa944186240ee136389d32b682ba6 + method = merge + cmdver = 0.4.3 diff --git a/tools/fado/LICENSE b/tools/fado/LICENSE new file mode 100644 index 0000000..0ad25db --- /dev/null +++ b/tools/fado/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +<https://www.gnu.org/licenses/>. diff --git a/tools/fado/Makefile b/tools/fado/Makefile new file mode 100644 index 0000000..135f512 --- /dev/null +++ b/tools/fado/Makefile @@ -0,0 +1,74 @@ +DEBUG ?= 0 +LLD ?= 0 +ASAN ?= 0 +EXPERIMENTAL?= 0 + +ELF := fado.elf + +CC := $(shell ./find_program.sh gcc clang clang-[0-9][0-9] clang-[0-9]) +LD := $(shell ./find_program.sh ld ld.lld ld.lld-*) +INC := -I include -I lib +WARNINGS := -Wall -Wextra -Wpedantic -Wshadow -Werror=implicit-function-declaration -Wvla -Wno-unused-function +CFLAGS := -std=c11 +LDFLAGS := + +ifeq ($(DEBUG),0) + OPTFLAGS := -O2 + CFLAGS += -Werror +else + OPTFLAGS := -O0 -g3 -DDEBUG_ON +endif + +ifneq ($(ASAN),0) + CFLAGS += -fsanitize=address -fsanitize=pointer-compare -fsanitize=pointer-subtract -fsanitize=undefined +endif + +ifneq ($(LLD),0) + LDFLAGS += -fuse-ld=lld +else +ifneq ($(LD),ld) + LDFLAGS += -fuse-ld=lld +endif +endif + +ifneq ($(EXPERIMENTAL),0) + CFLAGS += -DEXPERIMENTAL +endif + +# GCC is too stupid to be trusted with these warnings +ifeq ($(CC),gcc) + WARNINGS += -Wno-implicit-fallthrough -Wno-maybe-uninitialized +endif + +SRC_DIRS := $(shell find src -type d) +C_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.c)) +H_FILES := $(foreach dir,$(INC),$(wildcard $(dir)/*.h)) +O_FILES := $(foreach f,$(C_FILES:.c=.o),build/$f) + +LIB_DIRS := $(shell find lib -type d) +# exclude test file since we don't want it +C_LIB_FILES := $(filter-out lib/vc_vector/vc_vector_test.c, $(foreach dir,$(LIB_DIRS),$(wildcard $(dir)/*.c))) +O_LIB_FILES := $(foreach f,$(C_LIB_FILES:.c=.o),build/$f) + +# Main targets +all: $(ELF) + +clean: + $(RM) -r build $(ELF) + +format: + clang-format-11 -i $(C_FILES) $(H_FILES) lib/fairy/* + +.PHONY: all clean format + +# create build directories +$(shell mkdir -p $(foreach dir,$(SRC_DIRS),build/$(dir)) $(foreach dir,$(LIB_DIRS),build/$(dir))) + +$(ELF): $(O_FILES) $(O_LIB_FILES) + $(CC) $(INC) $(WARNINGS) $(CFLAGS) $(OPTFLAGS) $(LDFLAGS) -o $@ $^ + +build/%.o: %.c $(H_FILES) + $(CC) -c $(INC) $(WARNINGS) $(CFLAGS) $(OPTFLAGS) -o $@ $< + +build/lib/%.o: lib/%.c + $(CC) -c $(INC) $(WARNINGS) $(CFLAGS) $(OPTFLAGS) -o $@ $< diff --git a/tools/fado/README.md b/tools/fado/README.md new file mode 100644 index 0000000..61b32d6 --- /dev/null +++ b/tools/fado/README.md @@ -0,0 +1,77 @@ +# fado +*Fairy-Assisted (relocations for) Decomplied Overlays* +<!-- Nice backronym... --> + +Contains +- **Fairy** a library for reading relocatable MIPS ELF object files (big-endian, suitable for Nintendo 64 games) +- **Fado** a program for generating the `.ovl`/relocation section for Zelda64 overlay files +- **Mido** an automatic dependency file generator + +Compatible with both IDO and GCC (although [see below](N_B)). Both ordinary MIPS REL sections and RELA sections are now supported. + +Output format is the standard "Zelda64" .ovl section, with the relocs divided by section, as used by +- *The Legend of Zelda: Ocarina of Time* (all Nintendo 64/Gamecube/iQue releases) +- *The Legend of Zelda: Majora's Mask* (all Nintendo 64/Gamecube releases) + +In theory it will also work for other Nintendo 64 games that use this system, such as *Yoshi's Story*, but has yet to be tested with these. + + +## Explanation + +The overlay relocation sections used by Zelda64 is described [here](z64_relocation_section_format.md). Fado will produce a `.ovl` section compatible with this format, although as noted there, some compilers need persuasion to produce compatible objects. + + +## How to use + +Compile by running `make`. + +A standalone invocation of Fado would look something like + +```sh +./fado.elf z_en_hs2.o -n ovl_En_Hs2 -o ovl_En_Hs2_reloc.s +``` +This takes as input the compiled object file from the C file (e.g. [this one](https://github.com/zeldaret/oot/blob/eadc477187888e1ae078d021b4a00b1366f0c9a4/src/overlays/actors/ovl_En_Hs2/z_en_hs2.c)), the name of the overlay (`ovl_En_Hs2`) and will output an assembly file `ovl_En_Hs2_reloc.s` containing the relocation section. An example output is included in the repo [here](ovl_En_Hs_reloc.s). Fado will print information from the object file to assist with debugging, by splitting relocs by section, and for each, printing the type, offset, and associated symbol (or section if static): + +```mips +# TEXT RELOCS +.word 0x45000084 # R_MIPS_HI16 0x000084 .data +.word 0x4600008C # R_MIPS_LO16 0x00008C .data +.word 0x450000B4 # R_MIPS_HI16 0x0000B4 .rodata +.word 0x460000BC # R_MIPS_LO16 0x0000BC .rodata +.word 0x450000C0 # R_MIPS_HI16 0x0000C0 func_80A6F1A4 +.word 0x460000C4 # R_MIPS_LO16 0x0000C4 func_80A6F1A4 +``` + +If invoking in a makefile, you will probably want to generate these from a predefined filelist, and with the appropriate dependencies. [The Ocarina of Time decomp repository](http://github.com/zeldaret/oot) contains an example of how to do this using a supplementary program to parse the `spec` format. + +More information can be obtained by running + +```sh +./fado.elf --help +``` + +which contains information on the various options, such as automatic dependency file generation, etc. + + +## N.B. + +- Fado expects the linker script to output symbols for the section sizes, and for them to be declared separately, in the format + +``` +_SEGMENTNAMESegmentSECTIONSize +``` + +e.g. + +``` +_ovl_En_Hs2SegmentTextSize +``` + +etc. + +- By default Fado expects sections to be 0x10-aligned, as is usual for IDO. Some versions of GCC like to align sections to smaller widths, which Fado will handle appropriately, but the linker script must also address this, and at least the default settings seem unable to size the sections correctly due ot placing `fill`s in the wrong places. For now it is recommended to manually align sections to 0x10 if the compiler does not automatically. + - The experimental flag `--alignment`/`-a` can be passed to Fado, and it will use the alignment declared by each section in the elf file instead of padding them to 0x10 bytes, It should be noted this option has not been fully tested because currently we don't have any linker script tool that can properly address the incorrect placing of `fill`s. Fado must be rebuilt passing `EXPERIMENTAL=1` to be able to use this flag. + +- To prevent GCC producing non-compliant HI/LOs, you must pass *both* of the following compiler flags: `-mno-explicit-relocs -mno-split-addresses`. See [here](z64_relocation_section_format.md#hilo) for more details. + +- It is recommended, though not strictly required, that `-fno-merge-constants` is used for GCC, to avoid unpredictable section sizes, and comply with the Zelda64 relocation format's expectation of at most one rodata section. See [here](z64_relocation_section_format.md#rodata) for more details. diff --git a/tools/fado/find_program.sh b/tools/fado/find_program.sh new file mode 100755 index 0000000..77eb53c --- /dev/null +++ b/tools/fado/find_program.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +for i in "${@:2}" + do + RESULT=$(IFS=:;find $PATH -maxdepth 1 -name "$i" -print -quit 2> /dev/null | grep -o '[^/]*$') + COUNT=$(echo "$RESULT" | wc -c) + + if [ $COUNT -gt 1 ] + then + echo $RESULT + exit 0 + fi + done + +echo $1 +exit 0 diff --git a/tools/fado/include/fado.h b/tools/fado/include/fado.h new file mode 100644 index 0000000..6044d0a --- /dev/null +++ b/tools/fado/include/fado.h @@ -0,0 +1,8 @@ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#pragma once + +#include <stdio.h> + +void Fado_Relocs(FILE* outputFile, int inputFilesCount, FILE** inputFiles, const char* ovlName); +// void Fado_WriteRelocFile(FILE* outputFile, FILE** inputFiles, int inputFilesCount); diff --git a/tools/fado/include/help.h b/tools/fado/include/help.h new file mode 100644 index 0000000..35e8f59 --- /dev/null +++ b/tools/fado/include/help.h @@ -0,0 +1,25 @@ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#pragma once + +#include <getopt.h> +#include <unistd.h> + +typedef struct { + struct option longOpt; + char* helpArg; + char* helpMsg; +} OptInfo; + +typedef struct { + char* helpArg; + char* helpMsg; +} PosArgInfo; + +/* Formatting sizes used by Help_PrintHelp. Change them before calling Help_PrintHelp to use custom values */ +extern size_t helpTextWidth; +extern size_t helpDtIndent; +extern size_t helpDdIndent; + +void Help_PrintHelp(const char* prologue, size_t posArgCount, const PosArgInfo* posArgInfo, size_t optCount, + const OptInfo* optInfo, const char* epilogue); diff --git a/tools/fado/include/macros.h b/tools/fado/include/macros.h new file mode 100644 index 0000000..e201f8d --- /dev/null +++ b/tools/fado/include/macros.h @@ -0,0 +1,23 @@ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#pragma once + +#include "vc_vector/vc_vector.h" + +/* C macros */ +#define ARRAY_COUNT(arr) (signed long long)(sizeof(arr) / sizeof(arr[0])) +#define ARRAY_COUNTU(arr) (unsigned long long)(sizeof(arr) / sizeof(arr[0])) + +#define ALIGN(val, align) (((val) + (align - 1)) / (align) * (align)) + +/* Mathematical macros */ +#define ABS(x) ((x) < 0 ? -(x) : (x)) + +#define CLAMP(x, min, max) ((x) < (min) ? (min) : (x) > (max) ? (max) : (x)) +#define CLAMP_MAX(x, max) ((x) > (max) ? (max) : (x)) +#define CLAMP_MIN(x, min) ((x) < (min) ? (min) : (x)) +#define MEDIAN3(a1, a2, a3) \ + ((a2 >= a1) ? ((a3 >= a2) ? a2 : ((a1 >= a3) ? a1 : a3)) : ((a2 >= a3) ? a2 : ((a3 >= a1) ? a1 : a3))) + +/* vc_vector macros - really these should go in vc_vector.h, but not much choice without touching the library files */ +#define VC_FOREACH(i, v) for (i = vc_vector_begin(v); i != vc_vector_end(v); i = vc_vector_next(v, i)) diff --git a/tools/fado/include/mido.h b/tools/fado/include/mido.h new file mode 100644 index 0000000..b40d8a4 --- /dev/null +++ b/tools/fado/include/mido.h @@ -0,0 +1,6 @@ +#pragma once + +#include <stdio.h> +#include "vc_vector/vc_vector.h" + +int Mido_WriteDependencyFile(FILE* dependencyFile, const char* relocFile, vc_vector* inputFilesVector); diff --git a/tools/fado/include/mips_elf.h b/tools/fado/include/mips_elf.h new file mode 100644 index 0000000..122e98d --- /dev/null +++ b/tools/fado/include/mips_elf.h @@ -0,0 +1,659 @@ +/* This file contains enough of the ELF format definitions for MIPS to enable Fairy and Fado to work. More may need to + * be added later. The content is excerpted directly from elf.h from the GNU C Library, and therefore: */ +/* Copyright (C) 1995-2020 Free Software Foundation, Inc. */ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#ifndef MIPS_ELF_H +#define MIPS_ELF_H + +#include <stdint.h> + +/* Type for a 16-bit quantity. */ +typedef uint16_t Elf32_Half; + +/* Types for signed and unsigned 32-bit quantities. */ +typedef uint32_t Elf32_Word; +typedef int32_t Elf32_Sword; + +/* Types for signed and unsigned 64-bit quantities. */ +typedef uint64_t Elf32_Xword; +typedef int64_t Elf32_Sxword; + +/* Type of addresses. */ +typedef uint32_t Elf32_Addr; + +/* Type of file offsets. */ +typedef uint32_t Elf32_Off; + +/* Type for section indices, which are 16-bit quantities. */ +typedef uint16_t Elf32_Section; + +/* The ELF file header. This appears at the start of every ELF file. */ + +#define EI_NIDENT (16) + +typedef struct { + unsigned char e_ident[EI_NIDENT]; /* Magic number and other info */ + Elf32_Half e_type; /* Object file type */ + Elf32_Half e_machine; /* Architecture */ + Elf32_Word e_version; /* Object file version */ + Elf32_Addr e_entry; /* Entry point virtual address */ + Elf32_Off e_phoff; /* Program header table file offset */ + Elf32_Off e_shoff; /* Section header table file offset */ + Elf32_Word e_flags; /* Processor-specific flags */ + Elf32_Half e_ehsize; /* ELF header size in bytes */ + Elf32_Half e_phentsize; /* Program header table entry size */ + Elf32_Half e_phnum; /* Program header table entry count */ + Elf32_Half e_shentsize; /* Section header table entry size */ + Elf32_Half e_shnum; /* Section header table entry count */ + Elf32_Half e_shstrndx; /* Section header string table index */ +} Elf32_Ehdr; + +/* Fields in the e_ident array. The EI_* macros are indices into the + array. The macros under each EI_* macro are the values the byte + may have. */ + +#define EI_MAG0 0 /* File identification byte 0 index */ +#define ELFMAG0 0x7f /* Magic number byte 0 */ + +#define EI_MAG1 1 /* File identification byte 1 index */ +#define ELFMAG1 'E' /* Magic number byte 1 */ + +#define EI_MAG2 2 /* File identification byte 2 index */ +#define ELFMAG2 'L' /* Magic number byte 2 */ + +#define EI_MAG3 3 /* File identification byte 3 index */ +#define ELFMAG3 'F' /* Magic number byte 3 */ + +/* Conglomeration of the identification bytes, for easy testing as a word. */ +#define ELFMAG "\177ELF" +#define SELFMAG 4 + +#define EI_CLASS 4 /* File class byte index */ +#define ELFCLASSNONE 0 /* Invalid class */ +#define ELFCLASS32 1 /* 32-bit objects */ +#define ELFCLASS64 2 /* 64-bit objects */ +#define ELFCLASSNUM 3 + +#define EI_DATA 5 /* Data encoding byte index */ +#define ELFDATANONE 0 /* Invalid data encoding */ +#define ELFDATA2LSB 1 /* 2's complement, little endian */ +#define ELFDATA2MSB 2 /* 2's complement, big endian */ +#define ELFDATANUM 3 + +#define EI_VERSION 6 /* File version byte index */ + /* Value must be EV_CURRENT */ + +#define EI_OSABI 7 /* OS ABI identification */ +#define ELFOSABI_NONE 0 /* UNIX System V ABI */ +#define ELFOSABI_SYSV 0 /* Alias. */ +#define ELFOSABI_HPUX 1 /* HP-UX */ +#define ELFOSABI_NETBSD 2 /* NetBSD. */ +#define ELFOSABI_GNU 3 /* Object uses GNU ELF extensions. */ +#define ELFOSABI_LINUX ELFOSABI_GNU /* Compatibility alias. */ +#define ELFOSABI_SOLARIS 6 /* Sun Solaris. */ +#define ELFOSABI_AIX 7 /* IBM AIX. */ +#define ELFOSABI_IRIX 8 /* SGI Irix. */ +#define ELFOSABI_FREEBSD 9 /* FreeBSD. */ +#define ELFOSABI_TRU64 10 /* Compaq TRU64 UNIX. */ +#define ELFOSABI_MODESTO 11 /* Novell Modesto. */ +#define ELFOSABI_OPENBSD 12 /* OpenBSD. */ +#define ELFOSABI_ARM_AEABI 64 /* ARM EABI */ +#define ELFOSABI_ARM 97 /* ARM */ +#define ELFOSABI_STANDALONE 255 /* Standalone (embedded) application */ + +#define EI_ABIVERSION 8 /* ABI version */ + +#define EI_PAD 9 /* Byte index of padding bytes */ + +/* Legal values for e_type (object file type). */ + +#define ET_NONE 0 /* No file type */ +#define ET_REL 1 /* Relocatable file */ +#define ET_EXEC 2 /* Executable file */ +#define ET_DYN 3 /* Shared object file */ +#define ET_CORE 4 /* Core file */ +#define ET_NUM 5 /* Number of defined types */ +#define ET_LOOS 0xfe00 /* OS-specific range start */ +#define ET_HIOS 0xfeff /* OS-specific range end */ +#define ET_LOPROC 0xff00 /* Processor-specific range start */ +#define ET_HIPROC 0xffff /* Processor-specific range end */ + +/* Legal values for e_machine (architecture). */ + +#define EM_NONE 0 /* No machine */ +#define EM_M32 1 /* AT&T WE 32100 */ +#define EM_SPARC 2 /* SUN SPARC */ +#define EM_386 3 /* Intel 80386 */ +#define EM_68K 4 /* Motorola m68k family */ +#define EM_88K 5 /* Motorola m88k family */ +#define EM_IAMCU 6 /* Intel MCU */ +#define EM_860 7 /* Intel 80860 */ +#define EM_MIPS 8 /* MIPS R3000 big-endian */ +#define EM_S370 9 /* IBM System/370 */ +#define EM_MIPS_RS3_LE 10 /* MIPS R3000 little-endian */ +/* reserved 11-14 */ +#define EM_PARISC 15 /* HPPA */ +/* reserved 16 */ +#define EM_VPP500 17 /* Fujitsu VPP500 */ +#define EM_SPARC32PLUS 18 /* Sun's "v8plus" */ +#define EM_960 19 /* Intel 80960 */ +#define EM_PPC 20 /* PowerPC */ +#define EM_PPC64 21 /* PowerPC 64-bit */ +#define EM_S390 22 /* IBM S390 */ +#define EM_SPU 23 /* IBM SPU/SPC */ +/* reserved 24-35 */ +#define EM_V800 36 /* NEC V800 series */ +#define EM_FR20 37 /* Fujitsu FR20 */ +#define EM_RH32 38 /* TRW RH-32 */ +#define EM_RCE 39 /* Motorola RCE */ +#define EM_ARM 40 /* ARM */ +#define EM_FAKE_ALPHA 41 /* Digital Alpha */ +#define EM_SH 42 /* Hitachi SH */ +#define EM_SPARCV9 43 /* SPARC v9 64-bit */ +#define EM_TRICORE 44 /* Siemens Tricore */ +#define EM_ARC 45 /* Argonaut RISC Core */ +#define EM_H8_300 46 /* Hitachi H8/300 */ +#define EM_H8_300H 47 /* Hitachi H8/300H */ +#define EM_H8S 48 /* Hitachi H8S */ +#define EM_H8_500 49 /* Hitachi H8/500 */ +#define EM_IA_64 50 /* Intel Merced */ +#define EM_MIPS_X 51 /* Stanford MIPS-X */ +#define EM_COLDFIRE 52 /* Motorola Coldfire */ +#define EM_68HC12 53 /* Motorola M68HC12 */ +#define EM_MMA 54 /* Fujitsu MMA Multimedia Accelerator */ +#define EM_PCP 55 /* Siemens PCP */ +#define EM_NCPU 56 /* Sony nCPU embeeded RISC */ +#define EM_NDR1 57 /* Denso NDR1 microprocessor */ +#define EM_STARCORE 58 /* Motorola Start*Core processor */ +#define EM_ME16 59 /* Toyota ME16 processor */ +#define EM_ST100 60 /* STMicroelectronic ST100 processor */ +#define EM_TINYJ 61 /* Advanced Logic Corp. Tinyj emb.fam */ +#define EM_X86_64 62 /* AMD x86-64 architecture */ +#define EM_PDSP 63 /* Sony DSP Processor */ +#define EM_PDP10 64 /* Digital PDP-10 */ +#define EM_PDP11 65 /* Digital PDP-11 */ +#define EM_FX66 66 /* Siemens FX66 microcontroller */ +#define EM_ST9PLUS 67 /* STMicroelectronics ST9+ 8/16 mc */ +#define EM_ST7 68 /* STmicroelectronics ST7 8 bit mc */ +#define EM_68HC16 69 /* Motorola MC68HC16 microcontroller */ +#define EM_68HC11 70 /* Motorola MC68HC11 microcontroller */ +#define EM_68HC08 71 /* Motorola MC68HC08 microcontroller */ +#define EM_68HC05 72 /* Motorola MC68HC05 microcontroller */ +#define EM_SVX 73 /* Silicon Graphics SVx */ +#define EM_ST19 74 /* STMicroelectronics ST19 8 bit mc */ +#define EM_VAX 75 /* Digital VAX */ +#define EM_CRIS 76 /* Axis Communications 32-bit emb.proc */ +#define EM_JAVELIN 77 /* Infineon Technologies 32-bit emb.proc */ +#define EM_FIREPATH 78 /* Element 14 64-bit DSP Processor */ +#define EM_ZSP 79 /* LSI Logic 16-bit DSP Processor */ +#define EM_MMIX 80 /* Donald Knuth's educational 64-bit proc */ +#define EM_HUANY 81 /* Harvard University machine-independent object files */ +#define EM_PRISM 82 /* SiTera Prism */ +#define EM_AVR 83 /* Atmel AVR 8-bit microcontroller */ +#define EM_FR30 84 /* Fujitsu FR30 */ +#define EM_D10V 85 /* Mitsubishi D10V */ +#define EM_D30V 86 /* Mitsubishi D30V */ +#define EM_V850 87 /* NEC v850 */ +#define EM_M32R 88 /* Mitsubishi M32R */ +#define EM_MN10300 89 /* Matsushita MN10300 */ +#define EM_MN10200 90 /* Matsushita MN10200 */ +#define EM_PJ 91 /* picoJava */ +#define EM_OPENRISC 92 /* OpenRISC 32-bit embedded processor */ +#define EM_ARC_COMPACT 93 /* ARC International ARCompact */ +#define EM_XTENSA 94 /* Tensilica Xtensa Architecture */ +#define EM_VIDEOCORE 95 /* Alphamosaic VideoCore */ +#define EM_TMM_GPP 96 /* Thompson Multimedia General Purpose Proc */ +#define EM_NS32K 97 /* National Semi. 32000 */ +#define EM_TPC 98 /* Tenor Network TPC */ +#define EM_SNP1K 99 /* Trebia SNP 1000 */ +#define EM_ST200 100 /* STMicroelectronics ST200 */ +#define EM_IP2K 101 /* Ubicom IP2xxx */ +#define EM_MAX 102 /* MAX processor */ +#define EM_CR 103 /* National Semi. CompactRISC */ +#define EM_F2MC16 104 /* Fujitsu F2MC16 */ +#define EM_MSP430 105 /* Texas Instruments msp430 */ +#define EM_BLACKFIN 106 /* Analog Devices Blackfin DSP */ +#define EM_SE_C33 107 /* Seiko Epson S1C33 family */ +#define EM_SEP 108 /* Sharp embedded microprocessor */ +#define EM_ARCA 109 /* Arca RISC */ +#define EM_UNICORE 110 /* PKU-Unity & MPRC Peking Uni. mc series */ +#define EM_EXCESS 111 /* eXcess configurable cpu */ +#define EM_DXP 112 /* Icera Semi. Deep Execution Processor */ +#define EM_ALTERA_NIOS2 113 /* Altera Nios II */ +#define EM_CRX 114 /* National Semi. CompactRISC CRX */ +#define EM_XGATE 115 /* Motorola XGATE */ +#define EM_C166 116 /* Infineon C16x/XC16x */ +#define EM_M16C 117 /* Renesas M16C */ +#define EM_DSPIC30F 118 /* Microchip Technology dsPIC30F */ +#define EM_CE 119 /* Freescale Communication Engine RISC */ +#define EM_M32C 120 /* Renesas M32C */ +/* reserved 121-130 */ +#define EM_TSK3000 131 /* Altium TSK3000 */ +#define EM_RS08 132 /* Freescale RS08 */ +#define EM_SHARC 133 /* Analog Devices SHARC family */ +#define EM_ECOG2 134 /* Cyan Technology eCOG2 */ +#define EM_SCORE7 135 /* Sunplus S+core7 RISC */ +#define EM_DSP24 136 /* New Japan Radio (NJR) 24-bit DSP */ +#define EM_VIDEOCORE3 137 /* Broadcom VideoCore III */ +#define EM_LATTICEMICO32 138 /* RISC for Lattice FPGA */ +#define EM_SE_C17 139 /* Seiko Epson C17 */ +#define EM_TI_C6000 140 /* Texas Instruments TMS320C6000 DSP */ +#define EM_TI_C2000 141 /* Texas Instruments TMS320C2000 DSP */ +#define EM_TI_C5500 142 /* Texas Instruments TMS320C55x DSP */ +#define EM_TI_ARP32 143 /* Texas Instruments App. Specific RISC */ +#define EM_TI_PRU 144 /* Texas Instruments Prog. Realtime Unit */ +/* reserved 145-159 */ +#define EM_MMDSP_PLUS 160 /* STMicroelectronics 64bit VLIW DSP */ +#define EM_CYPRESS_M8C 161 /* Cypress M8C */ +#define EM_R32C 162 /* Renesas R32C */ +#define EM_TRIMEDIA 163 /* NXP Semi. TriMedia */ +#define EM_QDSP6 164 /* QUALCOMM DSP6 */ +#define EM_8051 165 /* Intel 8051 and variants */ +#define EM_STXP7X 166 /* STMicroelectronics STxP7x */ +#define EM_NDS32 167 /* Andes Tech. compact code emb. RISC */ +#define EM_ECOG1X 168 /* Cyan Technology eCOG1X */ +#define EM_MAXQ30 169 /* Dallas Semi. MAXQ30 mc */ +#define EM_XIMO16 170 /* New Japan Radio (NJR) 16-bit DSP */ +#define EM_MANIK 171 /* M2000 Reconfigurable RISC */ +#define EM_CRAYNV2 172 /* Cray NV2 vector architecture */ +#define EM_RX 173 /* Renesas RX */ +#define EM_METAG 174 /* Imagination Tech. META */ +#define EM_MCST_ELBRUS 175 /* MCST Elbrus */ +#define EM_ECOG16 176 /* Cyan Technology eCOG16 */ +#define EM_CR16 177 /* National Semi. CompactRISC CR16 */ +#define EM_ETPU 178 /* Freescale Extended Time Processing Unit */ +#define EM_SLE9X 179 /* Infineon Tech. SLE9X */ +#define EM_L10M 180 /* Intel L10M */ +#define EM_K10M 181 /* Intel K10M */ +/* reserved 182 */ +#define EM_AARCH64 183 /* ARM AARCH64 */ +/* reserved 184 */ +#define EM_AVR32 185 /* Amtel 32-bit microprocessor */ +#define EM_STM8 186 /* STMicroelectronics STM8 */ +#define EM_TILE64 187 /* Tileta TILE64 */ +#define EM_TILEPRO 188 /* Tilera TILEPro */ +#define EM_MICROBLAZE 189 /* Xilinx MicroBlaze */ +#define EM_CUDA 190 /* NVIDIA CUDA */ +#define EM_TILEGX 191 /* Tilera TILE-Gx */ +#define EM_CLOUDSHIELD 192 /* CloudShield */ +#define EM_COREA_1ST 193 /* KIPO-KAIST Core-A 1st gen. */ +#define EM_COREA_2ND 194 /* KIPO-KAIST Core-A 2nd gen. */ +#define EM_ARC_COMPACT2 195 /* Synopsys ARCompact V2 */ +#define EM_OPEN8 196 /* Open8 RISC */ +#define EM_RL78 197 /* Renesas RL78 */ +#define EM_VIDEOCORE5 198 /* Broadcom VideoCore V */ +#define EM_78KOR 199 /* Renesas 78KOR */ +#define EM_56800EX 200 /* Freescale 56800EX DSC */ +#define EM_BA1 201 /* Beyond BA1 */ +#define EM_BA2 202 /* Beyond BA2 */ +#define EM_XCORE 203 /* XMOS xCORE */ +#define EM_MCHP_PIC 204 /* Microchip 8-bit PIC(r) */ +/* reserved 205-209 */ +#define EM_KM32 210 /* KM211 KM32 */ +#define EM_KMX32 211 /* KM211 KMX32 */ +#define EM_EMX16 212 /* KM211 KMX16 */ +#define EM_EMX8 213 /* KM211 KMX8 */ +#define EM_KVARC 214 /* KM211 KVARC */ +#define EM_CDP 215 /* Paneve CDP */ +#define EM_COGE 216 /* Cognitive Smart Memory Processor */ +#define EM_COOL 217 /* Bluechip CoolEngine */ +#define EM_NORC 218 /* Nanoradio Optimized RISC */ +#define EM_CSR_KALIMBA 219 /* CSR Kalimba */ +#define EM_Z80 220 /* Zilog Z80 */ +#define EM_VISIUM 221 /* Controls and Data Services VISIUMcore */ +#define EM_FT32 222 /* FTDI Chip FT32 */ +#define EM_MOXIE 223 /* Moxie processor */ +#define EM_AMDGPU 224 /* AMD GPU */ +/* reserved 225-242 */ +#define EM_RISCV 243 /* RISC-V */ + +#define EM_BPF 247 /* Linux BPF -- in-kernel virtual machine */ +#define EM_CSKY 252 /* C-SKY */ + +#define EM_NUM 253 + +/* Old spellings/synonyms. */ + +#define EM_ARC_A5 EM_ARC_COMPACT + +/* If it is necessary to assign new unofficial EM_* values, please + pick large random numbers (0x8523, 0xa7f2, etc.) to minimize the + chances of collision with official or non-GNU unofficial values. */ + +#define EM_ALPHA 0x9026 + +/* Legal values for e_version (version). */ + +#define EV_NONE 0 /* Invalid ELF version */ +#define EV_CURRENT 1 /* Current version */ +#define EV_NUM 2 + +/* Section header. */ + +typedef struct { + Elf32_Word sh_name; /* Section name (string tbl index) */ + Elf32_Word sh_type; /* Section type */ + Elf32_Word sh_flags; /* Section flags */ + Elf32_Addr sh_addr; /* Section virtual addr at execution */ + Elf32_Off sh_offset; /* Section file offset */ + Elf32_Word sh_size; /* Section size in bytes */ + Elf32_Word sh_link; /* Link to another section */ + Elf32_Word sh_info; /* Additional section information */ + Elf32_Word sh_addralign; /* Section alignment */ + Elf32_Word sh_entsize; /* Entry size if section holds table */ +} Elf32_Shdr; + +/* Special section indices. */ + +#define SHN_UNDEF 0 /* Undefined section */ +#define SHN_LORESERVE 0xff00 /* Start of reserved indices */ +#define SHN_LOPROC 0xff00 /* Start of processor-specific */ +#define SHN_BEFORE 0xff00 /* Order section before all others (Solaris). */ +#define SHN_AFTER 0xff01 /* Order section after all others (Solaris). */ +#define SHN_HIPROC 0xff1f /* End of processor-specific */ +#define SHN_LOOS 0xff20 /* Start of OS-specific */ +#define SHN_HIOS 0xff3f /* End of OS-specific */ +#define SHN_ABS 0xfff1 /* Associated symbol is absolute */ +#define SHN_COMMON 0xfff2 /* Associated symbol is common */ +#define SHN_XINDEX 0xffff /* Index is in extra table. */ +#define SHN_HIRESERVE 0xffff /* End of reserved indices */ +/* Legal values for sh_type (section type). */ + +#define SHT_NULL 0 /* Section header table entry unused */ +#define SHT_PROGBITS 1 /* Program data */ +#define SHT_SYMTAB 2 /* Symbol table */ +#define SHT_STRTAB 3 /* String table */ +#define SHT_RELA 4 /* Relocation entries with addends */ +#define SHT_HASH 5 /* Symbol hash table */ +#define SHT_DYNAMIC 6 /* Dynamic linking information */ +#define SHT_NOTE 7 /* Notes */ +#define SHT_NOBITS 8 /* Program space with no data (bss) */ +#define SHT_REL 9 /* Relocation entries, no addends */ +#define SHT_SHLIB 10 /* Reserved */ +#define SHT_DYNSYM 11 /* Dynamic linker symbol table */ +#define SHT_INIT_ARRAY 14 /* Array of constructors */ +#define SHT_FINI_ARRAY 15 /* Array of destructors */ +#define SHT_PREINIT_ARRAY 16 /* Array of pre-constructors */ +#define SHT_GROUP 17 /* Section group */ +#define SHT_SYMTAB_SHNDX 18 /* Extended section indeces */ +#define SHT_NUM 19 /* Number of defined types. */ +#define SHT_LOOS 0x60000000 /* Start OS-specific. */ +#define SHT_GNU_ATTRIBUTES 0x6ffffff5 /* Object attributes. */ +#define SHT_GNU_HASH 0x6ffffff6 /* GNU-style hash table. */ +#define SHT_GNU_LIBLIST 0x6ffffff7 /* Prelink library list */ +#define SHT_CHECKSUM 0x6ffffff8 /* Checksum for DSO content. */ +#define SHT_LOSUNW 0x6ffffffa /* Sun-specific low bound. */ +#define SHT_SUNW_move 0x6ffffffa +#define SHT_SUNW_COMDAT 0x6ffffffb +#define SHT_SUNW_syminfo 0x6ffffffc +#define SHT_GNU_verdef 0x6ffffffd /* Version definition section. */ +#define SHT_GNU_verneed 0x6ffffffe /* Version needs section. */ +#define SHT_GNU_versym 0x6fffffff /* Version symbol table. */ +#define SHT_HISUNW 0x6fffffff /* Sun-specific high bound. */ +#define SHT_HIOS 0x6fffffff /* End OS-specific type */ +#define SHT_LOPROC 0x70000000 /* Start of processor-specific */ +#define SHT_HIPROC 0x7fffffff /* End of processor-specific */ +#define SHT_LOUSER 0x80000000 /* Start of application-specific */ +#define SHT_HIUSER 0x8fffffff /* End of application-specific */ + +/* Symbol table entry. */ + +typedef struct { + Elf32_Word st_name; /* Symbol name (string tbl index) */ + Elf32_Addr st_value; /* Symbol value */ + Elf32_Word st_size; /* Symbol size */ + unsigned char st_info; /* Symbol type and binding */ + unsigned char st_other; /* Symbol visibility */ + Elf32_Section st_shndx; /* Section index */ +} Elf32_Sym; + +/* How to extract and insert information held in the st_info field. */ + +#define ELF32_ST_BIND(val) (((unsigned char)(val)) >> 4) +#define ELF32_ST_TYPE(val) ((val)&0xf) +#define ELF32_ST_INFO(bind, type) (((bind) << 4) + ((type)&0xf)) + +/* Both Elf32_Sym and Elf64_Sym use the same one-byte st_info field. */ +#define ELF64_ST_BIND(val) ELF32_ST_BIND(val) +#define ELF64_ST_TYPE(val) ELF32_ST_TYPE(val) +#define ELF64_ST_INFO(bind, type) ELF32_ST_INFO((bind), (type)) + +/* Legal values for ST_BIND subfield of st_info (symbol binding). */ + +#define STB_LOCAL 0 /* Local symbol */ +#define STB_GLOBAL 1 /* Global symbol */ +#define STB_WEAK 2 /* Weak symbol */ +#define STB_NUM 3 /* Number of defined types. */ +#define STB_LOOS 10 /* Start of OS-specific */ +#define STB_GNU_UNIQUE 10 /* Unique symbol. */ +#define STB_HIOS 12 /* End of OS-specific */ +#define STB_LOPROC 13 /* Start of processor-specific */ +#define STB_HIPROC 15 /* End of processor-specific */ + +/* Legal values for ST_TYPE subfield of st_info (symbol type). */ + +#define STT_NOTYPE 0 /* Symbol type is unspecified */ +#define STT_OBJECT 1 /* Symbol is a data object */ +#define STT_FUNC 2 /* Symbol is a code object */ +#define STT_SECTION 3 /* Symbol associated with a section */ +#define STT_FILE 4 /* Symbol's name is file name */ +#define STT_COMMON 5 /* Symbol is a common data object */ +#define STT_TLS 6 /* Symbol is thread-local data object*/ +#define STT_NUM 7 /* Number of defined types. */ +#define STT_LOOS 10 /* Start of OS-specific */ +#define STT_GNU_IFUNC 10 /* Symbol is indirect code object */ +#define STT_HIOS 12 /* End of OS-specific */ +#define STT_LOPROC 13 /* Start of processor-specific */ +#define STT_HIPROC 15 /* End of processor-specific */ + +/* Symbol table indices are found in the hash buckets and chain table + of a symbol hash table section. This special index value indicates + the end of a chain, meaning no further symbols are found in that bucket. */ + +#define STN_UNDEF 0 /* End of a chain. */ + +/* How to extract and insert information held in the st_other field. */ + +#define ELF32_ST_VISIBILITY(o) ((o)&0x03) + +/* For ELF64 the definitions are the same. */ +#define ELF64_ST_VISIBILITY(o) ELF32_ST_VISIBILITY(o) + +/* Symbol visibility specification encoded in the st_other field. */ +#define STV_DEFAULT 0 /* Default symbol visibility rules */ +#define STV_INTERNAL 1 /* Processor specific hidden class */ +#define STV_HIDDEN 2 /* Sym unavailable in other modules */ +#define STV_PROTECTED 3 /* Not preemptible, not exported */ + +/* Relocation table entry without addend (in section of type SHT_REL). */ + +typedef struct { + Elf32_Addr r_offset; /* Address */ + Elf32_Word r_info; /* Relocation type and symbol index */ +} Elf32_Rel; + +/* Relocation table entry with addend (in section of type SHT_RELA). */ + +typedef struct { + Elf32_Addr r_offset; /* Address */ + Elf32_Word r_info; /* Relocation type and symbol index */ + Elf32_Sword r_addend; /* Addend */ +} Elf32_Rela; + +/* How to extract and insert information held in the r_info field. */ + +#define ELF32_R_SYM(val) ((val) >> 8) +#define ELF32_R_TYPE(val) ((val)&0xff) +#define ELF32_R_INFO(sym, type) (((sym) << 8) + ((type)&0xff)) + +/* MIPS R3000 specific definitions. */ + +/* Legal values for e_flags field of Elf32_Ehdr. */ + +#define EF_MIPS_NOREORDER 1 /* A .noreorder directive was used. */ +#define EF_MIPS_PIC 2 /* Contains PIC code. */ +#define EF_MIPS_CPIC 4 /* Uses PIC calling sequence. */ +#define EF_MIPS_XGOT 8 +#define EF_MIPS_64BIT_WHIRL 16 +#define EF_MIPS_ABI2 32 +#define EF_MIPS_ABI_ON32 64 +#define EF_MIPS_FP64 512 /* Uses FP64 (12 callee-saved). */ +#define EF_MIPS_NAN2008 1024 /* Uses IEEE 754-2008 NaN encoding. */ +#define EF_MIPS_ARCH 0xf0000000 /* MIPS architecture level. */ + +/* Legal values for MIPS architecture level. */ + +#define EF_MIPS_ARCH_1 0x00000000 /* -mips1 code. */ +#define EF_MIPS_ARCH_2 0x10000000 /* -mips2 code. */ +#define EF_MIPS_ARCH_3 0x20000000 /* -mips3 code. */ +#define EF_MIPS_ARCH_4 0x30000000 /* -mips4 code. */ +#define EF_MIPS_ARCH_5 0x40000000 /* -mips5 code. */ +#define EF_MIPS_ARCH_32 0x50000000 /* MIPS32 code. */ +#define EF_MIPS_ARCH_64 0x60000000 /* MIPS64 code. */ +#define EF_MIPS_ARCH_32R2 0x70000000 /* MIPS32r2 code. */ +#define EF_MIPS_ARCH_64R2 0x80000000 /* MIPS64r2 code. */ + +/* The following are unofficial names and should not be used. */ + +#define E_MIPS_ARCH_1 EF_MIPS_ARCH_1 +#define E_MIPS_ARCH_2 EF_MIPS_ARCH_2 +#define E_MIPS_ARCH_3 EF_MIPS_ARCH_3 +#define E_MIPS_ARCH_4 EF_MIPS_ARCH_4 +#define E_MIPS_ARCH_5 EF_MIPS_ARCH_5 +#define E_MIPS_ARCH_32 EF_MIPS_ARCH_32 +#define E_MIPS_ARCH_64 EF_MIPS_ARCH_64 + +/* Special section indices. */ + +#define SHN_MIPS_ACOMMON 0xff00 /* Allocated common symbols. */ +#define SHN_MIPS_TEXT 0xff01 /* Allocated test symbols. */ +#define SHN_MIPS_DATA 0xff02 /* Allocated data symbols. */ +#define SHN_MIPS_SCOMMON 0xff03 /* Small common symbols. */ +#define SHN_MIPS_SUNDEFINED 0xff04 /* Small undefined symbols. */ + +/* Legal values for sh_type field of Elf32_Shdr. */ + +#define SHT_MIPS_LIBLIST 0x70000000 /* Shared objects used in link. */ +#define SHT_MIPS_MSYM 0x70000001 +#define SHT_MIPS_CONFLICT 0x70000002 /* Conflicting symbols. */ +#define SHT_MIPS_GPTAB 0x70000003 /* Global data area sizes. */ +#define SHT_MIPS_UCODE 0x70000004 /* Reserved for SGI/MIPS compilers */ +#define SHT_MIPS_DEBUG 0x70000005 /* MIPS ECOFF debugging info. */ +#define SHT_MIPS_REGINFO 0x70000006 /* Register usage information. */ +#define SHT_MIPS_PACKAGE 0x70000007 +#define SHT_MIPS_PACKSYM 0x70000008 +#define SHT_MIPS_RELD 0x70000009 +#define SHT_MIPS_IFACE 0x7000000b +#define SHT_MIPS_CONTENT 0x7000000c +#define SHT_MIPS_OPTIONS 0x7000000d /* Miscellaneous options. */ +#define SHT_MIPS_SHDR 0x70000010 +#define SHT_MIPS_FDESC 0x70000011 +#define SHT_MIPS_EXTSYM 0x70000012 +#define SHT_MIPS_DENSE 0x70000013 +#define SHT_MIPS_PDESC 0x70000014 +#define SHT_MIPS_LOCSYM 0x70000015 +#define SHT_MIPS_AUXSYM 0x70000016 +#define SHT_MIPS_OPTSYM 0x70000017 +#define SHT_MIPS_LOCSTR 0x70000018 +#define SHT_MIPS_LINE 0x70000019 +#define SHT_MIPS_RFDESC 0x7000001a +#define SHT_MIPS_DELTASYM 0x7000001b +#define SHT_MIPS_DELTAINST 0x7000001c +#define SHT_MIPS_DELTACLASS 0x7000001d +#define SHT_MIPS_DWARF 0x7000001e /* DWARF debugging information. */ +#define SHT_MIPS_DELTADECL 0x7000001f +#define SHT_MIPS_SYMBOL_LIB 0x70000020 +#define SHT_MIPS_EVENTS 0x70000021 /* Event section. */ +#define SHT_MIPS_TRANSLATE 0x70000022 +#define SHT_MIPS_PIXIE 0x70000023 +#define SHT_MIPS_XLATE 0x70000024 +#define SHT_MIPS_XLATE_DEBUG 0x70000025 +#define SHT_MIPS_WHIRL 0x70000026 +#define SHT_MIPS_EH_REGION 0x70000027 +#define SHT_MIPS_XLATE_OLD 0x70000028 +#define SHT_MIPS_PDR_EXCEPTION 0x70000029 +#define SHT_MIPS_XHASH 0x7000002b + +/* Legal values for sh_flags field of Elf32_Shdr. */ + +#define SHF_MIPS_GPREL 0x10000000 /* Must be in global data area. */ +#define SHF_MIPS_MERGE 0x20000000 +#define SHF_MIPS_ADDR 0x40000000 +#define SHF_MIPS_STRINGS 0x80000000 +#define SHF_MIPS_NOSTRIP 0x08000000 +#define SHF_MIPS_LOCAL 0x04000000 +#define SHF_MIPS_NAMES 0x02000000 +#define SHF_MIPS_NODUPE 0x01000000 + +/* Symbol tables. */ + +/* MIPS specific values for `st_other'. */ +#define STO_MIPS_DEFAULT 0x0 +#define STO_MIPS_INTERNAL 0x1 +#define STO_MIPS_HIDDEN 0x2 +#define STO_MIPS_PROTECTED 0x3 +#define STO_MIPS_PLT 0x8 +#define STO_MIPS_SC_ALIGN_UNUSED 0xff + +/* MIPS specific values for `st_info'. */ +#define STB_MIPS_SPLIT_COMMON 13 + +/* MIPS relocs. */ + +#define R_MIPS_NONE 0 /* No reloc */ +#define R_MIPS_16 1 /* Direct 16 bit */ +#define R_MIPS_32 2 /* Direct 32 bit */ +#define R_MIPS_REL32 3 /* PC relative 32 bit */ +#define R_MIPS_26 4 /* Direct 26 bit shifted */ +#define R_MIPS_HI16 5 /* High 16 bit */ +#define R_MIPS_LO16 6 /* Low 16 bit */ +#define R_MIPS_GPREL16 7 /* GP relative 16 bit */ +#define R_MIPS_LITERAL 8 /* 16 bit literal entry */ +#define R_MIPS_GOT16 9 /* 16 bit GOT entry */ +#define R_MIPS_PC16 10 /* PC relative 16 bit */ +#define R_MIPS_CALL16 11 /* 16 bit GOT entry for function */ +#define R_MIPS_GPREL32 12 /* GP relative 32 bit */ + +#define R_MIPS_SHIFT5 16 +#define R_MIPS_SHIFT6 17 +#define R_MIPS_64 18 +#define R_MIPS_GOT_DISP 19 +#define R_MIPS_GOT_PAGE 20 +#define R_MIPS_GOT_OFST 21 +#define R_MIPS_GOT_HI16 22 +#define R_MIPS_GOT_LO16 23 +#define R_MIPS_SUB 24 +#define R_MIPS_INSERT_A 25 +#define R_MIPS_INSERT_B 26 +#define R_MIPS_DELETE 27 +#define R_MIPS_HIGHER 28 +#define R_MIPS_HIGHEST 29 +#define R_MIPS_CALL_HI16 30 +#define R_MIPS_CALL_LO16 31 +#define R_MIPS_SCN_DISP 32 +#define R_MIPS_REL16 33 +#define R_MIPS_ADD_IMMEDIATE 34 +#define R_MIPS_PJUMP 35 +#define R_MIPS_RELGOT 36 +#define R_MIPS_JALR 37 +#define R_MIPS_TLS_DTPMOD32 38 /* Module number 32 bit */ +#define R_MIPS_TLS_DTPREL32 39 /* Module-relative offset 32 bit */ +#define R_MIPS_TLS_DTPMOD64 40 /* Module number 64 bit */ +#define R_MIPS_TLS_DTPREL64 41 /* Module-relative offset 64 bit */ +#define R_MIPS_TLS_GD 42 /* 16 bit GOT offset for GD */ +#define R_MIPS_TLS_LDM 43 /* 16 bit GOT offset for LDM */ +#define R_MIPS_TLS_DTPREL_HI16 44 /* Module-relative offset, high 16 bits */ +#define R_MIPS_TLS_DTPREL_LO16 45 /* Module-relative offset, low 16 bits */ +#define R_MIPS_TLS_GOTTPREL 46 /* 16 bit GOT offset for IE */ +#define R_MIPS_TLS_TPREL32 47 /* TP-relative offset, 32 bit */ +#define R_MIPS_TLS_TPREL64 48 /* TP-relative offset, 64 bit */ +#define R_MIPS_TLS_TPREL_HI16 49 /* TP-relative offset, high 16 bits */ +#define R_MIPS_TLS_TPREL_LO16 50 /* TP-relative offset, low 16 bits */ +#define R_MIPS_GLOB_DAT 51 +#define R_MIPS_COPY 126 +#define R_MIPS_JUMP_SLOT 127 +/* Keep this the last entry. */ +#define R_MIPS_NUM 128 + +#endif /* MIPS_ELF_H */ diff --git a/tools/fado/lib/fairy/fairy.c b/tools/fado/lib/fairy/fairy.c new file mode 100644 index 0000000..8fc0903 --- /dev/null +++ b/tools/fado/lib/fairy/fairy.c @@ -0,0 +1,418 @@ +/** + * Functions for working with N64 ELF files. + */ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#include "fairy.h" + +#include <assert.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "vc_vector/vc_vector.h" +#include "macros.h" + +VerbosityLevel gVerbosity = VERBOSITY_NONE; +bool gUseElfAlignment = false; + +int Fairy_DebugPrintf(const char* file, int line, const char* func, VerbosityLevel level, const char* fmt, ...) { + if (gVerbosity >= level) { + int ret = 0; + va_list args; + va_start(args, fmt); + + if (gVerbosity >= VERBOSITY_DEBUG) { + ret += fprintf(stderr, "%s:%d:%s: ", file, line, func); + } + + ret += vfprintf(stderr, fmt, args); + va_end(args); + return ret; + } + return 0; +} + +/* Endian readers. MIPS is BE, so only need these */ +static Elf32_Half Fairy_ReadHalf(const uint8_t* data) { + return data[0] << 8 | data[1] << 0; +} + +static Elf32_Word Fairy_ReadWord(const uint8_t* data) { + return data[0] << 24 | data[1] << 16 | data[2] << 8 | data[3] << 0; +} + +static bool Fairy_VerifyMagic(const uint8_t* data) { + return (data[0] == 0x7F && data[1] == 'E' && data[2] == 'L' && data[3] == 'F'); +} + +static uint16_t Fairy_Swap16(uint16_t x) { + return ((x & 0xFF) << 0x8) | ((x & 0xFF00) >> 0x8); +} + +static uint32_t Fairy_Swap32(uint32_t x) { + return ((x & 0xFF) << 0x18) | ((x & 0xFF00) << 0x8) | ((x & 0xFF0000) >> 0x8) | ((x & 0xFF000000) >> 0x18); +} + +/* Both GCC and Clang define these, so we can avoid an endian header altogether */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define REEND16(x) Fairy_Swap16(x) +#define REEND32(x) Fairy_Swap32(x) +#else +#define REEND16(x) (x) +#define REEND32(x) (x) +#endif + +const char* Fairy_StringFromDefine(const FairyDefineString* dict, int define) { + size_t i; + for (i = 0; dict[i].string != NULL; i++) { + if (dict[i].define == define) { + return dict[i].string; + } + } + return NULL; +} + +/** + * Returns true if the string 'initial' is contained in the string 'string' + * 'initial' must be null-terminated, 'string' ideally is. + */ +bool Fairy_StartsWith(const char* string, const char* initial) { + char s; + char i; + do { + s = *string++; + i = *initial++; + if (i == '\0') { + return true; + } + } while (s == i); + return false; +} + +/* Reading functions */ + +/** + * Every reading function: + * - Returns the pointer to the struct + * - Takes the ouput struct or array as its first argument. This must be pre-allocated + * - Takes the input file as the second argument (At least until I am persuaded to read the whole file into RAM...) + * - The rest of the arguments are important information about the struct it is reading (offset and size, usually) + */ + +FairyFileHeader* Fairy_ReadFileHeader(FairyFileHeader* header, FILE* file) { + fseek(file, 0, SEEK_SET); + assert(fread(header, sizeof(char), 0x34, file) == 0x34); + + if (!Fairy_VerifyMagic(header->e_ident)) { + fprintf(stderr, "Not a valid ELF file.\n"); + return NULL; + } + + if (header->e_ident[EI_CLASS] != ELFCLASS32) { + fprintf(stderr, "Not a 32-bit ELF file.\n"); + return NULL; + } + + header->e_type = REEND16(header->e_type); + if (header->e_type != ET_REL) { + fprintf(stderr, "Not a relocatable object file.\n"); + return NULL; + } + + header->e_machine = REEND16(header->e_machine); + if (header->e_machine != EM_MIPS) { + fprintf(stderr, "Not a MIPS object file.\n"); + return NULL; + } + + header->e_version = REEND32(header->e_version); + header->e_entry = REEND32(header->e_entry); + header->e_phoff = REEND32(header->e_phoff); + header->e_shoff = REEND32(header->e_shoff); + header->e_flags = REEND32(header->e_flags); + header->e_ehsize = REEND16(header->e_ehsize); + header->e_phentsize = REEND16(header->e_phentsize); + header->e_phnum = REEND16(header->e_phnum); + header->e_shentsize = REEND16(header->e_shentsize); + header->e_shnum = REEND16(header->e_shnum); + header->e_shstrndx = REEND16(header->e_shstrndx); + + return header; +} + +/* tableOffset and number should be obtained from the file header */ +FairySecHeader* Fairy_ReadSectionTable(FairySecHeader* sectionTable, FILE* file, size_t tableOffset, size_t number) { + size_t entrySize = sizeof(FairySecHeader); + size_t tableSize = number * entrySize; + + fseek(file, tableOffset, SEEK_SET); + assert(fread(sectionTable, sizeof(char), tableSize, file) == tableSize); + + /* Since the section table happens to only have entries of width 4, we can byteswap it by pretending it is a raw + * uint32_t array */ + { + size_t i; + uint32_t* data = (uint32_t*)sectionTable; + for (i = 0; i < tableSize / sizeof(uint32_t); i++) { + data[i] = REEND32(data[i]); + } + } + + return sectionTable; +} + +size_t Fairy_ReadSymbolTable(FairySym** symbolTableOut, FILE* file, size_t tableOffset, size_t tableSize) { + size_t number = tableSize / sizeof(FairySym); + FairySym* symbolTable = malloc(tableSize); + + *symbolTableOut = NULL; + + if (symbolTable == NULL) { + return 0; + } + if (fseek(file, tableOffset, SEEK_SET) != 0 || fread(symbolTable, sizeof(char), tableSize, file) != tableSize) { + free(symbolTable); + return 0; + } + + /* Reend the variables that are wider than bytes */ + { + size_t i; + for (i = 0; i < number; i++) { + symbolTable[i].st_name = REEND32(symbolTable[i].st_name); + symbolTable[i].st_value = REEND32(symbolTable[i].st_value); + symbolTable[i].st_size = REEND32(symbolTable[i].st_size); + symbolTable[i].st_shndx = REEND16(symbolTable[i].st_shndx); + } + } + + *symbolTableOut = symbolTable; + return number; +} + +/* Can be used for both the section header string table and the strtab */ +char* Fairy_ReadStringTable(char* stringTable, FILE* file, size_t tableOffset, size_t tableSize) { + fseek(file, tableOffset, SEEK_SET); + assert(fread(stringTable, sizeof(char), tableSize, file) == tableSize); + return stringTable; +} + +/* offset and number are attained from the section table, the returned pointer must be freed */ +size_t Fairy_ReadRelocs(FairyRela** relocsOut, FILE* file, int type, size_t offset, size_t size) { + /* Final size of the relocation table, relocations of type SHT_REL need more space for extra addend of 0 */ + size_t finalSize = (type == SHT_REL) ? ((size * sizeof(FairyRela)) / sizeof(FairyRel)) : size; + void* readBuf = malloc(size); + FairyRela* relocTable = malloc(finalSize); + + *relocsOut = NULL; + + if (readBuf == NULL) { + return 0; + } + if (relocTable == NULL) { + free(readBuf); + return 0; + } + if (fseek(file, offset, SEEK_SET) != 0 || fread(readBuf, sizeof(char), size, file) != size) { + free(readBuf); + free(relocTable); + return 0; + } + + /* Reend the variables that are wider than bytes */ + { + size_t i; + uint32_t* data = (uint32_t*)readBuf; + for (i = 0; i < size / sizeof(uint32_t); i++) { + data[i] = REEND32(data[i]); + } + } + + /* Make the relocation table, for SHT_REL sections add an addend of 0 */ + if (type == SHT_REL) { + size_t i; + FairyRel* rel = (FairyRel*)readBuf; + + for (i = 0; i < size / sizeof(FairyRel); i++) { + relocTable[i].r_info = rel[i].r_info; + relocTable[i].r_offset = rel[i].r_offset; + relocTable[i].r_addend = 0; + } + } else { + memcpy(relocTable, readBuf, size); + } + free(readBuf); + + *relocsOut = relocTable; + return finalSize / sizeof(FairyRela); +} + +char* Fairy_GetSectionName(FairySecHeader* sectionTable, char* shstrtab, size_t index) { + return &shstrtab[sectionTable[index].sh_name]; +} + +/* Look up the index in the symbol table and return a pointer to the beginning of its string */ +char* Fairy_GetSymbolName(FairySym* symtab, char* strtab, size_t index) { + return &strtab[symtab[index].st_name]; +} + +/* FairyFileInfo functions */ + +void Fairy_InitFile(FairyFileInfo* fileInfo, FILE* file) { + FairyFileHeader fileHeader; + FairySecHeader* sectionTable; + char* shstrtab; + int i; + + assert(fileInfo != NULL); + assert(file != NULL); + + fileInfo->progBitsSections = vc_vector_create(3, sizeof(Elf32_Section), NULL); + for (i = 0; i < 3; i++) { + fileInfo->progBitsSizes[i] = 0; + } + Fairy_ReadFileHeader(&fileHeader, file); + + sectionTable = malloc(fileHeader.e_shnum * fileHeader.e_shentsize); + Fairy_ReadSectionTable(sectionTable, file, fileHeader.e_shoff, fileHeader.e_shnum); + + shstrtab = malloc(sectionTable[fileHeader.e_shstrndx].sh_size * sizeof(char)); + fseek(file, sectionTable[fileHeader.e_shstrndx].sh_offset, SEEK_SET); + assert(fread(shstrtab, sizeof(char), sectionTable[fileHeader.e_shstrndx].sh_size, file) == + sectionTable[fileHeader.e_shstrndx].sh_size); + + /* Search for the sections we need */ + { + size_t currentIndex; + FairySecHeader currentSection; + for (currentIndex = 0; currentIndex < 3; currentIndex++) { + fileInfo->relocTablesInfo[currentIndex].sectionData = NULL; + } + + for (currentIndex = 0; currentIndex < fileHeader.e_shnum; currentIndex++) { + size_t off = 0; + + currentSection = sectionTable[currentIndex]; + + switch (currentSection.sh_type) { + case SHT_PROGBITS: + assert(vc_vector_push_back(fileInfo->progBitsSections, ¤tIndex)); + + { + FairySection sectionType = FAIRY_SECTION_OTHER; + const char* sectionName = &shstrtab[currentSection.sh_name + 1]; + size_t alignedSize; + + /* Ignore the leading "." */ + if (strcmp(sectionName, "text") == 0) { + sectionType = FAIRY_SECTION_TEXT; + } else if (strcmp(sectionName, "data") == 0) { + sectionType = FAIRY_SECTION_DATA; + } else if (Fairy_StartsWith(sectionName, "rodata")) { /* May be several */ + sectionType = FAIRY_SECTION_RODATA; + } + + if (sectionType != FAIRY_SECTION_OTHER) { + if (gUseElfAlignment) { + /* Ensure the next file will start at its correct alignment */ + fileInfo->progBitsSizes[sectionType] = + ALIGN(fileInfo->progBitsSizes[sectionType], currentSection.sh_addralign); + + alignedSize = ALIGN(currentSection.sh_size, currentSection.sh_addralign); + + FAIRY_DEBUG_PRINTF("%s section alignment: 0x%X\n", sectionName, + currentSection.sh_addralign); + FAIRY_DEBUG_PRINTF("%s section size before align: 0x%X\n", sectionName, + currentSection.sh_size); + FAIRY_DEBUG_PRINTF("%s section size after align: 0x%X\n", sectionName, alignedSize); + + fileInfo->progBitsSizes[sectionType] += alignedSize; + } else { + fileInfo->progBitsSizes[sectionType] += ALIGN(currentSection.sh_size, 0x10); + } + + FAIRY_DEBUG_PRINTF("%s section size: 0x%X\n", sectionName, + fileInfo->progBitsSizes[sectionType]); + } + } + + break; + + case SHT_SYMTAB: + if (strcmp(&shstrtab[currentSection.sh_name + 1], "symtab") == 0) { + fileInfo->symtabInfo.sectionType = SHT_SYMTAB; + fileInfo->symtabInfo.sectionEntrySize = sizeof(FairySym); + fileInfo->symtabInfo.sectionEntryCount = + Fairy_ReadSymbolTable((FairySym**)&fileInfo->symtabInfo.sectionData, file, + currentSection.sh_offset, currentSection.sh_size); + } + break; + + case SHT_STRTAB: + if (strcmp(&shstrtab[currentSection.sh_name + 1], "strtab") == 0) { + FAIRY_DEBUG_PRINTF("%s", "strtab found\n"); + fileInfo->strtab = malloc(currentSection.sh_size); + Fairy_ReadStringTable(fileInfo->strtab, file, currentSection.sh_offset, currentSection.sh_size); + } + break; + + case SHT_RELA: + off += 1; + case SHT_REL: + off += 5; + /* This assumes only one reloc section of each name */ + // TODO: is this a problem? + { + FairySection relocSection = FAIRY_SECTION_OTHER; + + /* Ignore the first 5/6 chars, which will always be ".rel."/".rela." */ + if (strcmp(&shstrtab[currentSection.sh_name + off], "text") == 0) { + relocSection = FAIRY_SECTION_TEXT; + } else if (strcmp(&shstrtab[currentSection.sh_name + off], "data") == 0) { + relocSection = FAIRY_SECTION_DATA; + } else if (strcmp(&shstrtab[currentSection.sh_name + off], "rodata") == 0) { + relocSection = FAIRY_SECTION_RODATA; + } else { + break; + } + FAIRY_DEBUG_PRINTF("Found %s section\n", &shstrtab[currentSection.sh_name]); + + fileInfo->relocTablesInfo[relocSection].sectionType = SHT_RELA; + fileInfo->relocTablesInfo[relocSection].sectionEntrySize = sizeof(FairyRela); + fileInfo->relocTablesInfo[relocSection].sectionEntryCount = + Fairy_ReadRelocs((FairyRela**)&fileInfo->relocTablesInfo[relocSection].sectionData, file, + currentSection.sh_type, currentSection.sh_offset, currentSection.sh_size); + } + break; + + default: + break; + } + } + } + + free(sectionTable); + free(shstrtab); +} + +void Fairy_DestroyFile(FairyFileInfo* fileInfo) { + size_t i; + for (i = 0; i < ARRAY_COUNTU(fileInfo->relocTablesInfo); i++) { + if (fileInfo->relocTablesInfo[i].sectionData != NULL) { + FAIRY_DEBUG_PRINTF("Freeing reloc section %zd data\n", i); + free(fileInfo->relocTablesInfo[i].sectionData); + } + } + + vc_vector_release(fileInfo->progBitsSections); + + FAIRY_DEBUG_PRINTF("%s", "Freeing symtab data\n"); + free(fileInfo->symtabInfo.sectionData); + + FAIRY_DEBUG_PRINTF("%s", "Freeing strtab data\n"); + free(fileInfo->strtab); +} diff --git a/tools/fado/lib/fairy/fairy.h b/tools/fado/lib/fairy/fairy.h new file mode 100644 index 0000000..5a3cb3d --- /dev/null +++ b/tools/fado/lib/fairy/fairy.h @@ -0,0 +1,74 @@ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#pragma once + +#include <stddef.h> +#include <stdio.h> +#include "mips_elf.h" + +#include "vc_vector/vc_vector.h" + +#define FAIRY_DEF_STRING(prefix, x) \ + { prefix##x, #x } + +typedef enum { + VERBOSITY_NONE, + VERBOSITY_INFO, + VERBOSITY_DEBUG //, +} VerbosityLevel; + +extern VerbosityLevel gVerbosity; +extern bool gUseElfAlignment; + +typedef Elf32_Ehdr FairyFileHeader; +typedef Elf32_Shdr FairySecHeader; +typedef Elf32_Sym FairySym; +typedef Elf32_Rel FairyRel; +typedef Elf32_Rela FairyRela; + +typedef struct { + int define; + const char* string; +} FairyDefineString; + +typedef struct { + void* sectionData; + int sectionType; + size_t sectionEntryCount; + size_t sectionEntrySize; +} FairySectionInfo; + +typedef struct { + FairySectionInfo symtabInfo; + char* strtab; + Elf32_Word progBitsSizes[3]; + vc_vector* progBitsSections; + FairySectionInfo relocTablesInfo[3]; +} FairyFileInfo; + +typedef enum { + FAIRY_SECTION_TEXT, + FAIRY_SECTION_DATA, + FAIRY_SECTION_RODATA, + FAIRY_SECTION_OTHER //, +} FairySection; + +/* Prints debugging information to stderr. To be used via the macros. */ +int Fairy_DebugPrintf(const char* file, int line, const char* func, VerbosityLevel level, const char* fmt, ...); +#define FAIRY_INFO_PRINTF(fmt, ...) Fairy_DebugPrintf(__FILE__, __LINE__, __func__, VERBOSITY_INFO, fmt, __VA_ARGS__) +#define FAIRY_DEBUG_PRINTF(fmt, ...) Fairy_DebugPrintf(__FILE__, __LINE__, __func__, VERBOSITY_DEBUG, fmt, __VA_ARGS__) + +const char* Fairy_StringFromDefine(const FairyDefineString* dict, int define); +bool Fairy_StartsWith(const char* string, const char* initial); + +FairyFileHeader* Fairy_ReadFileHeader(FairyFileHeader* header, FILE* file); +FairySecHeader* Fairy_ReadSectionTable(FairySecHeader* sectionTable, FILE* file, size_t tableOffset, size_t number); +char* Fairy_ReadStringTable(char* stringTable, FILE* file, size_t tableOffset, size_t tableSize); +size_t Fairy_ReadSymbolTable(FairySym** symbolTableOut, FILE* file, size_t tableOffset, size_t tableSize); +size_t Fairy_ReadRelocs(FairyRela** relocsOut, FILE* file, int type, size_t offset, size_t size); + +char* Fairy_GetSectionName(FairySecHeader* sectionTable, char* shstrtab, size_t index); +char* Fairy_GetSymbolName(FairySym* symtab, char* strtab, size_t index); + +void Fairy_InitFile(FairyFileInfo* fileInfo, FILE* file); +void Fairy_DestroyFile(FairyFileInfo* fileInfo); diff --git a/tools/fado/lib/fairy/fairy_data.inc b/tools/fado/lib/fairy/fairy_data.inc new file mode 100644 index 0000000..5dfecfc --- /dev/null +++ b/tools/fado/lib/fairy/fairy_data.inc @@ -0,0 +1,126 @@ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#include "fairy.h" +#include "mips_elf.h" + +// clang-format off +static const FairyDefineString stTypes[] = { + FAIRY_DEF_STRING(STT_, NOTYPE), + FAIRY_DEF_STRING(STT_, OBJECT), + FAIRY_DEF_STRING(STT_, FUNC), + FAIRY_DEF_STRING(STT_, SECTION), + FAIRY_DEF_STRING(STT_, FILE), + FAIRY_DEF_STRING(STT_, COMMON), + FAIRY_DEF_STRING(STT_, TLS), + FAIRY_DEF_STRING(STT_, NUM), + FAIRY_DEF_STRING(STT_, LOOS), + FAIRY_DEF_STRING(STT_, GNU_IFUNC), + FAIRY_DEF_STRING(STT_, HIOS), + FAIRY_DEF_STRING(STT_, LOPROC), + FAIRY_DEF_STRING(STT_, HIPROC), + { 0 }, +}; + +static const FairyDefineString stBinds[] = { + FAIRY_DEF_STRING(STB_, LOCAL), + FAIRY_DEF_STRING(STB_, GLOBAL), + FAIRY_DEF_STRING(STB_, WEAK), + FAIRY_DEF_STRING(STB_, NUM), + FAIRY_DEF_STRING(STB_, LOOS), + FAIRY_DEF_STRING(STB_, GNU_UNIQUE), + FAIRY_DEF_STRING(STB_, HIOS), + FAIRY_DEF_STRING(STB_, LOPROC), + FAIRY_DEF_STRING(STB_, HIPROC), + { 0 }, +}; + +static const FairyDefineString stVisibilities[] = { + FAIRY_DEF_STRING(STV_, DEFAULT), + FAIRY_DEF_STRING(STV_, INTERNAL), + FAIRY_DEF_STRING(STV_, HIDDEN), + FAIRY_DEF_STRING(STV_, PROTECTED), + { 0 }, +}; + +/* TODO: understand this data better: there are several cases with the same number */ +static const FairyDefineString shTypes[] = { + FAIRY_DEF_STRING(SHT_, NULL), + FAIRY_DEF_STRING(SHT_, PROGBITS), + FAIRY_DEF_STRING(SHT_, SYMTAB), + FAIRY_DEF_STRING(SHT_, STRTAB), + FAIRY_DEF_STRING(SHT_, RELA), + FAIRY_DEF_STRING(SHT_, HASH), + FAIRY_DEF_STRING(SHT_, DYNAMIC), + FAIRY_DEF_STRING(SHT_, NOTE), + FAIRY_DEF_STRING(SHT_, NOBITS), + FAIRY_DEF_STRING(SHT_, REL), + FAIRY_DEF_STRING(SHT_, SHLIB), + FAIRY_DEF_STRING(SHT_, DYNSYM), + FAIRY_DEF_STRING(SHT_, INIT_ARRAY), + FAIRY_DEF_STRING(SHT_, FINI_ARRAY), + FAIRY_DEF_STRING(SHT_, PREINIT_ARRAY), + FAIRY_DEF_STRING(SHT_, GROUP), + FAIRY_DEF_STRING(SHT_, SYMTAB_SHNDX), + FAIRY_DEF_STRING(SHT_, NUM), + FAIRY_DEF_STRING(SHT_, LOOS), + FAIRY_DEF_STRING(SHT_, GNU_ATTRIBUTES), + FAIRY_DEF_STRING(SHT_, GNU_HASH), + FAIRY_DEF_STRING(SHT_, GNU_LIBLIST), + FAIRY_DEF_STRING(SHT_, CHECKSUM), + FAIRY_DEF_STRING(SHT_, LOSUNW), + FAIRY_DEF_STRING(SHT_, SUNW_move), + FAIRY_DEF_STRING(SHT_, SUNW_COMDAT), + FAIRY_DEF_STRING(SHT_, SUNW_syminfo), + FAIRY_DEF_STRING(SHT_, GNU_verdef), + FAIRY_DEF_STRING(SHT_, GNU_verneed), + FAIRY_DEF_STRING(SHT_, GNU_versym), + FAIRY_DEF_STRING(SHT_, HISUNW), + FAIRY_DEF_STRING(SHT_, HIOS), + FAIRY_DEF_STRING(SHT_, LOPROC), + FAIRY_DEF_STRING(SHT_, HIPROC), + FAIRY_DEF_STRING(SHT_, LOUSER), + FAIRY_DEF_STRING(SHT_, HIUSER), + FAIRY_DEF_STRING(SHT_, MIPS_LIBLIST), + FAIRY_DEF_STRING(SHT_, MIPS_MSYM), + FAIRY_DEF_STRING(SHT_, MIPS_CONFLICT), + FAIRY_DEF_STRING(SHT_, MIPS_GPTAB), + FAIRY_DEF_STRING(SHT_, MIPS_UCODE), + FAIRY_DEF_STRING(SHT_, MIPS_DEBUG), + FAIRY_DEF_STRING(SHT_, MIPS_REGINFO), + FAIRY_DEF_STRING(SHT_, MIPS_PACKAGE), + FAIRY_DEF_STRING(SHT_, MIPS_PACKSYM), + FAIRY_DEF_STRING(SHT_, MIPS_RELD), + FAIRY_DEF_STRING(SHT_, MIPS_IFACE), + FAIRY_DEF_STRING(SHT_, MIPS_CONTENT), + FAIRY_DEF_STRING(SHT_, MIPS_OPTIONS), + FAIRY_DEF_STRING(SHT_, MIPS_SHDR), + FAIRY_DEF_STRING(SHT_, MIPS_FDESC), + FAIRY_DEF_STRING(SHT_, MIPS_EXTSYM), + FAIRY_DEF_STRING(SHT_, MIPS_DENSE), + FAIRY_DEF_STRING(SHT_, MIPS_PDESC), + FAIRY_DEF_STRING(SHT_, MIPS_LOCSYM), + FAIRY_DEF_STRING(SHT_, MIPS_AUXSYM), + FAIRY_DEF_STRING(SHT_, MIPS_OPTSYM), + FAIRY_DEF_STRING(SHT_, MIPS_LOCSTR), + FAIRY_DEF_STRING(SHT_, MIPS_LINE), + FAIRY_DEF_STRING(SHT_, MIPS_RFDESC), + FAIRY_DEF_STRING(SHT_, MIPS_DELTASYM), + FAIRY_DEF_STRING(SHT_, MIPS_DELTAINST), + FAIRY_DEF_STRING(SHT_, MIPS_DELTACLASS), + FAIRY_DEF_STRING(SHT_, MIPS_DWARF), + FAIRY_DEF_STRING(SHT_, MIPS_DELTADECL), + FAIRY_DEF_STRING(SHT_, MIPS_SYMBOL_LIB), + FAIRY_DEF_STRING(SHT_, MIPS_EVENTS), + FAIRY_DEF_STRING(SHT_, MIPS_TRANSLATE), + FAIRY_DEF_STRING(SHT_, MIPS_PIXIE), + FAIRY_DEF_STRING(SHT_, MIPS_XLATE), + FAIRY_DEF_STRING(SHT_, MIPS_XLATE_DEBUG), + FAIRY_DEF_STRING(SHT_, MIPS_WHIRL), + FAIRY_DEF_STRING(SHT_, MIPS_EH_REGION), + FAIRY_DEF_STRING(SHT_, MIPS_XLATE_OLD), + FAIRY_DEF_STRING(SHT_, MIPS_PDR_EXCEPTION), + // FAIRY_DEF_STRING(SHT_, MIPS_XHASH), /* New in 2019 */ + { 0 }, +}; + +// clang-format on diff --git a/tools/fado/lib/fairy/fairy_print.c b/tools/fado/lib/fairy/fairy_print.c new file mode 100644 index 0000000..80b8fcd --- /dev/null +++ b/tools/fado/lib/fairy/fairy_print.c @@ -0,0 +1,463 @@ +/** + * Functions for printing various sections of an N64 ELF file using the functions in Fairy, similarly to readelf + */ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#include "fairy.h" + +#include <assert.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "fairy_data.inc" + +void Fairy_PrintSymbolTable(FILE* inputFile) { + FairyFileHeader fileHeader; + FairySecHeader* sectionTable; + size_t shstrndx; + char* shstrtab; + FairySym* symbolTable = NULL; + size_t symbolTableNum = 0; + char* strtab = NULL; + + Fairy_ReadFileHeader(&fileHeader, inputFile); + sectionTable = malloc(fileHeader.e_shentsize * fileHeader.e_shnum); + shstrndx = fileHeader.e_shstrndx; + + Fairy_ReadSectionTable(sectionTable, inputFile, fileHeader.e_shoff, fileHeader.e_shnum); + + shstrtab = malloc(sectionTable[shstrndx].sh_size * sizeof(char)); + + fseek(inputFile, sectionTable[shstrndx].sh_offset, SEEK_SET); + assert(fread(shstrtab, sizeof(char), sectionTable[shstrndx].sh_size, inputFile) == sectionTable[shstrndx].sh_size); + + { + size_t currentIndex; + size_t strtabndx = 0; + for (currentIndex = 0; currentIndex < fileHeader.e_shnum; currentIndex++) { + FairySecHeader currentHeader = sectionTable[currentIndex]; + + switch (currentHeader.sh_type) { + case SHT_SYMTAB: + if (strcmp(&shstrtab[currentHeader.sh_name], ".symtab") == 0) { + printf("symtab found\n"); + symbolTableNum = Fairy_ReadSymbolTable(&symbolTable, inputFile, currentHeader.sh_offset, + currentHeader.sh_size); + } + break; + + case SHT_STRTAB: + if (strcmp(&shstrtab[currentHeader.sh_name], ".strtab") == 0) { + strtabndx = currentIndex; + } + break; + + default: + break; + } + } + + if (symbolTable == NULL) { + puts("No symtab found."); + free(sectionTable); + return; + } + + if (strtabndx != 0) { + printf("strtab found\n"); + printf("Size: %X bytes\n", sectionTable[strtabndx].sh_size); + strtab = malloc(sectionTable[strtabndx].sh_size); + printf("and mallocked\n"); + fseek(inputFile, sectionTable[strtabndx].sh_offset, SEEK_SET); + printf("file offset sought: %X\n", sectionTable[strtabndx].sh_offset); + assert(fread(strtab, sizeof(char), sectionTable[strtabndx].sh_size, inputFile) == + sectionTable[strtabndx].sh_size); + printf("file read\n"); + } + } + + { + size_t currentIndex; + printf("Symbol table\n"); + printf(" Num: Value Size Type Bind Vis Ndx Name\n"); + for (currentIndex = 0; currentIndex < symbolTableNum; currentIndex++) { + FairySym currentSymbol = symbolTable[currentIndex]; + printf("%4zd: ", currentIndex); + printf("%08X ", currentSymbol.st_value); + printf("%4X ", currentSymbol.st_size); + printf("%-11s ", Fairy_StringFromDefine(stTypes, ELF32_ST_TYPE(currentSymbol.st_info))); + printf("%-10s ", Fairy_StringFromDefine(stBinds, ELF32_ST_BIND(currentSymbol.st_info))); + printf("%-11s ", Fairy_StringFromDefine(stVisibilities, ELF32_ST_VISIBILITY(currentSymbol.st_other))); + + if (currentSymbol.st_shndx != 0) { + printf("%3X ", currentSymbol.st_shndx); + } else { + printf("UND "); + } + + if (strtab != NULL) { + printf("%s", &strtab[currentSymbol.st_name]); + } else { + printf("%4X ", currentSymbol.st_name); + } + putchar('\n'); + } + } + + free(sectionTable); + free(symbolTable); + if (strtab != NULL) { + free(strtab); + } +} + +void Fairy_PrintRelocs(FILE* inputFile) { + FairyFileHeader fileHeader; + FairySecHeader* sectionTable; + FairyRela* relocs; + size_t shstrndx; + char* shstrtab; + size_t currentSection; + + Fairy_ReadFileHeader(&fileHeader, inputFile); + sectionTable = malloc(fileHeader.e_shentsize * fileHeader.e_shnum); + shstrndx = fileHeader.e_shstrndx; + + Fairy_ReadSectionTable(sectionTable, inputFile, fileHeader.e_shoff, fileHeader.e_shnum); + + shstrtab = malloc(sectionTable[shstrndx].sh_size * sizeof(char)); + + fseek(inputFile, sectionTable[shstrndx].sh_offset, SEEK_SET); + assert(fread(shstrtab, sizeof(char), sectionTable[shstrndx].sh_size, inputFile) == sectionTable[shstrndx].sh_size); + + for (currentSection = 0; currentSection < fileHeader.e_shnum; currentSection++) { + size_t nRelocs; + + if (sectionTable[currentSection].sh_type != SHT_REL || sectionTable[currentSection].sh_type != SHT_RELA) { + continue; + } + printf("Section size: %d\n", sectionTable[currentSection].sh_size); + + nRelocs = Fairy_ReadRelocs(&relocs, inputFile, sectionTable[currentSection].sh_type, + sectionTable[currentSection].sh_offset, sectionTable[currentSection].sh_size); + + // fseek(inputFile, sectionTable[currentSection].sh_offset, SEEK_SET); + // assert(fread(relocs, sizeof(char), sectionTable[currentSection].sh_size, inputFile) == + // sectionTable[currentSection].sh_size); + + printf("Relocs in section [%2zd]: %s:\n", currentSection, shstrtab + sectionTable[currentSection].sh_name); + printf("Offset Info Type Symbol\n"); + { + size_t currentReloc; + for (currentReloc = 0; currentReloc < nRelocs; currentReloc++) { + printf("%08X,%08X ", relocs[currentReloc].r_offset, relocs[currentReloc].r_info); + + switch (ELF32_R_TYPE(relocs[currentReloc].r_info)) { + case R_MIPS_NONE: + printf("%-15s", "R_MIPS_NONE"); + break; + case R_MIPS_16: + printf("%-15s", "R_MIPS_16"); + break; + case R_MIPS_32: + printf("%-15s", "R_MIPS_32"); + break; + case R_MIPS_REL32: + printf("%-15s", "R_MIPS_REL32"); + break; + case R_MIPS_26: + printf("%-15s", "R_MIPS_26"); + break; + case R_MIPS_HI16: + printf("%-15s", "R_MIPS_HI16"); + break; + case R_MIPS_LO16: + printf("%-15s", "R_MIPS_LO16"); + break; + default: + break; + } + + printf("%X", ELF32_R_SYM(relocs[currentReloc].r_info)); + + putchar('\n'); + } + putchar('\n'); + } + putchar('\n'); + + free(relocs); + } + free(sectionTable); + free(shstrtab); +} + +void Fairy_PrintSectionTable(FILE* inputFile) { + FairyFileHeader fileHeader; + FairySecHeader* sectionTable; + size_t shstrndx; + char* shstrtab; + size_t currentSection; + + Fairy_ReadFileHeader(&fileHeader, inputFile); + sectionTable = malloc(fileHeader.e_shentsize * fileHeader.e_shnum); + shstrndx = fileHeader.e_shstrndx; + + Fairy_ReadSectionTable(sectionTable, inputFile, fileHeader.e_shoff, fileHeader.e_shnum); + + shstrtab = malloc(sectionTable[shstrndx].sh_size * sizeof(char)); + + fseek(inputFile, sectionTable[shstrndx].sh_offset, SEEK_SET); + assert(fread(shstrtab, sizeof(char), sectionTable[shstrndx].sh_size, inputFile) == sectionTable[shstrndx].sh_size); + + printf("[Nr] Name Type Addr Off Size ES Flg Lk Inf Al\n"); + for (currentSection = 0; currentSection < fileHeader.e_shnum; currentSection++) { + FairySecHeader entry = sectionTable[currentSection]; + printf("[%2zd] ", currentSection); + printf("%-15s", shstrtab + entry.sh_name); + + printf("%-15s", Fairy_StringFromDefine(shTypes, entry.sh_type)); + + // printf("%08X ", entry.sh_type); + printf("%08X ", entry.sh_addr); + printf("%06X ", entry.sh_offset); + printf("%06X ", entry.sh_size); + printf("%02X ", entry.sh_entsize); + // printf("%08X ", entry.sh_flags); + + { + char flagChars[] = { 'W', 'A', 'X', 'M', 'S', 'I', 'L', 'O', 'G', 'T', 'C', 'x', 'o', 'E', 'p' }; + uint32_t flags = entry.sh_flags; + size_t shift; + int pad = 4; + for (shift = 0; shift < sizeof(flagChars); shift++) { + if ((flags >> shift) & 1) { + putchar(flagChars[shift]); + pad--; + } + } + if (pad > 0) { + printf("%*s", pad, ""); + } + } + + printf("%2X ", entry.sh_link); + printf("%3X ", entry.sh_info); + printf("%2X", entry.sh_addralign); + putchar('\n'); + } +} + +typedef enum { REL_SECTION_NONE, REL_SECTION_TEXT, REL_SECTION_DATA, REL_SECTION_RODATA } FairyOverlayRelSection; + +const char* relSectionStrings[] = { + NULL, + ".text", + ".data", + ".rodata", +}; + +static uint32_t Fairy_PackReloc(FairyOverlayRelSection sec, FairyRela rel) { + return (sec << 0x1E) | (ELF32_R_TYPE(rel.r_info) << 0x18) | rel.r_offset; +} + +void Fairy_PrintSectionSizes(FairySecHeader* sectionTable, FILE* inputFile, size_t tableSize, char* shstrtab) { + size_t number = tableSize / sizeof(FairySecHeader); + FairySecHeader currentHeader; + char* sectionName; + size_t relocSectionsCount = 0; + size_t* relocSectionIndices; + int* relocSectionSection; + size_t currentRelocSection = 0; + FairySecHeader symtabHeader; + FairySym* symtab; + FairySecHeader strtabHeader; + char* strtab = NULL; + // size_t symtabSize; + + uint32_t textSize = 0; + uint32_t dataSize = 0; + uint32_t rodataSize = 0; + uint32_t bssSize = 0; + uint32_t relocCount = 0; + + size_t currentSection; + bool symtabFound = false; + bool strtabFound = false; + /* Count the reloc sections */ + for (currentSection = 0; currentSection < number; currentSection++) { + if (sectionTable[currentSection].sh_type == SHT_REL || sectionTable[currentSection].sh_type == SHT_RELA) { + relocSectionsCount++; + } + } + printf("relocSectionsCount: %zd\n", relocSectionsCount); + + relocSectionIndices = malloc(relocSectionsCount * sizeof(int)); + relocSectionSection = malloc(relocSectionsCount * sizeof(int)); + + /* Find the section sizes and the reloc sections */ + for (currentSection = 0; currentSection < number; currentSection++) { + size_t off = 0; + + currentHeader = sectionTable[currentSection]; + sectionName = &shstrtab[currentHeader.sh_name + 1]; /* ignore the initial '.' */ + switch (currentHeader.sh_type) { + case SHT_PROGBITS: + if (Fairy_StartsWith(sectionName, "rodata")) { + printf("rodata\n"); + rodataSize += currentHeader.sh_size; + break; + } + if (Fairy_StartsWith(sectionName, "data")) { + printf("data\n"); + dataSize += currentHeader.sh_size; + break; + } + if (Fairy_StartsWith(sectionName, "text")) { + printf("text\n"); + textSize += currentHeader.sh_size; + break; + } + break; + + case SHT_NOBITS: + if (Fairy_StartsWith(sectionName, "bss")) { + printf("bss\n"); + bssSize += currentHeader.sh_size; + } + break; + + case SHT_RELA: + off += 1; + case SHT_REL: + relocSectionIndices[currentRelocSection] = currentSection; + off += 4; /* ignore the "rel."/"rela." part */ + if (Fairy_StartsWith(§ionName[off], "rodata")) { + printf("%s\n", sectionName); + relocSectionSection[currentRelocSection] = REL_SECTION_RODATA; + } else if (Fairy_StartsWith(§ionName[off], "data")) { + printf("%s\n", sectionName); + relocSectionSection[currentRelocSection] = REL_SECTION_DATA; + } else if (Fairy_StartsWith(§ionName[off], "text")) { + printf("%s\n", sectionName); + relocSectionSection[currentRelocSection] = REL_SECTION_TEXT; + } + + currentRelocSection++; + break; + + case SHT_SYMTAB: + if (Fairy_StartsWith(sectionName, "symtab")) { + symtabHeader = currentHeader; + symtabFound = true; + } + break; + + case SHT_STRTAB: + if (Fairy_StartsWith(sectionName, "strtab")) { + strtabHeader = currentHeader; + strtabFound = true; + } + break; + + default: + break; + } + } + /* Can use symbols here too */ + puts(".section .ovl"); + printf("# OverlayInfo\n"); + printf(".word 0x%08X # .text size\n", textSize); + printf(".word 0x%08X # .data size\n", dataSize); + printf(".word 0x%08X # .rodata size\n", rodataSize); + printf(".word 0x%08X # .bss size\n\n", bssSize); + + if (!symtabFound) { + fprintf(stderr, "Symbol table not found\n"); + return; + } + /* Obtain the symbol table */ + // TODO: Consider replacing this with a lighter-weight read: sufficient to get the name, shndx + Fairy_ReadSymbolTable(&symtab, inputFile, symtabHeader.sh_offset, symtabHeader.sh_size); + + if (!strtabFound) { + fprintf(stderr, "String table not found\n"); + } else { + /* Obtain the string table */ + strtab = malloc(strtabHeader.sh_size); + fseek(inputFile, strtabHeader.sh_offset, SEEK_SET); + assert(fread(strtab, sizeof(char), strtabHeader.sh_size, inputFile) == strtabHeader.sh_size); + } + + /* Do single-file relocs */ + { + FairyRela* relocs; + for (currentSection = 0; currentSection < relocSectionsCount; currentSection++) { + size_t currentReloc; + size_t nRelocs; + + currentHeader = sectionTable[relocSectionIndices[currentSection]]; + nRelocs = Fairy_ReadRelocs(&relocs, inputFile, currentHeader.sh_type, currentHeader.sh_offset, + currentHeader.sh_size); + + for (currentReloc = 0; currentReloc < nRelocs; currentReloc++) { + FairySym symbol = symtab[ELF32_R_SYM(relocs[currentReloc].r_info)]; + if (symbol.st_shndx == SHN_UNDEF) { + continue; // TODO: this is where multifile has to look elsewhere + } + + printf(".word 0x%08X", Fairy_PackReloc(relocSectionSection[currentSection], relocs[currentReloc])); + printf(" # %X (%s), %X, 0x%06X", relocSectionSection[currentSection], &shstrtab[currentHeader.sh_name], + ELF32_R_TYPE(relocs[currentReloc].r_info), relocs[currentReloc].r_offset); + if (strtab != NULL) { + printf(", %s", &strtab[symbol.st_name]); + } + putchar('\n'); + + relocCount++; + } + + free(relocs); + } + } + + printf(".word %d # relocCount\n", relocCount); + + { + uint32_t ovlSectionSize = ((relocCount + 8) & ~0x03) * sizeof(uint32_t); + + printf("\n.word 0x%08X # Overlay section size\n", ovlSectionSize); + } + + free(relocSectionIndices); + free(relocSectionSection); + if (strtab != NULL) { + free(strtab); + } +} + +void PrintZeldaReloc(FILE* inputFile) { + FairyFileHeader fileHeader; + FairySecHeader* sectionTable; + size_t shstrndx; + char* shstrtab; + + Fairy_ReadFileHeader(&fileHeader, inputFile); + sectionTable = malloc(fileHeader.e_shentsize * fileHeader.e_shnum); + shstrndx = fileHeader.e_shstrndx; + + Fairy_ReadSectionTable(sectionTable, inputFile, fileHeader.e_shoff, fileHeader.e_shnum); + + shstrtab = malloc(sectionTable[shstrndx].sh_size * sizeof(char)); + + fseek(inputFile, sectionTable[shstrndx].sh_offset, SEEK_SET); + assert(fread(shstrtab, sizeof(char), sectionTable[shstrndx].sh_size, inputFile) == sectionTable[shstrndx].sh_size); + + Fairy_PrintSectionSizes(sectionTable, inputFile, fileHeader.e_shentsize * fileHeader.e_shnum, shstrtab); + + free(sectionTable); + free(shstrtab); +} diff --git a/tools/fado/lib/fairy/fairy_print.h b/tools/fado/lib/fairy/fairy_print.h new file mode 100644 index 0000000..85061be --- /dev/null +++ b/tools/fado/lib/fairy/fairy_print.h @@ -0,0 +1,7 @@ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#include <stdio.h> + +void Fairy_PrintSymbolTable(FILE* inputFile); +void Fairy_PrintRelocs(FILE* inputFile); +void Fairy_PrintSectionTable(FILE* inputFile); diff --git a/tools/fado/lib/vc_vector/.gitrepo b/tools/fado/lib/vc_vector/.gitrepo new file mode 100644 index 0000000..7ab27b9 --- /dev/null +++ b/tools/fado/lib/vc_vector/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme +; +[subrepo] + remote = git@github.com:skogorev/vc_vector.git + branch = master + commit = 39108a4b0aeb904636514b37f418c590084220a7 + parent = 462bee5811d54659236ca51481d01b76c69a37a7 + method = merge + cmdver = 0.4.3 diff --git a/tools/fado/lib/vc_vector/.travis.yml b/tools/fado/lib/vc_vector/.travis.yml new file mode 100644 index 0000000..f30dfb8 --- /dev/null +++ b/tools/fado/lib/vc_vector/.travis.yml @@ -0,0 +1,6 @@ +language: c +compiler: + - gcc + - clang +script: + - make && make test diff --git a/tools/fado/lib/vc_vector/LICENSE.md b/tools/fado/lib/vc_vector/LICENSE.md new file mode 100644 index 0000000..05929e1 --- /dev/null +++ b/tools/fado/lib/vc_vector/LICENSE.md @@ -0,0 +1,21 @@ +#The MIT License (MIT) + +*Copyright (c) 2016 Skogorev Anton* + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/fado/lib/vc_vector/Makefile b/tools/fado/lib/vc_vector/Makefile new file mode 100644 index 0000000..129ee97 --- /dev/null +++ b/tools/fado/lib/vc_vector/Makefile @@ -0,0 +1,33 @@ +OUT_DIR := build +CFLAGS := -O2 -g -std=c99 -Wall -Wextra -Wpedantic -Werror + +SRCS := $(wildcard *.c) +OBJS := $(patsubst %.c,$(OUT_DIR)/%.o,$(SRCS)) + +LIB_NAME := vc-vector +SOBJ := $(OUT_DIR)/lib$(LIB_NAME).so + +.PHONY: all lib test clean + +all: lib + +lib: $(SOBJ) + +test: $(OUT_DIR)/test_runner + $(OUT_DIR)/test_runner + +clean: + rm -rf $(OUT_DIR) + +$(OUT_DIR): + mkdir -p $(OUT_DIR) + +$(SOBJ): vc_vector.c | $(OUT_DIR) + $(CC) $(CFLAGS) -shared -fPIC $< -o $@ + +$(OUT_DIR)/%.o: %.c | $(OUT_DIR) + $(CC) $(CFLAGS) -c $< -o $@ + +$(OUT_DIR)/test_runner: $(OBJS) | $(OUT_DIR) + $(CC) $^ -o $@ + diff --git a/tools/fado/lib/vc_vector/README.md b/tools/fado/lib/vc_vector/README.md new file mode 100644 index 0000000..c8f8b2a --- /dev/null +++ b/tools/fado/lib/vc_vector/README.md @@ -0,0 +1,92 @@ +# vc_vector +Fast simple C vector implementation + +[](https://travis-ci.org/skogorev/vc_vector) + +## Usage + +### Basic +```c +#include "vc_vector.h" + +int main() { + // Creates an empty vector with the default reserved size + // and without custom deleter. Vector will contain 'int' + vc_vector* v = vc_vector_create(0, sizeof(int), NULL); + if (!v) { + return 1; + } + + const int count = 10; + for (int i = 0; i < count; ++i) { + // The function takes a pointer to the elements, + // but the vector will make a copy of the element + vc_vector_push_back(v, &i); + } + + // Print each vector element + for (void* i = vc_vector_begin(v); + i != vc_vector_end(v); + i = vc_vector_next(v, i)) { + printf("%u; ", *(int*)i); + } + + vc_vector_release(v); + return 0; +} +``` + +### Advanced +```c +#include "vc_vector.h" + +struct Item { + int val1; + int val2; +}; + +int main() { + const int n = 10; + + // Creates an empty vector with the reserved size for the 'n' elements + // and with custom deleter 'free'. Vector will contain pointers to 'Item' + vc_vector* v = vc_vector_create(n, sizeof(struct Node*), free); + if (!v) { + return 1; + } + + struct Item* item = NULL; + const int count = n + 1; + // Vector automatically increases the reserved size when 'n + 1' will be added + for (int i = 0; i < count; ++i) { + // Creating item + item = malloc(sizeof(struct Item)); + if (!item) { + continue; + } + + item->val1 = i; + item->val2 = 0; + + // Pushing to the end of the vector + if (!vc_vector_push_back(v, item)) { + // If the item was not pushed, you have to delete it + free(item); + } + } + + // ... + + // Calls custom deleter 'free' for all items + // and releases the vector + vc_vector_release(v); + return 0; +} +``` + +## Projects that use vc_vector +[kraken.io](https://kraken.io/) + +## License + +[MIT License](LICENSE.md) diff --git a/tools/fado/lib/vc_vector/vc_vector.c b/tools/fado/lib/vc_vector/vc_vector.c new file mode 100644 index 0000000..426f1b0 --- /dev/null +++ b/tools/fado/lib/vc_vector/vc_vector.c @@ -0,0 +1,329 @@ +#include "vc_vector.h" +#include <stdlib.h> +#include <string.h> + +#define GROWTH_FACTOR 1.5 +#define DEFAULT_COUNT_OF_ELEMENTS 8 +#define MINIMUM_COUNT_OF_ELEMENTS 2 + +// ---------------------------------------------------------------------------- + +// vc_vector structure + +struct vc_vector { + size_t count; + size_t element_size; + size_t reserved_size; + char* data; + vc_vector_deleter* deleter; +}; + +// ---------------------------------------------------------------------------- + +// Auxiliary methods + +bool vc_vector_realloc(vc_vector* vector, size_t new_count) { + const size_t new_size = new_count * vector->element_size; + char* new_data = (char*)realloc(vector->data, new_size); + if (!new_data) { + return false; + } + + vector->reserved_size = new_size; + vector->data = new_data; + return true; +} + +// [first_index, last_index) +void vc_vector_call_deleter(vc_vector* vector, size_t first_index, size_t last_index) { + for (size_t i = first_index; i < last_index; ++i) { + vector->deleter(vc_vector_at(vector, i)); + } +} + +void vc_vector_call_deleter_all(vc_vector* vector) { + vc_vector_call_deleter(vector, 0, vc_vector_count(vector)); +} + +// ---------------------------------------------------------------------------- + +// Control + +vc_vector* vc_vector_create(size_t count_elements, size_t size_of_element, vc_vector_deleter* deleter) { + vc_vector* v = (vc_vector*)malloc(sizeof(vc_vector)); + if (v != NULL) { + v->data = NULL; + v->count = 0; + v->element_size = size_of_element; + v->deleter = deleter; + + if (count_elements < MINIMUM_COUNT_OF_ELEMENTS) { + count_elements = DEFAULT_COUNT_OF_ELEMENTS; + } + + if (size_of_element < 1 || + !vc_vector_realloc(v, count_elements)) { + free(v); + v = NULL; + } + } + + return v; +} + +vc_vector* vc_vector_create_copy(const vc_vector* vector) { + vc_vector* new_vector = vc_vector_create(vector->reserved_size / vector->count, + vector->element_size, + vector->deleter); + if (!new_vector) { + return new_vector; + } + + if (memcpy(vector->data, + new_vector->data, + new_vector->element_size * vector->count) == NULL) { + vc_vector_release(new_vector); + new_vector = NULL; + return new_vector; + } + + new_vector->count = vector->count; + return new_vector; +} + +void vc_vector_release(vc_vector* vector) { + if (vector->deleter != NULL) { + vc_vector_call_deleter_all(vector); + } + + if (vector->reserved_size != 0) { + free(vector->data); + } + + free(vector); +} + +bool vc_vector_is_equals(vc_vector* vector1, vc_vector* vector2) { + const size_t size_vector1 = vc_vector_size(vector1); + if (size_vector1 != vc_vector_size(vector2)) { + return false; + } + + return memcmp(vector1->data, vector2->data, size_vector1) == 0; +} + +float vc_vector_get_growth_factor(void) { + return GROWTH_FACTOR; +} + +size_t vc_vector_get_default_count_of_elements(void) { + return DEFAULT_COUNT_OF_ELEMENTS; +} + +size_t vc_vector_struct_size(void) { + return sizeof(vc_vector); +} + +// ---------------------------------------------------------------------------- + +// Element access + +void* vc_vector_at(vc_vector* vector, size_t index) { + return vector->data + index * vector->element_size; +} + +void* vc_vector_front(vc_vector* vector) { + return vector->data; +} + +void* vc_vector_back(vc_vector* vector) { + return vector->data + (vector->count - 1) * vector->element_size; +} + +void* vc_vector_data(vc_vector* vector) { + return vector->data; +} + +// ---------------------------------------------------------------------------- + +// Iterators + +void* vc_vector_begin(vc_vector* vector) { + return vector->data; +} + +void* vc_vector_end(vc_vector* vector) { + return vector->data + vector->element_size * vector->count; +} + +void* vc_vector_next(vc_vector* vector, void* i) { + return (char *)i + vector->element_size; +} + +// ---------------------------------------------------------------------------- + +// Capacity + +bool vc_vector_empty(vc_vector* vector) { + return vector->count == 0; +} + +size_t vc_vector_count(const vc_vector* vector) { + return vector->count; +} + +size_t vc_vector_size(const vc_vector* vector) { + return vector->count * vector->element_size; +} + +size_t vc_vector_max_count(const vc_vector* vector) { + return vector->reserved_size / vector->element_size; +} + +size_t vc_vector_max_size(const vc_vector* vector) { + return vector->reserved_size; +} + +bool vc_vector_reserve_count(vc_vector* vector, size_t new_count) { + if (new_count < vector->count) { + return false; + } + + size_t new_size = vector->element_size * new_count; + if (new_size == vector->reserved_size) { + return true; + } + + return vc_vector_realloc(vector, new_count); +} + +bool vc_vector_reserve_size(vc_vector* vector, size_t new_size) { + return vc_vector_reserve_count(vector, new_size / vector->element_size); +} + +// ---------------------------------------------------------------------------- + +// Modifiers + +void vc_vector_clear(vc_vector* vector) { + if (vector->deleter != NULL) { + vc_vector_call_deleter_all(vector); + } + + vector->count = 0; +} + +bool vc_vector_insert(vc_vector* vector, size_t index, const void* value) { + if (vc_vector_max_count(vector) < vector->count + 1) { + if (!vc_vector_realloc(vector, vc_vector_max_count(vector) * GROWTH_FACTOR)) { + return false; + } + } + + if (!memmove(vc_vector_at(vector, index + 1), + vc_vector_at(vector, index), + vector->element_size * (vector->count - index))) { + + return false; + } + + if (memcpy(vc_vector_at(vector, index), + value, + vector->element_size) == NULL) { + return false; + } + + ++vector->count; + return true; +} + +bool vc_vector_erase(vc_vector* vector, size_t index) { + if (vector->deleter != NULL) { + vector->deleter(vc_vector_at(vector, index)); + } + + if (!memmove(vc_vector_at(vector, index), + vc_vector_at(vector, index + 1), + vector->element_size * (vector->count - index))) { + return false; + } + + vector->count--; + return true; +} + +bool vc_vector_erase_range(vc_vector* vector, size_t first_index, size_t last_index) { + if (vector->deleter != NULL) { + vc_vector_call_deleter(vector, first_index, last_index); + } + + if (!memmove(vc_vector_at(vector, first_index), + vc_vector_at(vector, last_index), + vector->element_size * (vector->count - last_index))) { + return false; + } + + vector->count -= last_index - first_index; + return true; +} + +bool vc_vector_append(vc_vector* vector, const void* values, size_t count) { + const size_t count_new = count + vc_vector_count(vector); + + if (vc_vector_max_count(vector) < count_new) { + size_t max_count_to_reserved = vc_vector_max_count(vector) * GROWTH_FACTOR; + while (count_new > max_count_to_reserved) { + max_count_to_reserved *= GROWTH_FACTOR; + } + + if (!vc_vector_realloc(vector, max_count_to_reserved)) { + return false; + } + } + + if (memcpy(vector->data + vector->count * vector->element_size, + values, + vector->element_size * count) == NULL) { + return false; + } + + vector->count = count_new; + return true; +} + +bool vc_vector_push_back(vc_vector* vector, const void* value) { + if (!vc_vector_append(vector, value, 1)) { + return false; + } + + return true; +} + +bool vc_vector_pop_back(vc_vector* vector) { + if (vector->deleter != NULL) { + vector->deleter(vc_vector_back(vector)); + } + + vector->count--; + return true; +} + +bool vc_vector_replace(vc_vector* vector, size_t index, const void* value) { + if (vector->deleter != NULL) { + vector->deleter(vc_vector_at(vector, index)); + } + + return memcpy(vc_vector_at(vector, index), + value, + vector->element_size) != NULL; +} + +bool vc_vector_replace_multiple(vc_vector* vector, size_t index, const void* values, size_t count) { + if (vector->deleter != NULL) { + vc_vector_call_deleter(vector, index, index + count); + } + + return memcpy(vc_vector_at(vector, index), + values, + vector->element_size * count) != NULL; +} diff --git a/tools/fado/lib/vc_vector/vc_vector.h b/tools/fado/lib/vc_vector/vc_vector.h new file mode 100644 index 0000000..e57f832 --- /dev/null +++ b/tools/fado/lib/vc_vector/vc_vector.h @@ -0,0 +1,120 @@ +#ifndef VCVECTOR_H +#define VCVECTOR_H + +#include <stdbool.h> +#include <stdio.h> + +typedef struct vc_vector vc_vector; +typedef void (vc_vector_deleter)(void *); + +// ---------------------------------------------------------------------------- +// Control +// ---------------------------------------------------------------------------- + +// Constructs an empty vector with an reserver size for count_elements. +vc_vector* vc_vector_create(size_t count_elements, size_t size_of_element, vc_vector_deleter* deleter); + +// Constructs a copy of an existing vector. +vc_vector* vc_vector_create_copy(const vc_vector* vector); + +// Releases the vector. +void vc_vector_release(vc_vector* vector); + +// Compares vector content +bool vc_vector_is_equals(vc_vector* vector1, vc_vector* vector2); + +// Returns constant value of the vector growth factor. +float vc_vector_get_growth_factor(void); + +// Returns constant value of the vector default count of elements. +size_t vc_vector_get_default_count_of_elements(void); + +// Returns constant value of the vector struct size. +size_t vc_vector_struct_size(void); + +// ---------------------------------------------------------------------------- +// Element access +// ---------------------------------------------------------------------------- + +// Returns the item at index position in the vector. +void* vc_vector_at(vc_vector* vector, size_t index); + +// Returns the first item in the vector. +void* vc_vector_front(vc_vector* vector); + +// Returns the last item in the vector. +void* vc_vector_back(vc_vector* vector); + +// Returns a pointer to the data stored in the vector. The pointer can be used to access and modify the items in the vector. +void* vc_vector_data(vc_vector* vector); + +// ---------------------------------------------------------------------------- +// Iterators +// ---------------------------------------------------------------------------- + +// Returns a pointer to the first item in the vector. +void* vc_vector_begin(vc_vector* vector); + +// Returns a pointer to the imaginary item after the last item in the vector. +void* vc_vector_end(vc_vector* vector); + +// Returns a pointer to the next element of vector after 'i'. +void* vc_vector_next(vc_vector* vector, void* i); + +// ---------------------------------------------------------------------------- +// Capacity +// ---------------------------------------------------------------------------- + +// Returns true if the vector is empty; otherwise returns false. +bool vc_vector_empty(vc_vector* vector); + +// Returns the number of elements in the vector. +size_t vc_vector_count(const vc_vector* vector); + +// Returns the size (in bytes) of occurrences of value in the vector. +size_t vc_vector_size(const vc_vector* vector); + +// Returns the maximum number of elements that the vector can hold. +size_t vc_vector_max_count(const vc_vector* vector); + +// Returns the maximum size (in bytes) that the vector can hold. +size_t vc_vector_max_size(const vc_vector* vector); + +// Resizes the container so that it contains n elements. +bool vc_vector_reserve_count(vc_vector* vector, size_t new_count); + +// Resizes the container so that it contains new_size / element_size elements. +bool vc_vector_reserve_size(vc_vector* vector, size_t new_size); + +// ---------------------------------------------------------------------------- +// Modifiers +// ---------------------------------------------------------------------------- + +// Removes all elements from the vector (without reallocation). +void vc_vector_clear(vc_vector* vector); + +// The container is extended by inserting a new element at position. +bool vc_vector_insert(vc_vector* vector, size_t index, const void* value); + +// Removes from the vector a single element by 'index' +bool vc_vector_erase(vc_vector* vector, size_t index); + +// Removes from the vector a range of elements '[first_index, last_index)'. +bool vc_vector_erase_range(vc_vector* vector, size_t first_index, size_t last_index); + +// Inserts multiple values at the end of the vector. +bool vc_vector_append(vc_vector* vector, const void* values, size_t count); + +// Inserts value at the end of the vector. +bool vc_vector_push_back(vc_vector* vector, const void* value); + +// Removes the last item in the vector. +bool vc_vector_pop_back(vc_vector* vector); + +// Replace value by index in the vector. +bool vc_vector_replace(vc_vector* vector, size_t index, const void* value); + +// Replace multiple values by index in the vector. +bool vc_vector_replace_multiple(vc_vector* vector, size_t index, const void* values, size_t count); + +#endif // VCVECTOR_H diff --git a/tools/fado/lib/vc_vector/vc_vector_test.c b/tools/fado/lib/vc_vector/vc_vector_test.c new file mode 100644 index 0000000..03a0544 --- /dev/null +++ b/tools/fado/lib/vc_vector/vc_vector_test.c @@ -0,0 +1,353 @@ +#include "vc_vector_test.h" +#include <stdlib.h> +#include <string.h> +#include <inttypes.h> +#include "vc_vector.h" + +#define ASSERT_EQ(expected, actual) \ + do { \ + if ((expected) != (actual)) { \ + fprintf(stderr, \ + "Failed line %u. Expected: %"PRIuMAX". Actual: %"PRIuMAX".\n", \ + __LINE__, (uintmax_t)(expected), (uintmax_t)(actual)); \ + abort(); \ + } \ + } while (0) + +#define ASSERT_NE(not_expected, actual) \ + do { \ + if ((not_expected) == (actual)) { \ + fprintf(stderr, \ + "Failed line %u. Unexpected actual value: %"PRIuMAX".\n", \ + __LINE__, (uintmax_t)(actual)); \ + abort(); \ + } \ + } while (0) + +#define ASSERT_TRUE(actual) ASSERT_EQ(true, (actual)) + +#define ASSERT_FALSE(actual) ASSERT_EQ(false, (actual)) + +#define PRINT_VECTOR(vector, type, format) \ + do { \ + for (void* i = vc_vector_begin(vector); \ + i != vc_vector_end(vector); \ + i = vc_vector_next(vector, i)) { \ + fprintf(stderr, format, *(type*)i); \ + } \ + fprintf(stderr, "\n"); \ + } while (0) + +#define PRINT_VECTOR_INT(vector) PRINT_VECTOR(vector, int, "%d; ") +#define PRINT_VECTOR_STR(vector) PRINT_VECTOR(vector, char *, "%s; ") + +char *mystrdup(const char *s) { + size_t size = strlen(s) + 1; + char *copy = malloc(size); + if (copy != NULL) + memcpy(copy, s, size); + return copy; +} + +// ---------------------------------------------------------------------------- + +void test_vc_vector_create() { + const size_t size_of_type = sizeof(int); + const size_t default_count_of_elements = vc_vector_get_default_count_of_elements(); + + // Creating vector with default count of elements + vc_vector* vector = vc_vector_create(0, size_of_type, NULL); + ASSERT_NE(NULL, vector); + ASSERT_EQ(0, vc_vector_count(vector)); + ASSERT_EQ(0, vc_vector_size(vector)); + ASSERT_EQ(default_count_of_elements, vc_vector_max_count(vector)); + ASSERT_EQ(size_of_type * default_count_of_elements, vc_vector_max_size(vector)); + vc_vector_release(vector); + + // Creating vector with custom count of elements + const size_t test_count_of_elements = 7; + vector = vc_vector_create(test_count_of_elements, size_of_type, NULL); + ASSERT_NE(NULL, vector); + ASSERT_EQ(0, vc_vector_count(vector)); + ASSERT_EQ(0, vc_vector_size(vector)); + ASSERT_EQ(test_count_of_elements, vc_vector_max_count(vector)); + ASSERT_EQ(size_of_type * test_count_of_elements, vc_vector_max_size(vector)); + vc_vector_release(vector); + + // Creating vector with zero size of single element + vector = vc_vector_create(0, 0, NULL); + ASSERT_EQ(NULL, vector); + + // Creating copy of vector + vector = vc_vector_create(0, size_of_type, NULL); + ASSERT_NE(NULL, vector); + for (int i = 0; (size_t)i < test_count_of_elements; ++i) { + ASSERT_TRUE(vc_vector_push_back(vector, &i)); + } + + vc_vector* vector_copy = vc_vector_create_copy(vector); + ASSERT_NE(NULL, vector_copy); + ASSERT_TRUE(vc_vector_is_equals(vector, vector_copy)); + + vc_vector_release(vector_copy); + vc_vector_release(vector); + + printf("%s passed.\n", __func__); +} + +void test_vc_vector_element_access() { + const int test_num_start = 18; + const int test_num_end = 36; + const size_t size_of_type = sizeof(test_num_start); + + vc_vector* vector = vc_vector_create(0, size_of_type, NULL); + ASSERT_NE(0, vector); + for (int i = test_num_start; i <= test_num_end; ++i) { + ASSERT_TRUE(vc_vector_push_back(vector, &i)); + } + + ASSERT_EQ(test_num_start, *(int*)vc_vector_front(vector)); + ASSERT_EQ(test_num_start, *(int*)vc_vector_data(vector)); + ASSERT_EQ(test_num_end, *(int*)vc_vector_back(vector)); + + for (int i = test_num_start, j = 0; i <= test_num_end; ++i, ++j) { + ASSERT_EQ(i, *(int*)vc_vector_at(vector, j)); + } + + vc_vector_release(vector); + + printf("%s passed.\n", __func__); +} + +void test_vc_vector_iterators() { + vc_vector* vector = vc_vector_create(0, sizeof(int), NULL); + ASSERT_NE(NULL, vector); + + const size_t test_count_of_elements = 23; + for (int i = 0; (size_t)i < test_count_of_elements; ++i) { + ASSERT_TRUE(vc_vector_push_back(vector, &i)); + } + + int j = 0; + for (void* i = vc_vector_begin(vector); + i != vc_vector_end(vector); + i = vc_vector_next(vector, i), ++j) { + ASSERT_EQ(j, *(int*)i); + } + + ASSERT_EQ(test_count_of_elements, (size_t)j); + vc_vector_release(vector); + + printf("%s passed.\n", __func__); +} + +void test_vc_vector_capacity() { + const size_t size_of_element = sizeof(int); + const float growth_factor = vc_vector_get_growth_factor(); + ASSERT_EQ(1.5, growth_factor); + + const size_t count_of_elements_initialized = 22; + const size_t max_size_of_vector_initialized = count_of_elements_initialized * size_of_element; + const size_t count_of_elements_ended = 23; + const size_t size_of_vector_ended = count_of_elements_ended * size_of_element; + const size_t max_count_of_vector_ended = count_of_elements_initialized * growth_factor; + const size_t max_size_of_vector_ended = max_count_of_vector_ended * size_of_element; + + vc_vector* vector = vc_vector_create(count_of_elements_initialized, size_of_element, NULL); + ASSERT_NE(NULL, vector); + + ASSERT_EQ(0, vc_vector_count(vector)); + ASSERT_TRUE(vc_vector_empty(vector)); + ASSERT_EQ(0, vc_vector_size(vector)); + ASSERT_EQ(count_of_elements_initialized, vc_vector_max_count(vector)); + ASSERT_EQ(max_size_of_vector_initialized, vc_vector_max_size(vector)); + + for (int i = 0; (size_t)i < count_of_elements_ended; ++i) { + ASSERT_TRUE(vc_vector_push_back(vector, &i)); + } + + ASSERT_EQ(count_of_elements_ended, vc_vector_count(vector)); + ASSERT_FALSE(vc_vector_empty(vector)); + ASSERT_EQ(size_of_vector_ended, vc_vector_size(vector)); + ASSERT_EQ(max_count_of_vector_ended, vc_vector_max_count(vector)); + ASSERT_EQ(max_size_of_vector_ended, vc_vector_max_size(vector)); + + const size_t test_reserve_count_fail = 10; + ASSERT_FALSE(vc_vector_reserve_count(vector, test_reserve_count_fail)); + + const size_t test_reserve_count = 35; + ASSERT_TRUE(vc_vector_reserve_count(vector, test_reserve_count)); + ASSERT_EQ(test_reserve_count, vc_vector_max_count(vector)); + ASSERT_EQ(test_reserve_count * size_of_element, vc_vector_max_size(vector)); + + // Second time with the same value + ASSERT_TRUE(vc_vector_reserve_count(vector, test_reserve_count)); + ASSERT_EQ(test_reserve_count, vc_vector_max_count(vector)); + ASSERT_EQ(test_reserve_count * size_of_element, vc_vector_max_size(vector)); + + const size_t test_reserve_size = 123 * size_of_element; + ASSERT_TRUE(vc_vector_reserve_size(vector, test_reserve_size)); + ASSERT_EQ(test_reserve_size / size_of_element, vc_vector_max_count(vector)); + ASSERT_EQ(test_reserve_size, vc_vector_max_size(vector)); + + vc_vector_release(vector); + + printf("%s passed.\n", __func__); +} + +void test_vc_vector_modifiers() { + const int begin[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 + }; + const size_t size_of_element = sizeof(begin[0]); + const size_t count_of_elements = sizeof(begin) / size_of_element; + + // After deleting first, last and one middle elements + const int after_deleting_some_elements[] = { + /* 1, */ 2, 3, 4, 5, 6, 7, 8, 9, 10, /* 11, */ 12, 13, 14, 15, 16, 17, 18, 19, /* 20 */ + }; + + // After deleting first 3 elemets from begin, 4 from middle and 3 from end + const int after_deleting_some_ranges[] = { + /* 1, 2, 3, */ 4, 5, 6, 7, 8, /* 9, 10, 11, 12, */ 13, 14, 15, 16, 17, /* 18, 19, 20 */ + }; + + vc_vector* vector = vc_vector_create(0, size_of_element, NULL); + ASSERT_NE(NULL, vector); + + // Append test + + ASSERT_TRUE(vc_vector_append(vector, begin, count_of_elements)); + + ASSERT_EQ(count_of_elements, vc_vector_count(vector)); + for (size_t i = 0; i < vc_vector_count(vector); ++i) { + ASSERT_EQ(begin[i], *(int*)vc_vector_at(vector, i)); + } + + // Pop back test + + while (vc_vector_count(vector) > 0) { + ASSERT_TRUE(vc_vector_pop_back(vector)); + } + + ASSERT_TRUE(vc_vector_empty(vector)); + + // Push back test + + for (size_t i = 0; i < count_of_elements; ++i) { + ASSERT_TRUE(vc_vector_push_back(vector, &begin[i])); + } + + ASSERT_EQ(count_of_elements, vc_vector_count(vector)); + for (size_t i = 0; i < vc_vector_count(vector); ++i) { + ASSERT_EQ(begin[i], *(int*)vc_vector_at(vector, i)); + } + + // Erase test + + vc_vector_clear(vector); + ASSERT_TRUE(vc_vector_append(vector, begin, count_of_elements)); + + ASSERT_TRUE(vc_vector_erase(vector, 0)); + ASSERT_TRUE(vc_vector_erase(vector, vc_vector_count(vector) - 1)); + ASSERT_TRUE(vc_vector_erase(vector, vc_vector_count(vector) / 2)); + + ASSERT_EQ(sizeof(after_deleting_some_elements) / size_of_element, vc_vector_count(vector)); + for (size_t i = 0; i < vc_vector_count(vector); ++i) { + ASSERT_EQ(after_deleting_some_elements[i], *(int*)vc_vector_at(vector, i)); + } + + // Erase range test + + vc_vector_clear(vector); + ASSERT_TRUE(vc_vector_append(vector, begin, count_of_elements)); + + ASSERT_TRUE(vc_vector_erase_range(vector, 0, 3)); + ASSERT_TRUE(vc_vector_erase_range(vector, vc_vector_count(vector) - 3, vc_vector_count(vector))); + ASSERT_TRUE(vc_vector_erase_range(vector, vc_vector_count(vector) / 2 - 2, vc_vector_count(vector) / 2 + 2)); + + ASSERT_EQ(sizeof(after_deleting_some_ranges) / size_of_element, vc_vector_count(vector)); + for (size_t i = 0; i < vc_vector_count(vector); ++i) { + ASSERT_EQ(after_deleting_some_ranges[i], *(int*)vc_vector_at(vector, i)); + } + + // Insert test + + vc_vector_clear(vector); + for (size_t i = 1; i < count_of_elements - 1; ++i) { + ASSERT_TRUE(vc_vector_insert(vector, i - 1, &begin[i])); + } + + ASSERT_TRUE(vc_vector_insert(vector, 0, &begin[0])); + ASSERT_TRUE(vc_vector_insert(vector, vc_vector_count(vector), &begin[count_of_elements - 1])); + + ASSERT_EQ(count_of_elements, vc_vector_count(vector)); + for (size_t i = 0; i < vc_vector_count(vector); ++i) { + ASSERT_EQ(begin[i], *(int*)vc_vector_at(vector, i)); + } + + vc_vector_release(vector); + + printf("%s passed.\n", __func__); +} + +void test_vc_vector_strfreefunc(void *data) { + free(*(char **)data); +} + +void test_vc_vector_with_strfreefunc() { + // creates a vector of pointers to char, i.e. a vector of variable sized strings + vc_vector* vector = vc_vector_create(3, sizeof(char *), test_vc_vector_strfreefunc); + ASSERT_NE(NULL, vector); + + char *strs[] = { + mystrdup("abcde"), + mystrdup("edcba"), + mystrdup("1234554321"), + mystrdup("!@#$%"), + mystrdup("not empty string"), + mystrdup(""), + mystrdup("Hello World"), + mystrdup("xxxxx"), + mystrdup("yyyyy") + }; + + for (size_t i = 0; i < 3; ++i) { + ASSERT_TRUE(vc_vector_push_back(vector, &strs[i])); + } + + ASSERT_EQ(3, vc_vector_count(vector)); + + for (size_t i = 3; i < 6; ++i) { + ASSERT_TRUE(vc_vector_insert(vector, i, &strs[i])); + } + + ASSERT_EQ(6, vc_vector_count(vector)); + vc_vector_clear(vector); // strs[0-6] were freed + ASSERT_EQ(0, vc_vector_count(vector)); + + for (size_t i = 6; i < 9; ++i) { + ASSERT_TRUE(vc_vector_push_back(vector, &strs[i])); + } + + ASSERT_EQ(3, vc_vector_count(vector)); + + vc_vector_release(vector); + + printf("%s passed.\n", __func__); +} + +void vc_vector_run_tests() { + test_vc_vector_create(); + test_vc_vector_element_access(); + test_vc_vector_iterators(); + test_vc_vector_capacity(); + test_vc_vector_modifiers(); + test_vc_vector_with_strfreefunc(); +} + +int main() { + vc_vector_run_tests(); + printf("Tests passed.\n"); + return 0; +} diff --git a/tools/fado/lib/vc_vector/vc_vector_test.h b/tools/fado/lib/vc_vector/vc_vector_test.h new file mode 100644 index 0000000..9439ea3 --- /dev/null +++ b/tools/fado/lib/vc_vector/vc_vector_test.h @@ -0,0 +1,6 @@ +#ifndef VCVECTORTESTS_H +#define VCVECTORTESTS_H + +void vc_vector_run_tests(); + +#endif // VCVECTORTESTS_H diff --git a/tools/fado/ovl_En_Hs_reloc.s b/tools/fado/ovl_En_Hs_reloc.s new file mode 100644 index 0000000..d41d172 --- /dev/null +++ b/tools/fado/ovl_En_Hs_reloc.s @@ -0,0 +1,46 @@ +.section .ovl +# ovl_En_Hs2OverlayInfo +.word _ovl_En_Hs2SegmentTextSize +.word _ovl_En_Hs2SegmentDataSize +.word _ovl_En_Hs2SegmentRoDataSize +.word _ovl_En_Hs2SegmentBssSize + +.word 28 # relocCount + +# TEXT RELOCS +.word 0x45000084 # R_MIPS_HI16 0x000084 .data +.word 0x4600008C # R_MIPS_LO16 0x00008C .data +.word 0x450000B4 # R_MIPS_HI16 0x0000B4 .rodata +.word 0x460000BC # R_MIPS_LO16 0x0000BC .rodata +.word 0x450000C0 # R_MIPS_HI16 0x0000C0 func_80A6F1A4 +.word 0x460000C4 # R_MIPS_LO16 0x0000C4 func_80A6F1A4 +.word 0x450001DC # R_MIPS_HI16 0x0001DC func_80A6F1A4 +.word 0x460001E0 # R_MIPS_LO16 0x0001E0 func_80A6F1A4 +.word 0x4500022C # R_MIPS_HI16 0x00022C func_80A6F164 +.word 0x46000230 # R_MIPS_LO16 0x000230 func_80A6F164 +.word 0x44000238 # R_MIPS_26 0x000238 func_80A6F0B4 +.word 0x450003D0 # R_MIPS_HI16 0x0003D0 .rodata +.word 0x460003D8 # R_MIPS_LO16 0x0003D8 .rodata +.word 0x45000460 # R_MIPS_HI16 0x000460 .data +.word 0x46000464 # R_MIPS_LO16 0x000464 .data +.word 0x4500049C # R_MIPS_HI16 0x00049C EnHs2_OverrideLimbDraw +.word 0x460004B4 # R_MIPS_LO16 0x0004B4 EnHs2_OverrideLimbDraw +.word 0x450004A0 # R_MIPS_HI16 0x0004A0 EnHs2_PostLimbDraw +.word 0x460004B0 # R_MIPS_LO16 0x0004B0 EnHs2_PostLimbDraw + +# DATA RELOCS +.word 0x82000010 # R_MIPS_32 0x000010 EnHs2_Init +.word 0x82000014 # R_MIPS_32 0x000014 EnHs2_Destroy +.word 0x82000018 # R_MIPS_32 0x000018 EnHs2_Update +.word 0x8200001C # R_MIPS_32 0x00001C EnHs2_Draw + +# RODATA RELOCS +.word 0xC2000020 # R_MIPS_32 0x000020 .text +.word 0xC2000024 # R_MIPS_32 0x000024 .text +.word 0xC2000028 # R_MIPS_32 0x000028 .text +.word 0xC200002C # R_MIPS_32 0x00002C .text +.word 0xC2000030 # R_MIPS_32 0x000030 .text +.word 0 +.word 0 + +.word 0x00000090 # ovl_En_Hs2OverlayInfoOffset diff --git a/tools/fado/src/fado.c b/tools/fado/src/fado.c new file mode 100644 index 0000000..464f885 --- /dev/null +++ b/tools/fado/src/fado.c @@ -0,0 +1,308 @@ +/** + * Code specific to reading and outputting Zelda 64 relocations + */ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#include "fado.h" + +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "fairy/fairy.h" +#include "macros.h" +#include "vc_vector/vc_vector.h" + +/* String-finding-related functions */ + +bool Fado_CheckInProgBitsSections(Elf32_Section section, vc_vector* progBitsSections) { + Elf32_Section* i; + VC_FOREACH(i, progBitsSections) { + if (*i == section) { + return true; + } + } + return false; +} + +/** + * For each input file, construct a vector of pointers to the starts of the strings defined in that file. + */ +void Fado_ConstructStringVectors(vc_vector** stringVectors, FairyFileInfo* fileInfo, int numFiles) { + int currentFile; + size_t currentSym; + + for (currentFile = 0; currentFile < numFiles; currentFile++) { + FairySym* symtab = fileInfo[currentFile].symtabInfo.sectionData; + + stringVectors[currentFile] = vc_vector_create(0x40, sizeof(char**), NULL); + + /* Build a vector of pointers to defined symbols' names */ + for (currentSym = 0; currentSym < fileInfo[currentFile].symtabInfo.sectionEntryCount; currentSym++) { + if ((symtab[currentSym].st_shndx != STN_UNDEF) && + Fado_CheckInProgBitsSections(symtab[currentSym].st_shndx, fileInfo[currentFile].progBitsSections)) { + /* Have to pass a double pointer so it copies the pointer instead of the start of the string */ + char* stringPtr = &fileInfo[currentFile].strtab[symtab[currentSym].st_name]; + assert(vc_vector_push_back(stringVectors[currentFile], &stringPtr)); + } + } + } +} + +bool Fado_FindSymbolNameInOtherFiles(const char* name, int thisFile, vc_vector** stringVectors, int numFiles) { + int currentFile; + char** currentString; + + for (currentFile = 0; currentFile < numFiles; currentFile++) { + if (currentFile == thisFile) { + continue; + } + VC_FOREACH(currentString, stringVectors[currentFile]) { + if (strcmp(name, *currentString) == 0) { + FAIRY_DEBUG_PRINTF("Match found for %s\n", name); + return true; + } + } + } + FAIRY_DEBUG_PRINTF("No match found for %s\n", name); + return false; +} + +void Fado_DestroyStringVectors(vc_vector** stringVectors, int numFiles) { + int currentFile; + for (currentFile = 0; currentFile < numFiles; currentFile++) { + vc_vector_release(stringVectors[currentFile]); + } + free(stringVectors); +} + +typedef struct { + size_t symbolIndex; + int file; + uint32_t relocWord; +} FadoRelocInfo; + +/* Construct the Zelda64ovl-compatible reloc word from an ELF reloc */ +FadoRelocInfo Fado_MakeReloc(int file, FairySection section, FairyRela* data) { + FadoRelocInfo relocInfo = { 0 }; + uint32_t sectionPrefix = 0; + + relocInfo.symbolIndex = ELF32_R_SYM(data->r_info); + relocInfo.file = file; + + switch (section) { + case FAIRY_SECTION_TEXT: + sectionPrefix = 1; + break; + + case FAIRY_SECTION_DATA: + sectionPrefix = 2; + break; + + case FAIRY_SECTION_RODATA: + sectionPrefix = 3; + break; + + default: + fprintf(stderr, "warning: Relocation section is invalid.\n"); + break; + } + relocInfo.relocWord = + ((sectionPrefix & 3) << 0x1E) | (ELF32_R_TYPE(data->r_info) << 0x18) | (data->r_offset & 0xFFFFFF); + + return relocInfo; +} + +static const FairyDefineString relSectionNames[] = { + FAIRY_DEF_STRING(FAIRY_SECTION_, TEXT), + FAIRY_DEF_STRING(FAIRY_SECTION_, DATA), + FAIRY_DEF_STRING(FAIRY_SECTION_, RODATA), + { 0 }, +}; + +/* Taken from elf.h/mips_elf.h */ +static const FairyDefineString relTypeNames[] = { + FAIRY_DEF_STRING(, R_MIPS_NONE), /* No reloc */ + FAIRY_DEF_STRING(, R_MIPS_16), /* Direct 16 bit */ + FAIRY_DEF_STRING(, R_MIPS_32), /* Direct 32 bit */ + FAIRY_DEF_STRING(, R_MIPS_REL32), /* PC relative 32 bit */ + FAIRY_DEF_STRING(, R_MIPS_26), /* Direct 26 bit shifted */ + FAIRY_DEF_STRING(, R_MIPS_HI16), /* High 16 bit */ + FAIRY_DEF_STRING(, R_MIPS_LO16), /* Low 16 bit */ + FAIRY_DEF_STRING(, R_MIPS_GPREL16), /* GP relative 16 bit */ + FAIRY_DEF_STRING(, R_MIPS_LITERAL), /* 16 bit literal entry */ + FAIRY_DEF_STRING(, R_MIPS_GOT16), /* 16 bit GOT entry */ + FAIRY_DEF_STRING(, R_MIPS_PC16), /* PC relative 16 bit */ + FAIRY_DEF_STRING(, R_MIPS_CALL16), /* 16 bit GOT entry for function */ + FAIRY_DEF_STRING(, R_MIPS_GPREL32), /* GP relative 32 bit */ + FAIRY_DEF_STRING(, R_MIPS_SHIFT5), + FAIRY_DEF_STRING(, R_MIPS_SHIFT6), + FAIRY_DEF_STRING(, R_MIPS_64), + FAIRY_DEF_STRING(, R_MIPS_GOT_DISP), + FAIRY_DEF_STRING(, R_MIPS_GOT_PAGE), + FAIRY_DEF_STRING(, R_MIPS_GOT_OFST), + FAIRY_DEF_STRING(, R_MIPS_GOT_HI16), + FAIRY_DEF_STRING(, R_MIPS_GOT_LO16), + FAIRY_DEF_STRING(, R_MIPS_SUB), + FAIRY_DEF_STRING(, R_MIPS_INSERT_A), + FAIRY_DEF_STRING(, R_MIPS_INSERT_B), + FAIRY_DEF_STRING(, R_MIPS_DELETE), + FAIRY_DEF_STRING(, R_MIPS_HIGHER), + FAIRY_DEF_STRING(, R_MIPS_HIGHEST), + FAIRY_DEF_STRING(, R_MIPS_CALL_HI16), + FAIRY_DEF_STRING(, R_MIPS_CALL_LO16), + FAIRY_DEF_STRING(, R_MIPS_SCN_DISP), + FAIRY_DEF_STRING(, R_MIPS_REL16), + FAIRY_DEF_STRING(, R_MIPS_ADD_IMMEDIATE), + FAIRY_DEF_STRING(, R_MIPS_PJUMP), + FAIRY_DEF_STRING(, R_MIPS_RELGOT), + FAIRY_DEF_STRING(, R_MIPS_JALR), + FAIRY_DEF_STRING(, R_MIPS_TLS_DTPMOD32), /* Module number 32 bit */ + FAIRY_DEF_STRING(, R_MIPS_TLS_DTPREL32), /* Module-relative offset 32 bit */ + FAIRY_DEF_STRING(, R_MIPS_TLS_DTPMOD64), /* Module number 64 bit */ + FAIRY_DEF_STRING(, R_MIPS_TLS_DTPREL64), /* Module-relative offset 64 bit */ + FAIRY_DEF_STRING(, R_MIPS_TLS_GD), /* 16 bit GOT offset for GD */ + FAIRY_DEF_STRING(, R_MIPS_TLS_LDM), /* 16 bit GOT offset for LDM */ + FAIRY_DEF_STRING(, R_MIPS_TLS_DTPREL_HI16), /* Module-relative offset, high 16 bits */ + FAIRY_DEF_STRING(, R_MIPS_TLS_DTPREL_LO16), /* Module-relative offset, low 16 bits */ + FAIRY_DEF_STRING(, R_MIPS_TLS_GOTTPREL), /* 16 bit GOT offset for IE */ + FAIRY_DEF_STRING(, R_MIPS_TLS_TPREL32), /* TP-relative offset, 32 bit */ + FAIRY_DEF_STRING(, R_MIPS_TLS_TPREL64), /* TP-relative offset, 64 bit */ + FAIRY_DEF_STRING(, R_MIPS_TLS_TPREL_HI16), /* TP-relative offset, high 16 bits */ + FAIRY_DEF_STRING(, R_MIPS_TLS_TPREL_LO16), /* TP-relative offset, low 16 bits */ + FAIRY_DEF_STRING(, R_MIPS_GLOB_DAT), + FAIRY_DEF_STRING(, R_MIPS_COPY), + FAIRY_DEF_STRING(, R_MIPS_JUMP_SLOT), + FAIRY_DEF_STRING(, R_MIPS_NUM), +}; + +/** + * Find all the necessary relocations to retain (those defined in any input file), and print them in the appropriate + * format. + */ +void Fado_Relocs(FILE* outputFile, int inputFilesCount, FILE** inputFiles, const char* ovlName) { + /* General information structs */ + FairyFileInfo* fileInfos = malloc(inputFilesCount * sizeof(FairyFileInfo)); + + /* Symbol tables for each file */ + FairySym** symtabs = malloc(inputFilesCount * sizeof(FairySym*)); + + /* Lists of names of symbols defined in files of the overlay */ + vc_vector** stringVectors = malloc(inputFilesCount * sizeof(vc_vector*)); + + /* The relocs in the format we will print */ + vc_vector* relocList[FAIRY_SECTION_OTHER]; /* Maximum number of reloc sections */ + + /* Offset of current file's current section into the overlay's whole section */ + uint32_t sectionOffset[FAIRY_SECTION_OTHER] = { 0 }; + + /* Total number of relocs */ + uint32_t relocCount = 0; + + /* iterators */ + int currentFile; + FairySection section; + size_t relocIndex; + + for (currentFile = 0; currentFile < inputFilesCount; currentFile++) { + FAIRY_INFO_PRINTF("Begin initialising file %d info.\n", currentFile); + Fairy_InitFile(&fileInfos[currentFile], inputFiles[currentFile]); + FAIRY_INFO_PRINTF("Initialising file %d info complete.\n", currentFile); + + symtabs[currentFile] = fileInfos[currentFile].symtabInfo.sectionData; + } + + Fado_ConstructStringVectors(stringVectors, fileInfos, inputFilesCount); + FAIRY_INFO_PRINTF("%s", "symtabs set\n"); + + /* Construct relocList of all relevant relocs */ + for (section = FAIRY_SECTION_TEXT; section < FAIRY_SECTION_OTHER; section++) { + relocList[section] = vc_vector_create(0x100, sizeof(FadoRelocInfo), NULL); + + for (currentFile = 0; currentFile < inputFilesCount; currentFile++) { + FairyRela* relSection = fileInfos[currentFile].relocTablesInfo[section].sectionData; + + if (relSection != NULL) { + for (relocIndex = 0; relocIndex < fileInfos[currentFile].relocTablesInfo[section].sectionEntryCount; + relocIndex++) { + FadoRelocInfo currentReloc = Fado_MakeReloc(currentFile, section, &relSection[relocIndex]); + + if ((symtabs[currentFile][currentReloc.symbolIndex].st_shndx != STN_UNDEF) || + Fado_FindSymbolNameInOtherFiles( + &fileInfos[currentFile].strtab[symtabs[currentFile][currentReloc.symbolIndex].st_name], + currentFile, stringVectors, inputFilesCount)) { + + currentReloc.relocWord += sectionOffset[section]; + FAIRY_DEBUG_PRINTF("current section offset: %d\n", sectionOffset[section]); + vc_vector_push_back(relocList[section], ¤tReloc); + relocCount++; + } + } + } else { + FAIRY_INFO_PRINTF("%s", "Ignoring empty reloc section\n"); + } + + sectionOffset[section] += fileInfos[currentFile].progBitsSizes[section]; + FAIRY_INFO_PRINTF("section offset: %d\n", sectionOffset[section]); + } + } + + { + /* Write header */ + fprintf(outputFile, ".section .ovl\n"); + fprintf(outputFile, "# %sOverlayInfo\n", ovlName); + fprintf(outputFile, ".word _%sSegmentTextSize\n", ovlName); + fprintf(outputFile, ".word _%sSegmentDataSize\n", ovlName); + fprintf(outputFile, ".word _%sSegmentRoDataSize\n", ovlName); + fprintf(outputFile, ".word _%sSegmentBssSize\n", ovlName); + + fprintf(outputFile, "\n.word %d # relocCount\n", relocCount); + + /* Write reloc table */ + for (section = FAIRY_SECTION_TEXT; section < FAIRY_SECTION_OTHER; section++) { + if (vc_vector_count(relocList[section]) == 0) { + FAIRY_INFO_PRINTF("%s", "Ignoring empty reloc section\n"); + continue; + } + + fprintf(outputFile, "\n# %s RELOCS\n", Fairy_StringFromDefine(relSectionNames, section)); + + { + FadoRelocInfo* currentReloc; + VC_FOREACH(currentReloc, relocList[section]) { + fprintf(outputFile, ".word 0x%X # %-11s 0x%06X %s\n", currentReloc->relocWord, + Fairy_StringFromDefine(relTypeNames, (currentReloc->relocWord >> 0x18) & 0x3F), + currentReloc->relocWord & 0xFFFFFF, + Fairy_GetSymbolName(symtabs[currentReloc->file], fileInfos[currentReloc->file].strtab, + currentReloc->symbolIndex)); + } + } + } + + /* print pads and section size */ + for (relocCount += 5; ((relocCount + 1) & 3) != 0; relocCount++) { + fprintf(outputFile, ".word 0\n"); + } + fprintf(outputFile, "\n.word 0x%08X # %sOverlayInfoOffset\n", 4 * (relocCount + 1), ovlName); + } + + for (currentFile = 0; currentFile < inputFilesCount; currentFile++) { + Fairy_DestroyFile(&fileInfos[currentFile]); + FAIRY_INFO_PRINTF("Freed file %d\n", currentFile); + } + + for (section = FAIRY_SECTION_TEXT; section < FAIRY_SECTION_OTHER; section++) { + if (relocList[section] != NULL) { + vc_vector_release(relocList[section]); + } + FAIRY_INFO_PRINTF("Freed relocList[%d]\n", section); + } + + Fado_DestroyStringVectors(stringVectors, inputFilesCount); + FAIRY_INFO_PRINTF("%s", "Freed string vectors\n"); + free(symtabs); + FAIRY_INFO_PRINTF("%s", "Freed symtabs\n"); + free(fileInfos); +} diff --git a/tools/fado/src/help.c b/tools/fado/src/help.c new file mode 100644 index 0000000..1f05807 --- /dev/null +++ b/tools/fado/src/help.c @@ -0,0 +1,162 @@ +/** + * Getopt-compatible printing of formatted help. + */ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#include "help.h" + +#include <assert.h> +#include <getopt.h> +#include <stdbool.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> + +#include "macros.h" + +/* Values of the variables used by Help_PrintHelp. Defaults are taken from common terminal programs like grep */ +size_t helpTextWidth = 80; +size_t helpDtIndent = 2; +size_t helpDdIndent = 25; + +/** + * Prints a paragraph, word wrapped to helpTextWidth, with a hanging indent. initialColumn is used to determine how wide + * the first line should be, while indentFirstLine should be true if there is no previous text on the line (an ordinary + * paragraph), and false if there is (e.g. as a description list description) + */ +void Help_PrintFlowAndIndent(const char* string, size_t initialColumn, size_t textWidth, size_t hangingIndent, + bool indentFirstLine) { + size_t column = initialColumn; + size_t index = 0; + size_t inLength = strlen(string); + size_t lookAhead; + bool shouldBreak; + + assert(initialColumn < textWidth); + assert(hangingIndent < textWidth); + + if (indentFirstLine) { + printf("%*s", (int)initialColumn, ""); + } + + for (; index <= inLength; index++) { + shouldBreak = 0; + + if (column == textWidth) { + printf("%c\n%*s", string[index], (int)hangingIndent, ""); + column = hangingIndent; + continue; + } + + column++; + + switch (string[index]) { + case '\0': + return; + + case ' ': + if (column == hangingIndent) { + continue; + } + + for (lookAhead = 0; lookAhead <= textWidth - column; lookAhead++) { + // printf("%c\n", src[index + lookAhead]); + if (string[index + lookAhead + 1] == ' ' || string[index + lookAhead + 1] == '\0') { + putchar(' '); + shouldBreak = 1; + break; + } + } + if (shouldBreak) { /* Damn shared keywords. */ + break; + } + case '\n': + printf("\n%*s", (int)hangingIndent, ""); + column = hangingIndent; + break; + + default: + putchar(string[index]); + break; + } + } +} + +/** + * Prints help in the form + * ``` + * prologue (word wrapped) + * + * Positional arguments + * arg1 Description (word + * wrapped) + * arg2 Description (word + * wrapped) + * + * Options + * -a --long-name=ARG Description (word + * wrapped) + * + * epilogue (word wrapped) + * ``` + * where the positional arguments are described using the posArgInfo array, and options are fed using the OptInfo array, + * which should both be null-terminated. posArgCount/optCount is the actual number of positional arguments/options: it + * is used to guarantee no garbage is printed even if the user has not null-terminated the array. (optCount is required + * for constructing the getopt option array anyway.) + */ +void Help_PrintHelp(const char* prologue, size_t posArgCount, const PosArgInfo* posArgInfo, size_t optCount, + const OptInfo* optInfo, const char* epilogue) { + size_t i; + size_t dtLength; + int padding; + + Help_PrintFlowAndIndent(prologue, 0, helpTextWidth, 0, false); + + if (posArgCount != 0) { + printf("\nPositional Argument\n"); + for (i = 0; i < posArgCount; i++) { + if (posArgInfo[i].helpArg == 0) { + break; + } + + dtLength = helpDtIndent + strlen(posArgInfo[i].helpArg); + + padding = helpDdIndent - dtLength - 2; + printf("%*s%s%*s ", (int)helpDtIndent, "", posArgInfo[i].helpArg, CLAMP_MIN(padding, 0), ""); + + Help_PrintFlowAndIndent(posArgInfo[i].helpMsg, CLAMP_MIN(dtLength + 2, helpDdIndent), helpTextWidth, + helpDdIndent, false); + printf("\n"); + } + } + + if (optCount != 0) { + printf("\nOptions\n"); + + for (i = 0; i < optCount; i++) { + if (optInfo[i].longOpt.val == 0) { + break; + } + + dtLength = helpDtIndent + 6 + strlen(optInfo[i].longOpt.name); + + if (optInfo[i].helpArg == NULL) { + padding = helpDdIndent - dtLength - 2; + printf("%*s-%c, --%s%*s ", (int)helpDtIndent, "", optInfo[i].longOpt.val, optInfo[i].longOpt.name, + CLAMP_MIN(padding, 0), ""); + } else { + dtLength += 1 + strlen(optInfo[i].helpArg); + padding = helpDdIndent - dtLength - 2; + printf("%*s-%c, --%s=%s%*s ", (int)helpDtIndent, "", optInfo[i].longOpt.val, optInfo[i].longOpt.name, + optInfo[i].helpArg, CLAMP_MIN(padding, 0), ""); + } + Help_PrintFlowAndIndent(optInfo[i].helpMsg, CLAMP_MIN(dtLength + 2, helpDdIndent), helpTextWidth, + helpDdIndent, false); + printf("\n"); + } + } + + printf("\n"); + Help_PrintFlowAndIndent(epilogue, 0, helpTextWidth, 0, false); + printf("\n"); +} diff --git a/tools/fado/src/main.c b/tools/fado/src/main.c new file mode 100644 index 0000000..e6b7926 --- /dev/null +++ b/tools/fado/src/main.c @@ -0,0 +1,247 @@ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <getopt.h> + +#include "macros.h" +#include "fairy/fairy.h" +#include "fado.h" +#include "help.h" +#include "mido.h" +#include "vc_vector/vc_vector.h" + +#include "version.inc" + +void PrintVersion(void) { + printf("Fado (Fairy-Assisted relocations for Decompiled Overlays), version %s\n", versionNumber); + printf("Copyright (C) 2021 Elliptic Ellipsis\n"); + printf("%s\n", credits); + printf("Repository available at %s.\n", repo); +} + +#if defined _WIN32 || defined __CYGWIN__ +#define PATH_SEPARATOR '\\' +#else +#define PATH_SEPARATOR '/' +#endif + +/** + * (Bad) filename-parsing idea to get the overlay name from the filename. Output must be freed separately. + */ +char* GetOverlayNameFromFilename(const char* src) { + char* ret; + const char* ptr; + const char* start = src; + const char* end = src; + + for (ptr = src; *ptr != '\0'; ptr++) { + if (*ptr == PATH_SEPARATOR) { + start = end + 1; + end = ptr; + } + } + + if (end == src) { + return NULL; + } + + ret = malloc((end - start + 1) * sizeof(char)); + memcpy(ret, start, end - start); + ret[end - start] = '\0'; + + return ret; +} + +#define OPTSTR "M:n:o:v:ahV" +#define USAGE_STRING "Usage: %s [-hV] [-n name] [-o output_file] [-v level] input_files ...\n" + +#define HELP_PROLOGUE \ + "Fado (Fairy-Assisted relocations for Decompiled Overlays\n" \ + "Extract relocations from object files and convert them into the format required by Zelda 64 overlays.\n" +#define HELP_EPILOGUE repo + +// clang-format off +static const PosArgInfo posArgInfo[] = { + { "INPUT_FILE", "Every positional argument is an input file, and there should be at least one input file. All inputs are relocatable .o (object) ELF files" }, + { NULL, NULL } +}; + +static const OptInfo optInfo[] = { + { { "make-dependency", required_argument, NULL, 'M' }, "FILE", "Write the output file's Makefile dependencies to FILE" }, + { { "name", required_argument, NULL, 'n' }, "NAME", "Use NAME as the overlay name. Will use the deepest folder name in the input file's path if not specified" }, + { { "output-file", required_argument, NULL, 'o' }, "FILE", "Output to FILE. Will use stdout if none is specified" }, + { { "verbosity", required_argument, NULL, 'v' }, "N", "Verbosity level, one of 0 (None, default), 1 (Info), 2 (Debug)" }, + + { { "alignment", no_argument, NULL, 'a' }, NULL, "Experimental. Use the alignment declared by each section in the elf file instead of padding to 0x10 bytes. NOTE: It has not been properly tested because the tools we currently have are not compatible non 0x10 alignment" }, + + { { "help", no_argument, NULL, 'h' }, NULL, "Display this message and exit" }, + { { "version", no_argument, NULL, 'V' }, NULL, "Display version information" }, + + { { NULL, 0, NULL, '\0' }, NULL, NULL }, +}; +// clang-format on + +static size_t posArgCount = ARRAY_COUNT(posArgInfo); +static size_t optCount = ARRAY_COUNT(optInfo); +static struct option longOptions[ARRAY_COUNT(optInfo)]; + +void ConstructLongOpts(void) { + size_t i; + + for (i = 0; i < optCount; i++) { + longOptions[i] = optInfo[i].longOpt; + } +} + +int main(int argc, char** argv) { + int opt; + int inputFilesCount; + FILE** inputFiles; + FILE* outputFile = stdout; + char* outputFileName; + char* dependencyFileName = NULL; + char* ovlName = NULL; + + ConstructLongOpts(); + + if (argc < 2) { + printf(USAGE_STRING, argv[0]); + fprintf(stderr, "No input file specified\n"); + return EXIT_FAILURE; + } + + while (true) { + int optionIndex = 0; + + if ((opt = getopt_long(argc, argv, OPTSTR, longOptions, &optionIndex)) == -1) { + break; + } + + switch (opt) { + case 'M': + dependencyFileName = optarg; + break; + + case 'n': + ovlName = optarg; + break; + + case 'o': + outputFileName = optarg; + outputFile = fopen(optarg, "wb"); + if (outputFile == NULL) { + fprintf(stderr, "error: unable to open output file '%s' for writing\n", optarg); + return EXIT_FAILURE; + } + break; + + case 'v': + if (sscanf(optarg, "%u", &gVerbosity) == 0) { + fprintf(stderr, "warning: verbosity argument '%s' should be a nonnegative decimal integer\n", + optarg); + } + break; + + case 'a': +#ifndef EXPERIMENTAL + goto not_experimental_err; +#endif + gUseElfAlignment = true; + break; + + case 'h': + printf(USAGE_STRING, argv[0]); + Help_PrintHelp(HELP_PROLOGUE, posArgCount, posArgInfo, optCount, optInfo, HELP_EPILOGUE); + return EXIT_FAILURE; + + case 'V': + PrintVersion(); + return EXIT_FAILURE; + + default: + fprintf(stderr, "?? getopt returned character code 0x%X ??\n", opt); + break; + } + } + + FAIRY_INFO_PRINTF("%s", "Options processed\n"); + + { + int i; + + inputFilesCount = argc - optind; + if (inputFilesCount == 0) { + fprintf(stderr, "No input files specified. Exiting.\n"); + return EXIT_FAILURE; + } + + inputFiles = malloc(inputFilesCount * sizeof(FILE*)); + for (i = 0; i < inputFilesCount; i++) { + FAIRY_INFO_PRINTF("Using input file %s\n", argv[optind + i]); + inputFiles[i] = fopen(argv[optind + i], "rb"); + if (inputFiles[i] == NULL) { + fprintf(stderr, "error: unable to open input file '%s' for reading\n", argv[optind + i]); + return EXIT_FAILURE; + } + } + + FAIRY_INFO_PRINTF("Found %d input file%s\n", inputFilesCount, (inputFilesCount == 1 ? "" : "s")); + + if (ovlName == NULL) { // If a name has not been set using an arg + ovlName = GetOverlayNameFromFilename(argv[optind]); + Fado_Relocs(outputFile, inputFilesCount, inputFiles, ovlName); + free(ovlName); + } else { + Fado_Relocs(outputFile, inputFilesCount, inputFiles, ovlName); + } + + for (i = 0; i < inputFilesCount; i++) { + fclose(inputFiles[i]); + } + free(inputFiles); + if (outputFile != stdout) { + fclose(outputFile); + } + } + + if (dependencyFileName != NULL) { + int fileNameLength = strlen(outputFileName); + char* objectFile = malloc((strlen(outputFileName) + 1) * sizeof(char)); + vc_vector* inputFilesVector = vc_vector_create(inputFilesCount, sizeof(char*), NULL); + char* extensionStart; + FILE* dependencyFile = fopen(dependencyFileName, "w"); + + if (dependencyFile == NULL) { + fprintf(stderr, "error: unable to open dependency file '%s' for writing\n", dependencyFileName); + return EXIT_FAILURE; + } + + strcpy(objectFile, outputFileName); + extensionStart = strrchr(objectFile, '.'); + if (extensionStart == objectFile + fileNameLength) { + fprintf(stderr, "error: file name should not end in a '.'\n"); + return EXIT_FAILURE; + } + strcpy(extensionStart, ".o"); + vc_vector_append(inputFilesVector, &argv[optind], inputFilesCount); + + Mido_WriteDependencyFile(dependencyFile, objectFile, inputFilesVector); + + free(objectFile); + vc_vector_release(inputFilesVector); + fclose(dependencyFile); + } + + return EXIT_SUCCESS; + + goto not_experimental_err; // silences a warning +not_experimental_err: + fprintf( + stderr, + "Experimental option '-%c' passed in a non-EXPERIMENTAL build. Rebuild with 'make EXPERIMENTAL=1' to enable.\n", + opt); + return EXIT_FAILURE; +} diff --git a/tools/fado/src/mido.c b/tools/fado/src/mido.c new file mode 100644 index 0000000..bc5f3ed --- /dev/null +++ b/tools/fado/src/mido.c @@ -0,0 +1,20 @@ +#include "mido.h" + +#include <stdio.h> +#include "macros.h" +#include "vc_vector/vc_vector.h" + +int Mido_WriteDependencyFile(FILE* dependencyFile, const char* relocFile, vc_vector* inputFilesVector) { + char** inputFile; + + fprintf(dependencyFile, "%s:", relocFile); + + VC_FOREACH(inputFile, inputFilesVector) { + fprintf(dependencyFile, " %s", *inputFile); + } + fputs("\n\n", dependencyFile); + VC_FOREACH(inputFile, inputFilesVector) { + fprintf(dependencyFile, "%s:\n\n", *inputFile); + } + return 0; +} diff --git a/tools/fado/src/version.inc b/tools/fado/src/version.inc new file mode 100644 index 0000000..9bc9fbf --- /dev/null +++ b/tools/fado/src/version.inc @@ -0,0 +1,5 @@ +/* Copyright (C) 2021 Elliptic Ellipsis */ +/* SPDX-License-Identifier: AGPL-3.0-only */ +const char versionNumber[] = "1.3.1"; +const char credits[] = "Written by Elliptic Ellipsis\nwith additions from AngheloAlf and Tharo"; +const char repo[] = "https://github.com/EllipticEllipsis/fado/"; diff --git a/tools/fado/z64_relocation_section_format.md b/tools/fado/z64_relocation_section_format.md new file mode 100644 index 0000000..c537ff7 --- /dev/null +++ b/tools/fado/z64_relocation_section_format.md @@ -0,0 +1,114 @@ +# Zelda 64 overlay relocation section format + +Both Zelda 64 titles use the same custom dynamic overlay relocation format, which is + + +All elements are 4 bytes in width. + +| Offset | Description | Notes | +| ------- | ------------------------------------------- | ------------------------------------------------------------- | +| 0x00 | Size of overlay .text section | | +| 0x04 | Size of overlay .data section | | +| 0x08 | Size of overlay .rodata section | | +| 0x0C | Size of overlay .bss section | | +| 0x10 | Number of relocation entries | | +| 0x14- | Relocation entries | Must be sorted in increasing order by section, then offset | +| ... | | | +| | (zero padding of section to 0x10 alignment) | | +| End - 4 | Size of overlay .ovl section | Also the offset from the end of the rest of the section sizes | + + +## Relocation entries + +The only element that is not a single number are the relocation entries, which are bitpacked as follows: + +| 0x1F..0x1E | 0x1D..0x18 | 0x17..0x0 | +| ---------- | ---------- | ----------------------------- | +| ss | tttttt | oooo oooo oooo oooo oooo oooo | +| Section | Type | Offset | + + +### Section + +2 bits. Section where the instruction or data to be relocated is. + +| Value | Section | +| ----- | ------- | +| 1 | .text | +| 2 | .data | +| 3 | .rodata | + + +### Type + +6 bits. Four types of standard MIPS relocation are supported. They use the same values as the standard elf formats: + +| Value | Type | Description | +| ----- | ------------- | --------------------------------------------------------------------------------- | +| 2 | `R_MIPS_32` | A full word address (such as a pointer in data or an address in a jumptable) | +| 4 | `R_MIPS_26` | 26-bit direct relocation, for a J-type instruction | +| 5 | `R_MIPS_HI16` | High 16-bit, generally the top half of an address in an `li`/`lui` | +| 6 | `R_MIPS_LO16` | Low 16-bit, the bottom half of an address, such as in an `addiu`,`ori`,`lh`, etc. | + + +### Offset + +0x18 bits. Offset in bytes from the start of the section where the relocation occurs. + + +### Example + +``` +0x82000A30 = 0b1000 0010 0000 0000 0000 1010 0011 0000 +``` + +This splits as + +``` +0b10, 0b000010, 0b0000 0000 0000 1010 0011 0000 = 0x2, 0x2, 0xA30 +``` + +i.e. a full-word (`R_MIPS_32`) relocation at `.data + 0xA30`. + + +## Compiler compatibility + +### HI/LO + +The MIPS ELF format standard specifies that each LO be preceded by a unique HI associated to it (but multiple LOs may associate to the same HI), and the overlay relocation function acts based on this assumption. + +IDO complies with this consistently, but GCC in its wisdom decided that it was appropriate to violate this by default, and allow multiple HIs to associate to the same LO. GCC also likes to reorder relocations in the `.rel.*` sections. + +To prevent these you must pass *both* of the following compiler flags: + +``` +-mno-explicit-relocs -mno-split-addresses +``` + +(GNU do not document this behaviour themselves, although apparently it has been present for many years. It is also not even consistent between versions.) + +### rodata + +It should be clear from the description above that this system expects a single rodata section. Again, IDO will only ever produce one rodata section, but GCC will produce several, albeit only one containing relocatable rodata: the others are for "mergeable" strings and floats/doubles. The cleanest way to deal with this is to pass + +``` +-fno-merge-constants +``` + +which will force GCC to generate a single combined rodata section. If, however, you really think you will benefit from merging constants, to obtain relocations correctly offset from the start of the entire rodata section(s), the actual `.rodata` section must be explicitly linked first. + +For multi-file overlays, the situation is even more complicated, and Fado gets around this by adding up the sizes of all the rodata sections so that we may simply place one files' in one chunk: this means that each individual `.rodata` section should be linked before the others, i.e. + +``` +.text(1) +.text(2) +.data(1) +.data(2) +.rodata(1) +.rodata.cst4(1) +... +.rodata(2) +.rodata.cst4(2) +``` + +or similar. diff --git a/tools/first_diff.py b/tools/first_diff.py new file mode 100755 index 0000000..c0702c8 --- /dev/null +++ b/tools/first_diff.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import mapfile_parser +from pathlib import Path + + +def firstDiffMain(): + parser = argparse.ArgumentParser(description="Find the first difference(s) between the built ROM and the base ROM.") + + parser.add_argument("-c", "--count", type=int, default=5, help="find up to this many instruction difference(s)") + parser.add_argument("-v", "--version", help="Which version should be processed", default="jp") + + args = parser.parse_args() + + buildFolder = Path("build") + + BUILTROM = buildFolder / f"animalforest_uncompressed.{args.version}.z64" + BUILTMAP = buildFolder / f"animalforest.{args.version}.map" + + EXPECTEDROM = "expected" / BUILTROM + EXPECTEDMAP = "expected" / BUILTMAP + + exit(mapfile_parser.frontends.first_diff.doFirstDiff(BUILTMAP, EXPECTEDMAP, BUILTROM, EXPECTEDROM, args.count, mismatchSize=True)) + +if __name__ == "__main__": + firstDiffMain() diff --git a/tools/format.py b/tools/format.py new file mode 100755 index 0000000..4b6a24d --- /dev/null +++ b/tools/format.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 + +import argparse +import glob +import multiprocessing +import os +import re +import shutil +import subprocess +import sys +import tempfile +from functools import partial +from typing import List + + +# clang-format, clang-tidy and clang-apply-replacements default version +# Version 11 is used when available for more consistency between contributors +CLANG_VER = 11 + +# Clang-Format options (see .clang-format for rules applied) +FORMAT_OPTS = "-i -style=file" + +# Clang-Tidy options (see .clang-tidy for checks enabled) +TIDY_OPTS = "-p ." +TIDY_FIX_OPTS = "--fix --fix-errors" + +# Clang-Apply-Replacements options (used for multiprocessing) +APPLY_OPTS = "" + +# Compiler options used with Clang-Tidy +# Normal warnings are disabled with -Wno-everything to focus only on tidying +INCLUDES = "-Iinclude -Isrc -Ibuild -I. -Ilib/ultralib/include -Ilib/ultralib/include/PR -Ibin/jp -Ibin/cn" +DEFINES = "-D_LANGUAGE_C -DNON_MATCHING -D_MIPS_SZLONG=32" +COMPILER_OPTS = f"-fno-builtin -std=gnu90 -m32 -Wno-everything {INCLUDES} {DEFINES}" + + +VERBOSE = False + +def get_clang_executable(allowed_executables: List[str]): + for executable in allowed_executables: + try: + subprocess.check_call([executable, "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return executable + except FileNotFoundError or subprocess.CalledProcessError: + pass + return None + + +def get_tidy_version(tidy_executable: str): + tidy_version_run = subprocess.run([tidy_executable, "--version"], stdout=subprocess.PIPE, universal_newlines=True) + match = re.search(r"LLVM version ([0-9]+)", tidy_version_run.stdout) + return int(match.group(1)) + + +CLANG_FORMAT = get_clang_executable([f"clang-format-{CLANG_VER}"]) +if CLANG_FORMAT is None: + sys.exit(f"Error: clang-format-{CLANG_VER} not found") + +CLANG_TIDY = get_clang_executable([f"clang-tidy-{CLANG_VER}", "clang-tidy"]) +if CLANG_TIDY is None: + sys.exit(f"Error: neither clang-tidy-{CLANG_VER} nor clang-tidy found") + +CLANG_APPLY_REPLACEMENTS = get_clang_executable([f"clang-apply-replacements-{CLANG_VER}", "clang-apply-replacements"]) + +# Try to detect the clang-tidy version and add --fix-notes for version 13+ +# This is used to ensure all fixes are applied properly in recent versions +if get_tidy_version(CLANG_TIDY) >= 13: + TIDY_FIX_OPTS += " --fix-notes" + + +def list_chunks(list: List, chunk_length: int): + for i in range(0, len(list), chunk_length): + yield list[i : i + chunk_length] + + +def run_clang_format(files: List[str]): + exec_str = f"{CLANG_FORMAT} {FORMAT_OPTS} {' '.join(files)}" + subprocess.run(exec_str, shell=True) + + +def run_clang_tidy(files: List[str]): + exec_str = f"{CLANG_TIDY} {TIDY_OPTS} {TIDY_FIX_OPTS} {' '.join(files)} -- {COMPILER_OPTS}" + if VERBOSE: + subprocess.run(exec_str, shell=True) + else: + subprocess.run(exec_str, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def run_clang_tidy_with_export(tmp_dir: str, files: List[str]): + (handle, tmp_file) = tempfile.mkstemp(suffix=".yaml", dir=tmp_dir) + os.close(handle) + + exec_str = f"{CLANG_TIDY} {TIDY_OPTS} --export-fixes={tmp_file} {' '.join(files)} -- {COMPILER_OPTS}" + if VERBOSE: + subprocess.run(exec_str, shell=True) + else: + subprocess.run(exec_str, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def run_clang_apply_replacements(tmp_dir: str): + exec_str = f"{CLANG_APPLY_REPLACEMENTS} {APPLY_OPTS} {tmp_dir}" + subprocess.run(exec_str, shell=True) + + +def add_final_new_line(file: str): + # https://backreference.org/2010/05/23/sanitizing-files-with-no-trailing-newline/index.html + # "gets the last character of the file pipes it into read, which will exit with a nonzero exit + # code if it encounters EOF before newline (so, if the last character of the file isn't a newline). + # If read exits nonzero, then append a newline onto the file using echo (if read exits 0, + # that satisfies the ||, so the echo command isn't run)." (https://stackoverflow.com/a/34865616) + exec_str = f"tail -c1 {file} | read -r _ || echo >> {file}" + subprocess.run(exec_str, shell=True) + + +def format_files(src_files: List[str], extra_files: List[str], nb_jobs: int): + if nb_jobs != 1: + print(f"Formatting files with {nb_jobs} jobs") + else: + print(f"Formatting files with a single job (consider using -j to make this faster)") + + # Format files in chunks to improve performance while still utilizing jobs + file_chunks = list(list_chunks(src_files, (len(src_files) // nb_jobs) + 1)) + + print("Running clang-format...") + # clang-format only applies changes in the given files, so it's safe to run in parallel + with multiprocessing.get_context("fork").Pool(nb_jobs) as pool: + pool.map(run_clang_format, file_chunks) + + print("Running clang-tidy...") + if nb_jobs > 1: + # clang-tidy may apply changes in #included files, so when running it in parallel we use --export-fixes + # then we call clang-apply-replacements to apply all suggested fixes at the end + tmp_dir = tempfile.mkdtemp() + + try: + with multiprocessing.get_context("fork").Pool(nb_jobs) as pool: + pool.map(partial(run_clang_tidy_with_export, tmp_dir), file_chunks) + + run_clang_apply_replacements(tmp_dir) + finally: + shutil.rmtree(tmp_dir) + else: + run_clang_tidy(src_files) + + print("Adding missing final new lines...") + # Adding final new lines is safe to do in parallel and can be applied to all types of files + with multiprocessing.get_context("fork").Pool(nb_jobs) as pool: + pool.map(add_final_new_line, src_files + extra_files) + + print("Done formatting files.") + + +def main(): + parser = argparse.ArgumentParser(description="Format files in the codebase to enforce most style rules") + parser.add_argument("files", metavar="file", nargs="*") + parser.add_argument( + "-j", + dest="jobs", + type=int, + nargs="?", + default=1, + help="number of jobs to run (default: 1 without -j, number of cpus with -j)", + ) + parser.add_argument("-v", "--verbose", action="store_true") + args = parser.parse_args() + + global VERBOSE + VERBOSE = args.verbose + + nb_jobs = args.jobs or multiprocessing.cpu_count() + if nb_jobs > 1: + if CLANG_APPLY_REPLACEMENTS is None: + sys.exit( + f"Error: neither clang-apply-replacements-{CLANG_VER} nor clang-apply-replacements found (required to use -j)" + ) + + if args.files: + files = args.files + extra_files = [] + else: + files = glob.glob("src/**/*.c", recursive=True) + extra_files = glob.glob("assets/**/*.xml", recursive=True) + + format_files(files, extra_files, nb_jobs) + + +if __name__ == "__main__": + main() diff --git a/tools/m2ctx.py b/tools/m2ctx.py new file mode 100755 index 0000000..60cae1a --- /dev/null +++ b/tools/m2ctx.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys +import subprocess +import tempfile + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.abspath(os.path.join(script_dir, "..")) + +# Project-specific +CPP_FLAGS = [ + "-I.", + "-Iinclude", + "-Ibin", + "-Ilib/ultralib/include", + "-Ilib/ultralib/include/PR", + + "-D_LANGUAGE_C", + "-DF3DEX_GBI_2", + "-D__USE_ISOC99", + "-DNDEBUG", + "-D_FINALROM", + "-D_MIPS_SZLONG=32", + "-D__USE_ISOC99", + + "-DM2CTX", + "-DCC_CHECK=1" + "-DNON_MATCHING", + + "-ffreestanding", + "-std=gnu89", +] + +def import_c_file(in_file, version: str) -> str: + in_file = os.path.relpath(in_file, root_dir) + + cpp_command = ["gcc", "-E", "-P", "-undef", "-dM", *CPP_FLAGS, in_file] + cpp_command2 = ["gcc", "-E", "-P", "-undef", *CPP_FLAGS, in_file] + + with tempfile.NamedTemporaryFile(suffix=".c") as tmp: + stock_macros = subprocess.check_output(["gcc", "-E", "-P", "-undef", "-dM", tmp.name], cwd=root_dir, encoding="utf-8") + + out_text = "" + try: + out_text += subprocess.check_output(cpp_command, cwd=root_dir, encoding="utf-8") + out_text += subprocess.check_output(cpp_command2, cwd=root_dir, encoding="utf-8") + except subprocess.CalledProcessError: + print( + "Failed to preprocess input file, when running command:\n" + + " ".join(cpp_command), + file=sys.stderr, + ) + sys.exit(1) + + if not out_text: + print("Output is empty - aborting") + sys.exit(1) + + for line in stock_macros.strip().splitlines(): + out_text = out_text.replace(line + "\n", "") + return out_text + +def main(): + parser = argparse.ArgumentParser( + description="""Create a context file which can be used for m2c""" + ) + parser.add_argument( + "c_file", + help="""File from which to create context""", + ) + parser.add_argument("-v", "--version", help="Which version should be processed", default="us", choices=["us", "cn"]) + args = parser.parse_args() + + output = import_c_file(args.c_file, args.version) + + with open(os.path.join(root_dir, "ctx.c"), "w", encoding="UTF-8") as f: + f.write(output) + + +if __name__ == "__main__": + main() diff --git a/tools/permuter_settings.toml b/tools/permuter_settings.toml new file mode 100644 index 0000000..b08f38f --- /dev/null +++ b/tools/permuter_settings.toml @@ -0,0 +1,9 @@ +compiler_type = "ido" + +[preserve_macros] +NULL = "int" +"g[DS]P.*" = "void" +"gs[DS]P.*" = "void" +"gDma.*" = "void" +"G_IM_SIZ_.*" = "int" +"G_[AC]C.*" = "int" diff --git a/tools/pj64_syms.py b/tools/pj64_syms.py new file mode 100755 index 0000000..7ff957c --- /dev/null +++ b/tools/pj64_syms.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import mapfile_parser +from pathlib import Path + + +def mapToPj64symsMain(): + parser = argparse.ArgumentParser() + parser.add_argument("-v", "--version", help="version to process", default="jp") + parser.add_argument("-o", "--output", default="DOUBUTSUNOMORI.sym", type=Path) + + args = parser.parse_args() + + version: str = args.version + output: Path = args.output + mapPath = Path("build") / f"animalforest.{version}.map" + + exit(mapfile_parser.frontends.pj64_syms.doPj64Syms(mapPath, output)) + +if __name__ == "__main__": + mapToPj64symsMain() diff --git a/tools/splat/.github/workflows/mypy.yml b/tools/splat/.github/workflows/mypy.yml new file mode 100644 index 0000000..d5bb1fa --- /dev/null +++ b/tools/splat/.github/workflows/mypy.yml @@ -0,0 +1,26 @@ +name: mypy + +on: + push: + pull_request: + +jobs: + checks: + runs-on: ubuntu-latest + name: mypy + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Dependencies + run: | + pip install mypy + pip install black + pip install -r requirements.txt + pip install types-PyYAML + - name: mypy + run: mypy --show-column-numbers --hide-error-context . + - name: black + run: black --check . diff --git a/tools/splat/.gitignore b/tools/splat/.gitignore new file mode 100644 index 0000000..38da5f7 --- /dev/null +++ b/tools/splat/.gitignore @@ -0,0 +1,10 @@ +.idea/ +venv/ +.vscode/ +__pycache__/ +.mypy_cache/ +util/n64/Yay0decompress +*.ld +*.n64 +*.yaml +*.z64 diff --git a/tools/splat/.gitrepo b/tools/splat/.gitrepo new file mode 100644 index 0000000..5899cd2 --- /dev/null +++ b/tools/splat/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme +; +[subrepo] + remote = git@github.com:ethteck/splat.git + branch = master + commit = 4fec014c10aa1fc90d200fe6722000425bf7d43f + parent = 6a5e64cc4c26398cc607bb02cdc12a056ddf42d1 + method = merge + cmdver = 0.4.3 diff --git a/tools/splat/CHANGELOG.md b/tools/splat/CHANGELOG.md new file mode 100644 index 0000000..0fad6fb --- /dev/null +++ b/tools/splat/CHANGELOG.md @@ -0,0 +1,361 @@ +# splat Release Notes + +### 0.13.5 + +* An error will be produced if a symbol is declared with an unknown type in the symbol_addrs file. + * The current list of known symbols is `'func', 'label', 'jtbl', 'jtbl_label', 's8', 'u8', 's16', 'u16', 's32', 'u32', 's64', 'u64', 'f32', 'f64', 'Vec3f', 'asciz', 'char*', 'char'`. + * Custom types are allowed if they start with a capital letter. + +### 0.13.4 + +* Renamed `follows_vram_symbol` segment option to `vram_of_symbol` to more accurately reflect what it's used for - to set the segment's vram based on a symbol. +* Refactored the `appears_after_overlays_addr` feature so that expressions are written at the latest possible moment in the linker script. This fixes errors and warnings regarding forward references to later symbols. + +### 0.13.3 + +* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieve this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays. + +### 0.13.2 + +* Actually implemented `ld_use_follows`. Oopz + +### 0.13.1 + +* Added `ld_wildcard_sections` option (disabled by default), which adds a wildcard to the linker script for section linking. This can be helpful for modern GCC, which creates additional rodata sections such as ".rodata.xyz". +* Added `ld_use_follows` option (enabled by default), which, if disabled, makes splat ignore follows_vram and follows_symbols. This helps for fixing matching builds while being able to add infrastructure to the yaml for non-matching builds by just re-enabling the option. + +### 0.13.0 + +* Automatically generate `INCLUDE_RODATA`/`#pragma GLOBAL_ASM` directives for non migrated rodata symbols when creating new C files. +* Non migrated rodata symbols will now only be produced if the C file has a corresponding rodata file with the same name and the C file has a `INCLUDE_RODATA`/`#pragma GLOBAL_ASM` directive referencing the symbol, similar to how functions are disassembled. + * Because of this, the `partial_migration` attribute has lost its purpose and has been removed. +* Rodata symbol files are now included in the autogenerated dependency files too. + +### 0.12.14 + +* New option: `pair_rodata_to_text`. + * If enabled, splat will try to find to which text segment an unpaired rodata segment belongs and it will hint it to the user. + +### 0.12.13 + +* bss segments can now omit the rom offset. + +### 0.12.12 + +* Try to detect and warn to the user if a gap between two migrated rodata symbols is detected and suggest possible solutions to the user. + +### 0.12.11 + +* New disassembly option in the yaml: `allow_data_addends`. + * Allows enabling/disabling using addends on all `.data` symbols. +* Three new options for symbols: `name_end`, `allow_addend` and `dont_allow_addend`. + * `name_end`: allows to provide a closing name for any symbol. Useful for handwritten asm which usually have an "end" name. + * `allow_addend` and `dont_allow_addend`: Allow overriding the global `allow_data_addends` option for allowing addends on data symbols. + +### 0.12.10 + +* Allows passing user-created relocs to the disassembler via the `reloc_addrs.txt` file, allowing to improve the automatic disassembly. +* Multiple reloc_addrs files can be specified in the yaml with the `reloc_addrs_path` option. + +### 0.12.9 + +* Added `format_sym_name()` to the vtx segment so it, too, can be extended + +### 0.12.8 + +* The gfx and vtx segments now have a `data_only` option, which, if enabled, will emit only the plain data for the type and omit the enclosing symbol definition. This mode is useful when you want to manually declare the symbol and then #include the extracted data within the declaration. +* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component. + +### 0.12.7 + +* Allow setting a different macro for jumptable labels with `asm_jtbl_label_macro` + * The currently recommended one is `jlabel` instead of `glabel` +* Two new options for symbols: `force_migration` and `force_not_migration` + * Useful for weird cases where the disassembler decided a rodata symbol must (or must not) be migrated when it really shouldn't (or should) +* Fix `str_encoding` defaulting to `False` instead of `None` +* Output empty rules in generated dependency files to avoid issues when the function file does not exist anymore (i.e. when it gets matched) +* Allow changing the `include_macro_inc` option in the yaml + +### 0.12.6 + +* Adds two new N64-specific segments: + * IPL3: Allows setting its correct VRAM address without messing the global segment detection + * RSP: Allows disassembling using the RSP instruction set instead of the default one +* PS2 was added as a new platform option. + * When this is selected the R5900 instruction set will be used when disassembling instead of the default one. + +### 0.12.5 + +* Update minimal spimdisasm version to 1.7.1. +* Fix spimdisasm>=1.7.0 non being able to see symbols which only are referenced by other data symbols. +* An check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user. + +### 0.12.4 + +* Fixed a bug involving the order of attributes in symbol_addrs preventing proper range searching during calls to `get_symbol` + +### 0.12.3: Initial Gamecube Support +Initial support for Gamecube disk images has been set up! Disassembly is not currently supported, and a more comprehensive explanation of Gamecube support will come once that is finished. + +* The Symbol class is now hashable +* Added the ability for segments to specify a file path (`path`) to receive that file's contents as their split input +* The `generated_s_preamble` option now will be applied to data files created by spimdisasm +* Rewrote symbol range check code to be more efficient +* Fixed bug that allowed empty top-level segments of type `code`. +* Fixed progress bars to properly update their descriptions +* Fixed bug pertaining to symbols getting assigned to segments they shouldn't if their segment is given in symbol_addrs (`segment:`) + +### 0.12.2 +* Fixed bug where `given_dir` was possibly not a `Path` + +### 0.12.1 +* The constructor for `Segment` takes far fewer arguments now, which will affect (and hopefully simplify) any custom segments that are implemented. + +* The new option `string_encoding` can be set at the global or segment level and will influence the encoding for strings in rodata during disassembly. The default encoding used is EUC-JP, as it was previously. + +## 0.12.0: Performance Boost + +In this release, we bring many performance improvements, making splat dramatically faster. We have observed speedups of 10-20x, though your results may vary. + +* Linker script `_romPos` alignment statements now take a form that is friendlier to different assemblers. + +* Fixed the default value of `use_legacy_include_asm` to be what it was before 0.11.2 + +### 0.11.2 +* The way options are parsed and accessed has been completely refactored. The following option names have changed: + +`linker_symbol_header_path` -> `ld_symbol_header_path` + +`asm_endlabels` -> `asm_end_label` + +Additionally, any custom segments or code that needs to read options will have to accommodate the new API for doing so. Options are now fields of an object named `opts` within the existing `options` namespace. Because the options are fields, `get_` is no longer necessary. To give an example: + +Before: `options.get_asm_path()` + +After: `options.opts.asm_path` + +The clean_up_path function in linker_entry.py now uses a cache, offering a small performance improvement during the linker script writing phase. + +### 0.11.1 +* The linker script now includes a `_SIZE` symbol for each segment. +* The new `create_asm_dependencies`, if enabled, will cause splat to create `.asmproc.d` files that can inform a build system which asm files a c file depends upon. If your build system is configured correctly, this can allow triggering a rebuild of a C file when its included asm files are modified. +* Splat no longer depends directly on pypng and now instead uses [n64img](https://github.com/decompals/n64img). Currently, all image behavior uses the exact same code. Eventually, n64img will be implemented in C and support rebuilding images as well. + +## 0.11.0: Spimdisasm Returns + +Spimdisasm now handles data (data, rodata, bss) disassembly in splat! This includes a few changes in behavior: + +* Rodata will be migrated to c files' asm function files when a .rodata subsegment is used that corresponds with an identically-named c file. Some symbols may not be automatically migrated to functions when it is not clear if they belong to the function itself (an example of which being const arrays). In this case, the `partial_migration` option can be enabled for the given .rodata subsegment and splat will create .s files for these unmigrated rodata symbols. These files can then be included in your c files, or you can go ahead and migrate these symbols to c and disable the `partial_migration` feature. + +* BSS can now be disassembled as well, and the size of a code segment's bss section can be specified with the `bss_size` option. This option will tell splat how large the bss section is in bytes so BSS can properly be handled during disassembly. For bss subsegments, the rom address will of course not change, but the vram address should still be specified. This currently can only be done in the dict form of segment representation, rather than the list form. + +Thanks again to [AngheloAlf](https://github.com/AngheloAlf) for adding this functionality and continuing to improve splat's disassembler. + +## 0.10.0: The Linker Script Update + +Linker scripts splat produces are now capable of being shift-friendly. Rom addresses will automatically shift, and ram addresses will still be hard-coded unless the new segment option `follows_vram` is specified. The value of this option should be the name of a segment (a) that this segment (b) should follow in memory. If a grows or shrinks, b's start address will also do so to accommodate it. + +The `enable_ld_alignment_hack` option and corresponding behavior has been removed. This proved to add too much complexity to the linker script generation code and was becoming quite a burden to keep dealing with. Apologies for any inconvenience this may cause. But trust me: in the long run, it's good you won't be depending on that madness. + +### 0.9.5 +* Changes have been made to the linker script such that it is more shiftable. Rather than setting the rom position to hard-coded addresses, it increments the position by the size of the previous segment. Some projects may experience some alignment-related issues after this change. If specified, the new segment option `align: n` will add an `ALIGN(n)` directive for that section's linker segment. + +### 0.9.4 +* A new linker script section is now automatically created when the .bss section begins, using NOLOAD as opposed to the previous hacky rom rewinding we were previously doing. Additionally, `ld_section_labels` now includes `.rodata` by default. + +### 0.9.3 +* Added `add_set_gp_64` option (true by default), which allows controlling whether to add ".set gp=64" to asm/hasm files + +### 0.9.2 +* Added "palette" argument to ci4/ci8 segments so that segments' palettes can be manually specified + +### 0.9.1 +* Fixed a bug in which local labels and jump table labels could replace raw words in data blobs during data disassembly + +## 0.9.0: The Big Update +### Introducing [spimdisasm](https://github.com/Decompollaborate/spimdisasm)! +* Thanks to [AngheloAlf](https://github.com/AngheloAlf), we now have a much better MIPS disassembler in splat! spimdisasm has much better hi/lo matching, much lower ram usage, and plenty of other goodies. + +We plan to roll this out in phases. Currently, it only handles actual code disassembly. Later on, we will probably migrate our current data assembly code to use spimdisasm as well. + +**NOTICE**: This integration has been tested on a variety of games and configurations. However, with any giant change to the platform like this, there are bound to be things we didn't catch. Please be patient with us as we handle these remaining issues. Though from what we've seen already, the slight bugs one may come across are totally worth the much improved disassembly. + +### gfx segment type +* A new `gfx` segment type is available, which creates a c file containing a disassembled display list according to the segment's start and end offsets. Thanks to [Glank](https://github.com/glankk) and [Tharo](https://github.com/thar0/) for their work on [libgfxd](https://github.com/glankk/libgfxd) and [pygfxd](https://github.com/thar0/pygfxd/), respectively, for helping make this a possibility in splat. + +### API breaking changes +* Some `Segment()` arguments have changed, which may cause extensions to break. Please see the `__init__` function for `Segment` for more details. + +### symbol_addrs.txt changes +* symbol_addrs now supports the `segment:` attribute, which allows specifying the symbol's top-level segment. This can be helpful for symbol resolution when overlays use overlapping vram ranges. See `exclusive_ram_id` below for more information. + +### Global options changes + +The new `symbol_name_format` option allows specification of how symbols will be named. This can be set as a global option and also changed per-segment. `symbol_name_format_no_rom` is used when the symbol does not have a rom address (BSS). + + The following substitutions are allowed: + +`$ROM` - the rom address of the symbol, hex-formatted and padded to 6 characters (ABCF10, 000030, 123456) (note: only for `symbol_name_format`, usage in `symbol_name_format_no_rom` will cause an error) + +`$VRAM` - the vram address of the symbol, hex-formatted and padded to 8 characters (00030010, 00020015, ABCDEF10) + +`$SEG` - the name of the top-level segment in which the symbol resides + +The default values for these options are as follows + +`symbol_name_format` : `$VRAM` + +`symbol_name_format_no_rom` : `$VRAM_$SEG` + +The appropriate prefix string will still automatically be applied depending on the type of the symbol: `D_` for data, `jtbl_` for jump tables, and `func_` for functions. This functionality may be customizable in the future. + +---- +The `auto_all_sections` option now should be a list of section names (`[".data", ".rodata", ".bss"]` by default) indicating the sections that should be linked from .o files built from source files (.c or asm/hasm .s files), when no subsegment explicitly indicates linking this type of section. + +For example, if any subsegment of a code segment is of segment type `data` or `.data`, the `.data` section from all `c`/`asm`/`hasm` subsegments will not be linked unless explicitly indicated with a relevant `.data` subsegment. + +Previously, this option was a bool, and it enabled this feature for all sections specified in `section_order`. Now, the desired sections must be specified manually. The default value for this option retains previous behavior. + +---- +The new `mips_abi_float_regs` option allows for changing the format of float registers for MIPS disassembly. The default value does not change any prior behavior, but `o32` is heavily encouraged and may become the default option in the future. For more information, see this [great writeup](https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d). + +---- +The new `gfx_ucode` option allows for specifying the target for the graphics macro format, which is used in the gfx segment type. The default is `f3dex2`. + + +### Segment options changes + +The new `exclusive_ram_id` segment option allows specifying an identifer that will prevent the segment from seeing any symbols from other segments with the same identifer. This is useful when multiple segments are mapped to the same vram address at runtime and should never be able to refer to each other's symbols. Setting all of these segments to have the same value for this option will prevent their symbols from clashing / meshing unexpectedly. + +---- + +The `overlay` setting on segments has been removed. Please see `symbol_name_format` above for info on how to influence the names of symbols, which can be applied at the segment level as well as the global level. + +---- +## 0.8.0: Arbitrary Section Order +* You can now use the option `section_order` to define the binary section order for your target binary. By default, this is `[".text", ".data", ".rodata", ".bss"]`. See options.py for more details +* Documented all options in options.py +* Support for SN64 games (thanks Wiseguy!) +* More consistent handling of paths (thanks Mkst!) +* Various other cleanup and fixes across the board + +### 0.7.10: WIP PSX support +* WIP PSX support has been added, thanks to @mkst! (https://github.com/ethteck/splat/pull/99) + * Many segments have moved to a "common" package + * Endianness of the input binary is now a configurable option +* Linker hack restored but is now optional and off by default + +### 0.7.9 +* Finally removed the dumb linker section alignment hack +* Added version number to output on execution + +### 0.7.8 + +* Fixed a bug relating to a linker section alignment hack (thanks Wiseguy!) +* Fixed a bug in linker_entry.py's clean_up_path that should make this function more versatile (thanks Wiseguy!) + +### 0.7.7 + +* Disassembly now reads the `size` property of a function in symbol_addrs.txt to disassemble `size / 4` number of instructions. Feel free to specify the size of your functions in your symbol_addrs file if splat's disassembly is chopping a function too short or making a function too long. + +### 0.7.6 + +* Fixed a bug involving detection of defined functions in c files for GLOBAL_ASM-using projects +* Added options to disable the creation of undefined_funcs/syms_auto.txt files +* Added a Vtx segment type for creating c files containg model vertex data in the n64 libultra Vtx format +* Added a `cpp` segment type which is identical to `c` but looks for a file with the extension ".cpp" instead of ".c". + +### 0.7.5: all_ types and auto_all_sections + +If you have a group segment with multiple c files and want splat to automatically create linker entries at a given position for each code object (c, asm, hasm) in the segment, you can use an `all_` type for that section. For example, you can add `[auto, all_bss]` as the last subsegment in a segment. This will direct splat to create a linker entry for each code object in the segment. This saves a lot of time when it comes to manually adding .bss subsegments for bss support, for example. The same thing can be done for data and rodata sections, but note this should probably be done later into a project when all data / rodata is migrated to c files, as the `all_` types lose the rom positioning information that's necessary for splat to do proper disassembly. + +The `auto_all_sections` option, when set to true, will automatically add `all_` types into every group. This is only done for a section in a group if no other manual declarations for that section exist. For example, if you have 30 c files in a group and a .data later on for one of them, `auto_all_sections` will not interfere with your `.data` subsegment. If you remove this, however, splat will use `auto_all_sections` to implicitly `.data` subsegments for all of your code objects behind the scenes. This feature is again particualrly helpful for bss support, as it will create bss linker entries for every file in your project (assuming you don't have any manual .bss subsegments), which eliminates the need to create dummy .bss subsegments just for the sake of configuring the linker script. + +### 0.7.2 + +* Data disassembly changes: + * String detection has been improved. Please send me false positives / negatives as you see them and I can try to improve it further! + * Symbols in a data segment pointed to by other symbols will now properly be split out as their own symbols + +### 0.7.1 + +* Image segment changes: + * Added `flip_x` and `flip_y` boolean parameters to replace `flip`. + * `flip` is deprecated and will produce a warning when used. + * Fixed flipping of `ci4` and `ci8` images. + * Fixed `extract: false` (and `start: auto`) behaviour. + +## 0.7.0: The Path Update + +* Significantly better performance, especially when using the cache feature (`--use-cache` CLI arg). +* BREAKING: Some cli args for splat have been renamed. Please consult the usage output (-h or no args) for more information. + * `--new` has been renamed to `--use-cache` + * `--modes` arg changes: + * Image modes have been combined into the `img` mode + * Code and ASM modes have been combined into the `code` mode +* BREAKING: The `name` attribute of a segment now should no longer be a subdirectory but rather a meaningful name for the segment which will be used as the name of the linker section. If your `name` was previously a directory, please change it into a `dir`. +* BREAKING: `subsections` has been renamed to `subsegments` +* New `dir` segment attribute specifies a subdirectory into which files will be saved. You can combine `dir` ("foo") with a subsection file name containing a subdirectory ("bar/out"), and the paths will be joined (foo/bar/out.c) + * If the `dir` attribute is specified but the `name` isn't, the `name` becomes `dir` with directory separation slashes replaced with underscores (foo/bar/baz -> foo_bar_baz) +* BREAKING: Many configuration options have been renamed. `_dir` options have been changed to the suffix `_path`. +* BREAKING: Assets (non-code, like `bin` and images) are now placed in the directory `asset_path` (defaults to `assets`). +* Linker symbol header generation. Set the `linker_symbol_header_path` option to use. + * `typedef u8[] Addr;` is recommended in your `common.h` header. +* You can now provide `auto` as the `start` attribute for a segment, e.g. `[auto, c, my_file]`. This causes the segment to not be extracted, but linked. This feature is intended for modding. +* Providing just a ROM address but no type or name for a segment is now valid anywhere in `segments` or `subsegments` rather than just at the end of the ROM. It specifies the end of the previous segment for types that need it (`palette`, `bin`, `Yay0`) and causes the linker to simply write padding until that address. +* The linker script file is left untouched if the contents have not changed since the previous split. +* You can now group together segments with `type: group` (similar to `code`). Note that any ASM or C segments must live under a `type: code` segment, not a basic `group`. + +### 0.6.5: Bugfixes, rodata migration, and made options static + +If you wrote a custom extension, options should be imported and statically referenced +`from util import options` + +see options.py for more info on how to now get and set options + +BREAKING: vram can only be specified on a segment if the segment is defined as a dict in the config + +### 0.6.3: More refactoring +**Breaking Change**: The command line args to split.py have changed. Currently, only the config path is now a required argument to splat. The old `rom` and `outdir` parameters are now optional (`--rom`, `--outdir`). Now, you can add rom and out directory paths in the yaml. + +The `out_dir` option specifies a directory relative to the config file. If your config file is in a subdirectory of the main repo, you can set `out_dir: ../`, for example. + +The `target_path` option spcifies a path to the binary file to split, relative to the `out_dir`. If your `baserom.z64` is in the top-level of the repo, you can set `target_path: baserom.z64`, for example. + +### 0.6.2: subsegments +I've begun a refactor of the code "files" code, which makes everything cleaner and easier to extend. +There's also a new option, `create_new_c_files`, which disables the creation of nonexistent c files. This behavior is on by default, but if you want to disable it for any reason, you now have the option to do so. + +I am also working on adding bss support as well. It should almost be all set, aside from the changes needed in the linker script. + +**Breaking change**: The `files` field in `code` segments should now be renamed to `subsegments`. + +### 0.6.1: `assets_dir` option + +This release adds a new `assets_dir` option in `splat.yaml`s that allows you to override the default `img`, `bin`, and other directories that segments output to. + +Want to interdisperse split assets with your sourcecode? `assets_dir: src`! +Want to have all assets live in a single directory? `assets_dir: assets`! + +## 0.6: The Symbol Update +Internally, there's a new Symbol class which stores information about a symbol and is stored in a couple places during disassembly. Many things should be improved, such as reconciling symbols within overlays, things being named functions vs data symbols, and more. + +**Breaking change**: The format to symbol_addrs.txt has been updated. After specifying the name and address of a symbol (`symbol = addr;`), optional properties of symbols can be set via inline comment, space delimited, in any order. The properties are of the format `name:value` + * `type:` supports `func` mostly right now but will support `label` and `data` later on. Internally, `jtbl` is used as well, for jump tables. Splat uses type information during disassembly to disambiguate symbols with the same addresses. + * `rom:` is for the hex rom address of the symbol, beginning with `0x`. If available, this information is extremely valuable for use in disambiguating symbols. + * `size:` specifies the size of the symbol, which splat will use to generate offsets during disassembly. Uses the same format as `rom:` + +**function example**: `FuncNameHere = 0x80023423; // type:func rom:0x10023` + +**data example**: `gSomeDataVar = 0x80024233; // type:data size:0x100` + +## 0.5 The Rename Update +* n64splat name changed to splat + * Some refactoring was done to support other platforms besides n64 in the future + * New `platform` option, which defaults to `n64` + * This will cause breaking changes in custom segments, so please refer to one of the changes in one of the n64 base segments for details +* Support for custom artifact paths + * New `undefined_syms_auto_path` option + * New `undefined_funcs_auto_path` option + * New `cache_path` option + * (All path-like options' names now end with `_path`) diff --git a/tools/splat/LICENSE b/tools/splat/LICENSE new file mode 100644 index 0000000..42621fb --- /dev/null +++ b/tools/splat/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Ethan Roseman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/splat/Makefile b/tools/splat/Makefile new file mode 100644 index 0000000..b72aafe --- /dev/null +++ b/tools/splat/Makefile @@ -0,0 +1,11 @@ +UTIL_DIR := util + +default: all + +all: Yay0decompress + +Yay0decompress: + gcc $(UTIL_DIR)/n64/Yay0decompress.c -fPIC -shared -O3 -Wall -Wextra -o $(UTIL_DIR)/n64/Yay0decompress + +clean: + rm -f $(UTIL_DIR)/n64/Yay0decompress diff --git a/tools/splat/README.md b/tools/splat/README.md new file mode 100644 index 0000000..de7abc7 --- /dev/null +++ b/tools/splat/README.md @@ -0,0 +1,9 @@ +# splat +A binary splitting tool to assist with decompilation and modding projects + +Currently, only N64 and PSX binaries are supported. + +Please check out the [wiki](https://github.com/ethteck/splat/wiki) for more information including [examples](https://github.com/ethteck/splat/wiki/Examples) of projects that use splat. + +### Requirements +splat requires Python 3.8+. Package requirements can be installed via `pip3 install -U -r requirements.txt` diff --git a/tools/splat/create_config.py b/tools/splat/create_config.py new file mode 100755 index 0000000..cf34481 --- /dev/null +++ b/tools/splat/create_config.py @@ -0,0 +1,192 @@ +#! /usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +from segtypes.gc.rarc import GcSegRarc +from util.gc import gcinfo + +from util.n64 import find_code_length, rominfo + +parser = argparse.ArgumentParser( + description="Create a splat config from an N64 ROM or a GameCube disc image." +) +parser.add_argument("file", help="Path to a .z64/.n64 ROM or .iso/.gcm GameCube image") + + +def main(file_path: Path): + if not file_path.exists(): + sys.exit(f"File {file_path} does not exist ({file_path.absolute()})") + if file_path.is_dir(): + sys.exit(f"Path {file_path} is a directory ({file_path.absolute()})") + + # Check for N64 ROM + if file_path.suffix.lower() == ".n64" or file_path.suffix.lower() == ".z64": + create_n64_config(file_path) + return + + file_bytes = file_path.read_bytes() + + # Check for GC disc image + if int.from_bytes(file_bytes[0x1C:0x20], byteorder="big") == 0xC2339F3D: + create_gc_config(file_path, file_bytes) + + +def create_n64_config(rom_path: Path): + rom_bytes = rominfo.read_rom(rom_path) + + rom = rominfo.get_info(rom_path, rom_bytes) + basename = rom.name.replace(" ", "").lower() + + header = f"""\ +name: {rom.name.title()} ({rom.get_country_name()}) +sha1: {rom.sha1} +options: + basename: {basename} + target_path: {rom_path.with_suffix(".z64")} + base_path: . + compiler: {rom.compiler} + find_file_boundaries: True + header_encoding: {rom.header_encoding} + platform: n64 + # undefined_funcs_auto: True + # undefined_funcs_auto_path: undefined_funcs_auto.txt + # undefined_syms_auto: True + # undefined_syms_auto_path: undefined_syms_auto.txt + # symbol_addrs_path: symbol_addrs.txt + # asm_path: asm + # src_path: src + # build_path: build + # extensions_path: tools/splat_ext + # mips_abi_float_regs: o32 + # section_order: [".text", ".data", ".rodata", ".bss"] + # auto_all_sections: [".data", ".rodata", ".bss"] + # libultra_symbols: True + # hardware_regs: True +""" + + first_section_end = find_code_length.run(rom_bytes, 0x1000, rom.entry_point) + + segments = f"""\ +segments: + - name: header + type: header + start: 0x0 + + - name: boot + type: bin + start: 0x40 + + - name: entry + type: code + start: 0x1000 + vram: 0x{rom.entry_point:X} + subsegments: + - [0x1000, hasm] + + - name: main + type: code + start: 0x{0x1000 + rom.entrypoint_info.entry_size:X} + vram: 0x{rom.entry_point + rom.entrypoint_info.entry_size:X} + follows_vram: entry +""" + + if rom.entrypoint_info.bss_size is not None: + segments += f"""\ + bss_size: 0x{rom.entrypoint_info.bss_size:X} +""" + + segments += f"""\ + subsegments: + - [0x{0x1000 + rom.entrypoint_info.entry_size:X}, asm] +""" + + if ( + rom.entrypoint_info.bss_size is not None + and rom.entrypoint_info.bss_start_address is not None + ): + bss_start = rom.entrypoint_info.bss_start_address - rom.entry_point + 0x1000 + # first_section_end points to the start of data + segments += f"""\ + - [0x{first_section_end:X}, data] + - {{ start: 0x{bss_start:X}, type: bss, vram: 0x{rom.entrypoint_info.bss_start_address:08X} }} +""" + # Point next segment to the detected end of the main one + first_section_end = bss_start + + segments += f"""\ + + - type: bin + start: 0x{first_section_end:X} + follows_vram: main + - [0x{rom.size:X}] +""" + + out_file = f"{basename}.yaml" + with open(out_file, "w", newline="\n") as f: + print(f"Writing config to {out_file}") + f.write(header) + f.write(segments) + + +def create_gc_config(iso_path: Path, iso_bytes: bytes): + gc = gcinfo.get_info(iso_path, iso_bytes) + basename = gc.system_code + gc.game_code + gc.region_code + gc.publisher_code + + header = f"""\ +name: \"{gc.name.title()} ({gc.get_region_name()})\" +system_code: {gc.system_code} +game_code: {gc.game_code} +region_code: {gc.region_code} +publisher_code: {gc.publisher_code} +sha1: {gc.sha1} +options: + filesystem_path: filesystem + basename: {basename} + target_path: {iso_path.with_suffix(".iso")} + base_path: . + compiler: {gc.compiler} + platform: gc + # undefined_funcs_auto: True + # undefined_funcs_auto_path: undefined_funcs_auto.txt + # undefined_syms_auto: True + # undefined_syms_auto_path: undefined_syms_auto.txt + # symbol_addrs_path: symbol_addrs.txt + # asm_path: asm + # src_path: src + # build_path: build + # extensions_path: tools/splat_ext + # section_order: [".text", ".data", ".rodata", ".bss"] + # auto_all_sections: [".data", ".rodata", ".bss"] +""" + + segments = f"""\ +segments: + - name: filesystem + type: fst + path: filesystem/sys/fst.bin + - name: bootinfo + type: bootinfo + path: filesystem/sys/boot.bin + - name: bi2 + type: bi2 + path: filesystem/sys/bi2.bin + - name: apploader + type: apploader + path: filesystem/sys/apploader.img + - name: main + type: dol + path: filesystem/sys/main.dol +""" + + out_file = f"{basename}.yaml" + with open(out_file, "w", newline="\n") as f: + print(f"Writing config to {out_file}") + f.write(header) + f.write(segments) + + +if __name__ == "__main__": + args = parser.parse_args() + main(Path(args.file)) diff --git a/tools/splat/mypy.ini b/tools/splat/mypy.ini new file mode 100644 index 0000000..6e7c2ef --- /dev/null +++ b/tools/splat/mypy.ini @@ -0,0 +1,4 @@ +[mypy] +ignore_missing_imports = True +check_untyped_defs = True +mypy_path = stubs diff --git a/tools/splat/platforms/gc.py b/tools/splat/platforms/gc.py new file mode 100644 index 0000000..af08a28 --- /dev/null +++ b/tools/splat/platforms/gc.py @@ -0,0 +1,5 @@ +from util.gc import gcfst + + +def init(target_bytes: bytes): + gcfst.split_iso(target_bytes) diff --git a/tools/splat/platforms/n64.py b/tools/splat/platforms/n64.py new file mode 100644 index 0000000..ed3ee1c --- /dev/null +++ b/tools/splat/platforms/n64.py @@ -0,0 +1,10 @@ +from util import compiler, log, options, palettes, symbols + + +def init(target_bytes: bytes): + symbols.spim_context.fillDefaultBannedSymbols() + + if options.opts.libultra_symbols: + symbols.spim_context.globalSegment.fillLibultraSymbols() + if options.opts.hardware_regs: + symbols.spim_context.globalSegment.fillHardwareRegs(True) diff --git a/tools/splat/platforms/psx.py b/tools/splat/platforms/psx.py new file mode 100644 index 0000000..09da192 --- /dev/null +++ b/tools/splat/platforms/psx.py @@ -0,0 +1,2 @@ +def init(target_bytes: bytes): + pass diff --git a/tools/splat/requirements.txt b/tools/splat/requirements.txt new file mode 100644 index 0000000..711d553 --- /dev/null +++ b/tools/splat/requirements.txt @@ -0,0 +1,10 @@ +PyYAML +pylibyaml +tqdm +intervaltree +colorama +# This value should be keep in sync with the version listed on split.py +spimdisasm>=1.12.0 +rabbitizer>=1.4.0 +pygfxd +n64img>=0.1.4 diff --git a/tools/splat/segtypes/__init__.py b/tools/splat/segtypes/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/splat/segtypes/__init__.py diff --git a/tools/splat/segtypes/address_range.py b/tools/splat/segtypes/address_range.py new file mode 100644 index 0000000..0c57cc0 --- /dev/null +++ b/tools/splat/segtypes/address_range.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +@dataclass +class AddressRange: + start: int + end: int + + def contains(self, addr: int) -> bool: + return self.start <= addr < self.end diff --git a/tools/splat/segtypes/common/__init__.py b/tools/splat/segtypes/common/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/splat/segtypes/common/__init__.py diff --git a/tools/splat/segtypes/common/asm.py b/tools/splat/segtypes/common/asm.py new file mode 100644 index 0000000..db7a184 --- /dev/null +++ b/tools/splat/segtypes/common/asm.py @@ -0,0 +1,39 @@ +from pathlib import Path +from typing import Optional + +from util import options + +from segtypes.common.codesubsegment import CommonSegCodeSubsegment + + +class CommonSegAsm(CommonSegCodeSubsegment): + @staticmethod + def is_text() -> bool: + return True + + def out_path(self) -> Optional[Path]: + return options.opts.asm_path / self.dir / f"{self.name}.s" + + def scan(self, rom_bytes: bytes): + if ( + self.rom_start is not None + and self.rom_end is not None + and self.rom_start != self.rom_end + ): + self.scan_code(rom_bytes) + + def get_file_header(self): + return [] + + def split(self, rom_bytes: bytes): + if not self.rom_start == self.rom_end and self.spim_section is not None: + out_path = self.out_path() + if out_path: + out_path.parent.mkdir(parents=True, exist_ok=True) + + self.print_file_boundaries() + + with open(out_path, "w", newline="\n") as f: + for line in self.get_file_header(): + f.write(line + "\n") + f.write(self.spim_section.disassemble()) diff --git a/tools/splat/segtypes/common/bin.py b/tools/splat/segtypes/common/bin.py new file mode 100644 index 0000000..b72a201 --- /dev/null +++ b/tools/splat/segtypes/common/bin.py @@ -0,0 +1,28 @@ +from pathlib import Path +from typing import Optional + +from util import log, options + +from segtypes.common.segment import CommonSegment + + +class CommonSegBin(CommonSegment): + def out_path(self) -> Optional[Path]: + return options.opts.asset_path / self.dir / f"{self.name}.bin" + + def split(self, rom_bytes): + path = self.out_path() + assert path is not None + path.parent.mkdir(parents=True, exist_ok=True) + + if self.rom_end is None: + log.error( + f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" + ) + + with open(path, "wb") as f: + assert isinstance(self.rom_start, int) + assert isinstance(self.rom_end, int) + + f.write(rom_bytes[self.rom_start : self.rom_end]) + self.log(f"Wrote {self.name} to {path}") diff --git a/tools/splat/segtypes/common/bss.py b/tools/splat/segtypes/common/bss.py new file mode 100644 index 0000000..ef79311 --- /dev/null +++ b/tools/splat/segtypes/common/bss.py @@ -0,0 +1,60 @@ +import spimdisasm +from util import options, symbols, log + +from segtypes.common.data import CommonSegData + + +class CommonSegBss(CommonSegData): + def get_linker_section(self) -> str: + return ".bss" + + @staticmethod + def is_noload() -> bool: + return True + + def disassemble_data(self, rom_bytes: bytes): + if not isinstance(self.rom_start, int): + log.error( + f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" + ) + + # Supposedly logic error, not user error + assert isinstance(self.rom_end, int), f"{self.name} {self.rom_end}" + + # Supposedly logic error, not user error + segment_rom_start = self.get_most_parent().rom_start + assert isinstance(segment_rom_start, int), f"{self.name} {segment_rom_start}" + + if not isinstance(self.vram_start, int): + log.error( + f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" + ) + + next_subsegment = self.parent.get_next_subsegment_for_ram(self.vram_start) + if next_subsegment is None: + bss_end = self.get_most_parent().vram_end + else: + bss_end = next_subsegment.vram_start + assert isinstance(bss_end, int), f"{self.name} {bss_end}" + + self.spim_section = spimdisasm.mips.sections.SectionBss( + symbols.spim_context, + self.rom_start, + self.rom_end, + self.vram_start, + bss_end, + self.name, + segment_rom_start, + self.get_exclusive_ram_id(), + ) + + self.spim_section.analyze() + self.spim_section.setCommentOffset(self.rom_start) + + for spim_sym in self.spim_section.symbolList: + symbols.create_symbol_from_spim_symbol( + self.get_most_parent(), spim_sym.contextSym + ) + + def should_scan(self) -> bool: + return options.opts.is_mode_active("code") and self.vram_start is not None diff --git a/tools/splat/segtypes/common/c.py b/tools/splat/segtypes/common/c.py new file mode 100644 index 0000000..76d6981 --- /dev/null +++ b/tools/splat/segtypes/common/c.py @@ -0,0 +1,428 @@ +import os +import re +from pathlib import Path +from typing import Optional, Set, List, Tuple + +import spimdisasm + +from util import log, options, symbols +from util.compiler import GCC, SN64, IDO +from util.symbols import Symbol + +from segtypes.common.codesubsegment import CommonSegCodeSubsegment +from segtypes.common.group import CommonSegGroup +from segtypes.common.rodata import CommonSegRodata + + +class CommonSegC(CommonSegCodeSubsegment): + defined_funcs: Set[str] = set() + global_asm_funcs: Set[str] = set() + global_asm_rodata_syms: Set[str] = set() + + STRIP_C_COMMENTS_RE = re.compile( + r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', + re.DOTALL | re.MULTILINE, + ) + + C_FUNC_RE = re.compile( + r"^(?:static\s+)?[^\s]+\s+([^\s(]+)\(([^;)]*)\)[^;]+?{", re.MULTILINE + ) + + C_GLOBAL_ASM_IDO_RE = re.compile( + r"GLOBAL_ASM\(\"(\w+\/)*(\w+)\.s\"\)", re.MULTILINE + ) + + @staticmethod + def strip_c_comments(text): + def replacer(match): + s = match.group(0) + if s.startswith("/"): + return " " + else: + return s + + return re.sub(CommonSegC.STRIP_C_COMMENTS_RE, replacer, text) + + @staticmethod + def get_funcs_defined_in_c(c_file: Path) -> Set[str]: + with open(c_file, "r") as f: + text = CommonSegC.strip_c_comments(f.read()) + + return set(m.group(1) for m in CommonSegC.C_FUNC_RE.finditer(text)) + + @staticmethod + def find_all_instances(string: str, sub: str): + start = 0 + while True: + start = string.find(sub, start) + if start == -1: + return + yield start + start += len(sub) + + @staticmethod + def get_close_parenthesis(string: str, pos: int): + paren_count = 0 + while True: + cur_char = string[pos] + if cur_char == "(": + paren_count += 1 + elif cur_char == ")": + if paren_count == 0: + return pos + 1 + else: + paren_count -= 1 + pos += 1 + + @staticmethod + def find_include_macro(text: str, macro_name: str): + for pos in CommonSegC.find_all_instances(text, f"{macro_name}("): + close_paren_pos = CommonSegC.get_close_parenthesis( + text, pos + len(f"{macro_name}(") + ) + macro_contents = text[pos:close_paren_pos] + macro_args = macro_contents.split(",") + if options.opts.use_legacy_include_asm: + if len(macro_args) >= 3: + yield macro_args[2].strip(" )") + else: + if len(macro_args) >= 2: + yield macro_args[1].strip(" )") + + @staticmethod + def find_include_asm(text: str): + return CommonSegC.find_include_macro(text, "INCLUDE_ASM") + + @staticmethod + def find_include_rodata(text: str): + return CommonSegC.find_include_macro(text, "INCLUDE_RODATA") + + @staticmethod + def get_global_asm_funcs(c_file: Path) -> Set[str]: + with c_file.open() as f: + text = CommonSegC.strip_c_comments(f.read()) + if options.opts.compiler in [GCC, SN64]: + return set(CommonSegC.find_include_asm(text)) + else: + return set( + m.group(2) for m in CommonSegC.C_GLOBAL_ASM_IDO_RE.finditer(text) + ) + + @staticmethod + def get_global_asm_rodata_syms(c_file: Path) -> Set[str]: + with c_file.open() as f: + text = CommonSegC.strip_c_comments(f.read()) + if options.opts.compiler in [GCC, SN64]: + return set(CommonSegC.find_include_rodata(text)) + else: + return set( + m.group(2) for m in CommonSegC.C_GLOBAL_ASM_IDO_RE.finditer(text) + ) + + @staticmethod + def is_text() -> bool: + return True + + def out_path(self) -> Optional[Path]: + return options.opts.src_path / self.dir / f"{self.name}.c" + + def scan(self, rom_bytes: bytes): + if ( + self.rom_start is not None + and self.rom_end is not None + and self.rom_start != self.rom_end + ): + path = self.out_path() + if path: + if options.opts.do_c_func_detection and os.path.exists(path): + # TODO run cpp? + self.defined_funcs = self.get_funcs_defined_in_c(path) + self.global_asm_funcs = self.get_global_asm_funcs(path) + self.global_asm_rodata_syms = self.get_global_asm_rodata_syms(path) + symbols.to_mark_as_defined.update(self.defined_funcs) + symbols.to_mark_as_defined.update(self.global_asm_funcs) + symbols.to_mark_as_defined.update(self.global_asm_rodata_syms) + + self.scan_code(rom_bytes) + + def split(self, rom_bytes: bytes): + if self.rom_start != self.rom_end: + asm_out_dir = options.opts.nonmatchings_path / self.dir + asm_out_dir.mkdir(parents=True, exist_ok=True) + + self.print_file_boundaries() + + assert self.spim_section is not None and isinstance( + self.spim_section, spimdisasm.mips.sections.SectionText + ), f"{self.name}, rom_start:{self.rom_start}, rom_end:{self.rom_end}" + + rodata_spim_segment = None + if ( + options.opts.migrate_rodata_to_functions + and self.rodata_sibling is not None + ): + assert isinstance( + self.rodata_sibling, CommonSegRodata + ), self.rodata_sibling.type + if self.rodata_sibling.spim_section is not None: + assert isinstance( + self.rodata_sibling.spim_section, + spimdisasm.mips.sections.SectionRodata, + ) + rodata_spim_segment = self.rodata_sibling.spim_section + + # Precompute function-rodata pairings + symbols_entries = ( + spimdisasm.mips.FunctionRodataEntry.getAllEntriesFromSections( + self.spim_section, rodata_spim_segment + ) + ) + + is_new_c_file = False + + # Check and create the C file + c_path = self.out_path() + if c_path: + if not c_path.exists() and options.opts.create_c_files: + self.create_c_file(asm_out_dir, c_path, symbols_entries) + is_new_c_file = True + + self.create_asm_dependencies_file( + c_path, asm_out_dir, is_new_c_file, symbols_entries + ) + + # Produce the asm files for functions + for entry in symbols_entries: + if entry.function is not None: + if ( + entry.function.getName() in self.global_asm_funcs + or is_new_c_file + ): + func_sym = self.get_symbol( + entry.function.vram, + in_segment=True, + type="func", + local_only=True, + ) + assert func_sym is not None + + self.create_c_asm_file(entry, asm_out_dir, func_sym) + else: + for spim_rodata_sym in entry.rodataSyms: + if ( + spim_rodata_sym.getName() in self.global_asm_rodata_syms + or is_new_c_file + ): + rodata_sym = self.get_symbol( + spim_rodata_sym.vram, in_segment=True, local_only=True + ) + assert rodata_sym is not None + + self.create_unmigrated_rodata_file( + spim_rodata_sym, asm_out_dir, rodata_sym + ) + + def get_c_preamble(self): + ret = [] + + preamble = options.opts.generated_c_preamble + ret.append(preamble) + ret.append("") + + return ret + + def check_gaps_in_migrated_rodata( + self, + func: spimdisasm.mips.symbols.SymbolFunction, + rodata_list: List[spimdisasm.mips.symbols.SymbolBase], + ): + for index in range(len(rodata_list) - 1): + rodata_sym = rodata_list[index] + next_rodata_sym = rodata_list[index + 1] + + if rodata_sym.vramEnd != next_rodata_sym.vram: + log.write( + f"\nA gap was detected in migrated rodata symbols!", status="warn" + ) + log.write( + f"\t In function '{func.getName()}' (0x{func.vram:08X}), gap detected between '{rodata_sym.getName()}' (0x{rodata_sym.vram:08X}) and '{next_rodata_sym.getName()}' (0x{next_rodata_sym.vram:08X})" + ) + log.write( + f"\t The address of the missing rodata symbol is 0x{rodata_sym.vramEnd:08X}" + ) + log.write( + f"\t Try to force the migration of that symbol with `force_migration:True` in the symbol_addrs.txt file; or avoid the migration of symbols around this address with `force:not_migration:True`" + ) + + def create_c_asm_file( + self, + func_rodata_entry: spimdisasm.mips.FunctionRodataEntry, + out_dir: Path, + func_sym: Symbol, + ): + outpath = out_dir / self.name / (func_sym.name + ".s") + + # Skip extraction if the file exists and the symbol is marked as extract=false + if outpath.exists() and not func_sym.extract: + return + + outpath.parent.mkdir(parents=True, exist_ok=True) + + with outpath.open("w", newline="\n") as f: + if options.opts.asm_inc_header: + f.write( + options.opts.c_newline.join(options.opts.asm_inc_header.split("\n")) + ) + + func_rodata_entry.writeToFile(f) + + if func_rodata_entry.function is not None: + self.check_gaps_in_migrated_rodata( + func_rodata_entry.function, func_rodata_entry.rodataSyms + ) + self.check_gaps_in_migrated_rodata( + func_rodata_entry.function, func_rodata_entry.lateRodataSyms + ) + + self.log(f"Disassembled {func_sym.name} to {outpath}") + + def create_unmigrated_rodata_file( + self, + spim_rodata_sym: spimdisasm.mips.symbols.SymbolBase, + out_dir: Path, + rodata_sym: Symbol, + ): + outpath = out_dir / self.name / (rodata_sym.name + ".s") + + # Skip extraction if the file exists and the symbol is marked as extract=false + if outpath.exists() and not rodata_sym.extract: + return + + outpath.parent.mkdir(parents=True, exist_ok=True) + + with outpath.open("w", newline="\n") as f: + if options.opts.include_macro_inc: + f.write('.include "macro.inc"\n\n') + preamble = options.opts.generated_s_preamble + if preamble: + f.write(preamble + "\n") + assert rodata_sym.linker_section is not None, rodata_sym.name + f.write(f".section {rodata_sym.linker_section}\n\n") + f.write(spim_rodata_sym.disassemble()) + + self.log(f"Disassembled {rodata_sym.name} to {outpath}") + + def get_c_line_include_macro( + self, + spim_sym: spimdisasm.mips.symbols.SymbolBase, + asm_out_dir: Path, + macro_name: str, + ) -> str: + if options.opts.compiler == IDO: + # IDO uses the asm processor to embeed assembly and it doesn't require a special directive to include symbols + asm_outpath = Path( + os.path.join(asm_out_dir, self.name, spim_sym.getName() + ".s") + ) + rel_asm_outpath = os.path.relpath(asm_outpath, options.opts.base_path) + return f'#pragma GLOBAL_ASM("{rel_asm_outpath}")' + + if options.opts.use_legacy_include_asm: + rel_asm_out_dir = asm_out_dir.relative_to(options.opts.nonmatchings_path) + return f'{macro_name}(const s32, "{rel_asm_out_dir / self.name}", {spim_sym.getName()});' + + return f'{macro_name}("{asm_out_dir / self.name}", {spim_sym.getName()});' + + def get_c_lines_for_function( + self, func: spimdisasm.mips.symbols.SymbolFunction, asm_out_dir: Path + ) -> List[str]: + c_lines = [] + + # Terrible hack to "auto-decompile" empty functions + if ( + options.opts.auto_decompile_empty_functions + and func.instructions[0].isReturn() + and func.instructions[1].isNop() + ): + c_lines.append("void " + func.getName() + "(void) {") + c_lines.append("}") + else: + c_lines.append( + self.get_c_line_include_macro(func, asm_out_dir, "INCLUDE_ASM") + ) + c_lines.append("") + return c_lines + + def get_c_lines_for_rodata_sym( + self, rodata_sym: spimdisasm.mips.symbols.SymbolBase, asm_out_dir: Path + ): + c_lines = [ + self.get_c_line_include_macro(rodata_sym, asm_out_dir, "INCLUDE_RODATA") + ] + c_lines.append("") + return c_lines + + def create_c_file( + self, + asm_out_dir: Path, + c_path: Path, + symbols_entries: List[spimdisasm.mips.FunctionRodataEntry], + ): + c_lines = self.get_c_preamble() + + for entry in symbols_entries: + if entry.function is not None: + c_lines += self.get_c_lines_for_function(entry.function, asm_out_dir) + else: + for rodata_sym in entry.rodataSyms: + c_lines += self.get_c_lines_for_rodata_sym(rodata_sym, asm_out_dir) + + c_path.parent.mkdir(parents=True, exist_ok=True) + with c_path.open("w") as f: + f.write("\n".join(c_lines)) + log.write(f"Wrote {self.name} to {c_path}") + + def create_asm_dependencies_file( + self, + c_path: Path, + asm_out_dir: Path, + is_new_c_file: bool, + symbols_entries: List[spimdisasm.mips.FunctionRodataEntry], + ): + if not options.opts.create_asm_dependencies: + return + if ( + len(self.global_asm_funcs) + len(self.global_asm_rodata_syms) + ) == 0 and not is_new_c_file: + return + + assert self.spim_section is not None + + build_path = options.opts.build_path + + dep_path = build_path / c_path.with_suffix(".asmproc.d") + dep_path.parent.mkdir(parents=True, exist_ok=True) + with dep_path.open("w") as f: + o_path = build_path / c_path.with_suffix(".o") + f.write(f"{o_path}:") + depend_list = [] + for entry in symbols_entries: + if entry.function is not None: + func_name = entry.function.getName() + + if func_name in self.global_asm_funcs or is_new_c_file: + outpath = asm_out_dir / self.name / (func_name + ".s") + depend_list.append(outpath) + f.write(f" \\\n {outpath}") + else: + for rodata_sym in entry.rodataSyms: + rodata_name = rodata_sym.getName() + + if rodata_name in self.global_asm_rodata_syms or is_new_c_file: + outpath = asm_out_dir / self.name / (rodata_name + ".s") + depend_list.append(outpath) + f.write(f" \\\n {outpath}") + + f.write("\n") + + for depend_file in depend_list: + f.write(f"{depend_file}:\n") diff --git a/tools/splat/segtypes/common/code.py b/tools/splat/segtypes/common/code.py new file mode 100644 index 0000000..dc9a5bc --- /dev/null +++ b/tools/splat/segtypes/common/code.py @@ -0,0 +1,371 @@ +import typing +from collections import OrderedDict +from typing import Dict, List, Optional, Tuple, Set + +from util import log, options +from util.range import Range +from util.symbols import Symbol + +from segtypes.common.group import CommonSegGroup +from segtypes.segment import Segment + +CODE_TYPES = ["c", "asm", "hasm"] + + +def dotless_type(type: str) -> str: + return type[1:] if type[0] == "." else type + + +# code group +class CommonSegCode(CommonSegGroup): + def __init__( + self, + rom_start: Optional[int], + rom_end: Optional[int], + type: str, + name: str, + vram_start: Optional[int], + args: list, + yaml, + ): + self.bss_size: int = yaml.get("bss_size", 0) if isinstance(yaml, dict) else 0 + + super().__init__( + rom_start, + rom_end, + type, + name, + vram_start, + args=args, + yaml=yaml, + ) + + self.reported_file_split = False + self.jtbl_glabels_to_add: Set[int] = set() + self.jumptables: Dict[int, Tuple[int, int]] = {} + self.rodata_syms: Dict[int, List[Symbol]] = {} + self.align = 0x10 + + @property + def needs_symbols(self) -> bool: + return True + + @property + def vram_end(self) -> Optional[int]: + if self.vram_start is not None and self.size is not None: + return self.vram_start + self.size + self.bss_size + else: + return None + + def check_rodata_sym_impl(self, func_addr: int, sym: Symbol, rodata_section: Range): + if rodata_section.is_complete(): + assert rodata_section.start is not None + assert rodata_section.end is not None + + rodata_start: int = rodata_section.start + rodata_end: int = rodata_section.end + if rodata_start <= sym.vram_start < rodata_end: + if func_addr not in self.rodata_syms: + self.rodata_syms[func_addr] = [] + self.rodata_syms[func_addr].append(sym) + + # Prepare symbol for migration to the function + def check_rodata_sym(self, func_addr: int, sym: Symbol): + rodata_section = self.section_boundaries.get(".rodata") + if rodata_section is not None: + self.check_rodata_sym_impl(func_addr, sym, rodata_section) + rodata_section = self.section_boundaries.get(".rdata") + if rodata_section is not None: + self.check_rodata_sym_impl(func_addr, sym, rodata_section) + + def handle_alls(self, segs: List[Segment], base_segs) -> bool: + for i, elem in enumerate(segs): + if elem.type.startswith("all_"): + alls = [] + + rep_type = f"{elem.type[4:]}" + replace_class = Segment.get_class_for_type(rep_type) + + for base in base_segs.items(): + if isinstance(elem.rom_start, int) and isinstance( + self.rom_start, int + ): + # Shoddy rom to ram + assert self.vram_start is not None, self.vram_start + vram_start = elem.rom_start - self.rom_start + self.vram_start + else: + vram_start = None + rep: Segment = replace_class( + rom_start=elem.rom_start, + rom_end=elem.rom_end, + type=rep_type, + name=base[0], + vram_start=vram_start, + args=[], + yaml={}, + ) + rep.extract = False + rep.given_subalign = self.given_subalign + rep.exclusive_ram_id = self.get_exclusive_ram_id() + rep.given_dir = self.given_dir + rep.given_symbol_name_format = self.symbol_name_format + rep.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom + rep.sibling = base[1] + rep.parent = self + if rep.special_vram_segment: + self.special_vram_segment = True + alls.append(rep) + + # Insert alls into segs at i + del segs[i] + segs[i:i] = alls + return True + return False + + # Find places we should automatically add "all_data" / "all_rodata" / "all_bss" + def find_inserts( + self, found_sections: typing.OrderedDict[str, Range] + ) -> "OrderedDict[str, int]": + inserts: OrderedDict[str, int] = OrderedDict() + + section_order = self.section_order.copy() + section_order.remove(".text") + + for i, section in enumerate(section_order): + if section not in options.opts.auto_all_sections: + continue + + if not found_sections[section].has_start(): + search_done = False + for j in range(i - 1, -1, -1): + end = found_sections[section_order[j]].end + if end is not None: + inserts[section] = end + search_done = True + break + if not search_done: + inserts[section] = -1 + pass + + return inserts + + def parse_subsegments(self, segment_yaml) -> List[Segment]: + if "subsegments" not in segment_yaml: + if not self.parent: + raise Exception( + f"No subsegments provided in top-level code segment {self.name}" + ) + return [] + + base_segments: OrderedDict[str, Segment] = OrderedDict() + ret = [] + prev_start: Optional[int] = -1 + inserts: OrderedDict[ + str, int + ] = ( + OrderedDict() + ) # Used to manually add "all_" types for sections not otherwise defined in the yaml + + self.section_boundaries = OrderedDict( + (s_name, Range()) for s_name in options.opts.section_order + ) + + found_sections = OrderedDict( + (s_name, Range()) for s_name in self.section_boundaries + ) # Stores yaml index where a section was first found + found_sections.pop(".text") + + # Mark any manually added dot types + cur_section = None + + for i, subsection_yaml in enumerate(segment_yaml["subsegments"]): + # endpos marker + if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1: + continue + + typ = Segment.parse_segment_type(subsection_yaml) + if typ.startswith("all_"): + typ = typ[4:] + if not typ.startswith("."): + typ = f".{typ}" + + if typ in found_sections: + if cur_section is None: + # Starting point + found_sections[typ].start = i + cur_section = typ + else: + if cur_section != typ: + # We're changing sections + if found_sections[cur_section].has_end(): + log.error( + f"Section {cur_section} end encountered but was already ended earlier!" + ) + if found_sections[typ].has_start(): + log.error( + f"Section {typ} start encounted but has already started earlier!" + ) + + # End the current section + found_sections[cur_section].end = i + + # Start the next section + found_sections[typ].start = i + cur_section = typ + + if cur_section is not None: + found_sections[cur_section].end = -1 + + inserts = self.find_inserts(found_sections) + + last_rom_end = 0 + + for i, subsection_yaml in enumerate(segment_yaml["subsegments"]): + # endpos marker + if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1: + continue + + typ = Segment.parse_segment_type(subsection_yaml) + start = Segment.parse_segment_start(subsection_yaml) + + # Add dummy segments to be expanded later + if typ.startswith("all_"): + dummy_seg = Segment( + rom_start=start, + rom_end=None, + type=typ, + name="", + vram_start=None, + args=[], + yaml={}, + ) + dummy_seg.given_subalign = self.given_subalign + dummy_seg.exclusive_ram_id = self.exclusive_ram_id + dummy_seg.given_dir = self.given_dir + dummy_seg.given_symbol_name_format = self.symbol_name_format + dummy_seg.given_symbol_name_format_no_rom = ( + self.symbol_name_format_no_rom + ) + ret.append(dummy_seg) + continue + + segment_class = Segment.get_class_for_type(typ) + + end = self.get_next_seg_start(i, segment_yaml["subsegments"]) + + if ( + isinstance(start, int) + and isinstance(prev_start, int) + and start < prev_start + ): + log.error( + f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})" + ) + + vram = None + if start is not None: + assert isinstance(start, int) + vram = self.get_most_parent().rom_to_ram(start) + + if segment_class.is_noload(): + # Pretend bss's rom address is after the last actual rom segment + start = last_rom_end + # and it has a rom size of zero + end = last_rom_end + + segment: Segment = Segment.from_yaml( + segment_class, subsection_yaml, start, end, vram + ) + + segment.sibling = base_segments.get(segment.name, None) + if segment.is_rodata() and segment.sibling is not None: + segment.sibling.rodata_sibling = segment + + segment.parent = self + if segment.special_vram_segment: + self.special_vram_segment = True + + for i, section in enumerate(self.section_order): + if not self.section_boundaries[section].has_start() and dotless_type( + section + ) == dotless_type(segment.type): + if i > 0: + prev_section = self.section_order[i - 1] + self.section_boundaries[prev_section].end = segment.vram_start + self.section_boundaries[section].start = segment.vram_start + + segment.bss_contains_common = self.bss_contains_common + ret.append(segment) + + if segment.is_text(): + base_segments[segment.name] = segment + + prev_start = start + if end is not None: + last_rom_end = end + + # Add the automatic all_ sections + orig_len = len(ret) + for section in reversed(inserts): + idx = inserts[section] + + if idx == -1: + idx = orig_len + + # bss hack TODO maybe rethink + if ( + section == "bss" + and self.vram_start is not None + and self.rom_end is not None + and self.rom_start is not None + ): + rom_start = self.rom_end + vram_start = self.vram_start + self.rom_end - self.rom_start + else: + rom_start = None + vram_start = None + + new_seg = Segment( + rom_start=rom_start, + rom_end=None, + type="all_" + section, + name="", + vram_start=vram_start, + args=[], + yaml={}, + ) + new_seg.given_subalign = self.given_subalign + new_seg.exclusive_ram_id = self.exclusive_ram_id + new_seg.given_dir = self.given_dir + new_seg.given_symbol_name_format = self.symbol_name_format + new_seg.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom + ret.insert(idx, new_seg) + + check = True + while check: + check = self.handle_alls(ret, base_segments) + + # TODO why is this necessary? + rodata_section = self.section_boundaries.get( + ".rodata" + ) or self.section_boundaries.get(".rdata") + if ( + rodata_section is not None + and rodata_section.has_start() + and not rodata_section.has_end() + ): + assert self.vram_end is not None + rodata_section.end = self.vram_end + + return ret + + def scan(self, rom_bytes): + # Always scan code first + for sub in self.subsegments: + if sub.type in CODE_TYPES and sub.should_scan(): + sub.scan(rom_bytes) + + # Scan everyone else + for sub in self.subsegments: + if sub.type not in CODE_TYPES and sub.should_scan(): + sub.scan(rom_bytes) diff --git a/tools/splat/segtypes/common/codesubsegment.py b/tools/splat/segtypes/common/codesubsegment.py new file mode 100644 index 0000000..0423705 --- /dev/null +++ b/tools/splat/segtypes/common/codesubsegment.py @@ -0,0 +1,198 @@ +from typing import Optional + +import spimdisasm +import rabbitizer + +from util import options, symbols, log + +from segtypes import segment +from segtypes.common.code import CommonSegCode + +from segtypes.segment import Segment + + +# abstract class for c, asm, data, etc +class CommonSegCodeSubsegment(Segment): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + vram = segment.parse_segment_vram(self.yaml) + if vram is not None: + self.vram_start = vram + + self.str_encoding: Optional[str] = ( + self.yaml.get("str_encoding", None) if isinstance(self.yaml, dict) else None + ) + + self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None + self.instr_category = rabbitizer.InstrCategory.CPU + if options.opts.platform == "ps2": + self.instr_category = rabbitizer.InstrCategory.R5900 + + @property + def needs_symbols(self) -> bool: + return True + + def get_linker_section(self) -> str: + return ".text" + + def scan_code(self, rom_bytes, is_hasm=False): + if not isinstance(self.rom_start, int): + log.error( + f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" + ) + + # Supposedly logic error, not user error + assert isinstance(self.rom_end, int), self.rom_end + + # Supposedly logic error, not user error + segment_rom_start = self.get_most_parent().rom_start + assert isinstance(segment_rom_start, int), segment_rom_start + + if not isinstance(self.vram_start, int): + log.error( + f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" + ) + + self.spim_section = spimdisasm.mips.sections.SectionText( + symbols.spim_context, + self.rom_start, + self.rom_end, + self.vram_start, + self.name, + rom_bytes, + segment_rom_start, + self.get_exclusive_ram_id(), + ) + + self.spim_section.isHandwritten = is_hasm + self.spim_section.instrCat = self.instr_category + + self.spim_section.analyze() + self.spim_section.setCommentOffset(self.rom_start) + + for func in self.spim_section.symbolList: + assert isinstance(func, spimdisasm.mips.symbols.SymbolFunction) + + self.process_insns(func) + + # Process jumptable labels and pass them to spimdisasm + self.gather_jumptable_labels(rom_bytes) + for jtbl_label_vram in self.parent.jtbl_glabels_to_add: + sym = self.create_symbol( + jtbl_label_vram, True, type="jtbl_label", define=True + ) + sym.type = "jtbl_label" + symbols.add_symbol_to_spim_section(self.spim_section, sym) + + def process_insns( + self, + func_spim: spimdisasm.mips.symbols.SymbolFunction, + ): + assert isinstance(self.parent, CommonSegCode) + assert func_spim.vram is not None + assert func_spim.vramEnd is not None + assert self.spim_section is not None + self.parent: CommonSegCode = self.parent + + symbols.create_symbol_from_spim_symbol( + self.get_most_parent(), func_spim.contextSym + ) + + # Gather symbols found by spimdisasm and create those symbols in splat's side + for referenced_vram in func_spim.instrAnalyzer.referencedVrams: + context_sym = self.spim_section.getSymbol( + referenced_vram, tryPlusOffset=False + ) + if context_sym is not None: + if context_sym.type == spimdisasm.common.SymbolSpecialType.jumptable: + self.parent.jumptables[referenced_vram] = ( + func_spim.vram, + func_spim.vramEnd, + ) + symbols.create_symbol_from_spim_symbol( + self.get_most_parent(), context_sym + ) + + # Main loop + for i, insn in enumerate(func_spim.instructions): + instr_offset = i * 4 + + # update pointer accesses from this function + if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset: + sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset] + + context_sym = self.spim_section.getSymbol( + sym_address, tryPlusOffset=False + ) + if context_sym is not None: + sym = symbols.create_symbol_from_spim_symbol( + self.get_most_parent(), context_sym + ) + + if self.parent: + self.parent.check_rodata_sym(func_spim.vram, sym) + + def print_file_boundaries(self): + if not options.opts.find_file_boundaries or not self.spim_section: + return + + assert isinstance(self.rom_start, int) + + for in_file_offset in self.spim_section.fileBoundaries: + if (in_file_offset % 16) != 0: + continue + + if not self.parent.reported_file_split: + self.parent.reported_file_split = True + + # Look up for the last symbol in this boundary + sym_addr = 0 + for sym in self.spim_section.symbolList: + symOffset = sym.inFileOffset - self.spim_section.inFileOffset + if in_file_offset == symOffset: + break + sym_addr = sym.vram + + print( + f"\nSegment {self.name}, symbol at vram {sym_addr:X} ends with extra nops, indicating a likely file split." + ) + print( + "File split suggestions for this segment will follow in config yaml format:" + ) + print(f" - [0x{self.rom_start+in_file_offset:X}, {self.type}]") + + def gather_jumptable_labels(self, rom_bytes): + assert isinstance(self.rom_start, int) + assert isinstance(self.vram_start, int) + + # TODO: use the seg_symbols for this + # jumptables = [j.type == "jtbl" for j in self.seg_symbols] + for jumptable in self.parent.jumptables: + start, end = self.parent.jumptables[jumptable] + rom_offset = self.rom_start + jumptable - self.vram_start + + if rom_offset <= 0: + return + + while rom_offset: + word = rom_bytes[rom_offset : rom_offset + 4] + word_int = int.from_bytes(word, options.opts.endianness) + if word_int >= start and word_int <= end: + self.parent.jtbl_glabels_to_add.add(word_int) + else: + break + + rom_offset += 4 + + def should_scan(self) -> bool: + return ( + options.opts.is_mode_active("code") + and self.rom_start is not None + and self.rom_end is not None + ) + + def should_split(self) -> bool: + return ( + self.extract and options.opts.is_mode_active("code") and self.should_scan() + ) # only split if the segment was scanned first diff --git a/tools/splat/segtypes/common/data.py b/tools/splat/segtypes/common/data.py new file mode 100644 index 0000000..e97ce6d --- /dev/null +++ b/tools/splat/segtypes/common/data.py @@ -0,0 +1,124 @@ +from pathlib import Path +from typing import Optional + +import spimdisasm +from util import options, symbols, log + +from segtypes.common.codesubsegment import CommonSegCodeSubsegment +from segtypes.common.group import CommonSegGroup + + +class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup): + def out_path(self) -> Optional[Path]: + if self.type.startswith("."): + if self.sibling: + # C file + return self.sibling.out_path() + else: + # Implied C file + return options.opts.src_path / self.dir / f"{self.name}.c" + else: + # ASM + return options.opts.data_path / self.dir / f"{self.name}.{self.type}.s" + + def scan(self, rom_bytes: bytes): + CommonSegGroup.scan(self, rom_bytes) + + if self.should_scan(): + self.disassemble_data(rom_bytes) + + def split(self, rom_bytes: bytes): + super().split(rom_bytes) + + if ( + not self.type.startswith(".") + and self.spim_section + and self.should_self_split() + ): + path = self.out_path() + + if path: + path.parent.mkdir(parents=True, exist_ok=True) + + self.print_file_boundaries() + + with open(path, "w", newline="\n") as f: + f.write('.include "macro.inc"\n\n') + preamble = options.opts.generated_s_preamble + if preamble: + f.write(preamble + "\n") + f.write(f".section {self.get_linker_section()}\n\n") + + f.write(self.spim_section.disassemble()) + + def should_self_split(self) -> bool: + return options.opts.is_mode_active("data") + + def should_scan(self) -> bool: + # Ensure data segments are scanned even if extract is False so subsegments get scanned too + # Check for not None so we avoid scanning "auto" segments + return self.rom_start is not None and self.rom_end is not None + + def should_split(self) -> bool: + return True + + def cache(self): + return [CommonSegCodeSubsegment.cache(self), CommonSegGroup.cache(self)] + + def get_linker_section(self) -> str: + return ".data" + + def get_linker_entries(self): + return CommonSegCodeSubsegment.get_linker_entries(self) + + def disassemble_data(self, rom_bytes): + if not isinstance(self.rom_start, int): + log.error( + f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" + ) + + # Supposedly logic error, not user error + assert isinstance(self.rom_end, int), self.rom_end + + # Supposedly logic error, not user error + segment_rom_start = self.get_most_parent().rom_start + assert isinstance(segment_rom_start, int), segment_rom_start + + if not isinstance(self.vram_start, int): + log.error( + f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" + ) + + self.spim_section = spimdisasm.mips.sections.SectionData( + symbols.spim_context, + self.rom_start, + self.rom_end, + self.vram_start, + self.name, + rom_bytes, + segment_rom_start, + self.get_exclusive_ram_id(), + ) + + self.spim_section.analyze() + self.spim_section.setCommentOffset(self.rom_start) + + rodata_encountered = False + + for symbol in self.spim_section.symbolList: + symbols.create_symbol_from_spim_symbol( + self.get_most_parent(), symbol.contextSym + ) + + # Hint to the user that we are now in the .rodata section and no longer in the .data section (assuming rodata follows data) + if not rodata_encountered and self.get_most_parent().rodata_follows_data: + if symbol.contextSym.isJumpTable(): + rodata_encountered = True + print( + f"Data segment {self.name}, symbol at vram {symbol.contextSym.vram:X} is a jumptable, indicating the start of the rodata section _may_ be near here." + ) + print( + "Please note the real start of the rodata section may be way before this point." + ) + if symbol.contextSym.vromAddress is not None: + print(f" - [0x{symbol.contextSym.vromAddress:X}, rodata]") diff --git a/tools/splat/segtypes/common/decompressor.py b/tools/splat/segtypes/common/decompressor.py new file mode 100644 index 0000000..3bdc5ea --- /dev/null +++ b/tools/splat/segtypes/common/decompressor.py @@ -0,0 +1,48 @@ +from typing import Optional, Any + +from util import log, options +from util.n64.decompressor import Decompressor + +from segtypes.n64.segment import N64Segment + + +class CommonSegDecompressor(N64Segment): + decompressor: Decompressor + compression_type = "" # "Mio0" -> filename.Mio0.o + + def split(self, rom_bytes): + if self.decompressor is None: + log.error("Decompressor is not a standalone segment type") + + out_dir = options.opts.asset_path / self.dir + out_dir.mkdir(parents=True, exist_ok=True) + + if self.rom_end is None: + log.error( + f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" + ) + + out_path = out_dir / f"{self.name}.bin" + with open(out_path, "wb") as f: + assert isinstance(self.rom_start, int) + assert isinstance(self.rom_end, int) + + self.log(f"Decompressing {self.name}") + compressed_bytes = rom_bytes[self.rom_start : self.rom_end] + decompressed_bytes = self.decompressor.decompress(compressed_bytes) + f.write(decompressed_bytes) + self.log(f"Wrote {self.name} to {out_path}") + + def get_linker_entries(self): + from segtypes.linker_entry import LinkerEntry + + return [ + LinkerEntry( + self, + [options.opts.asset_path / self.dir / f"{self.name}.bin"], + options.opts.asset_path + / self.dir + / f"{self.name}.{self.compression_type}", + self.get_linker_section(), + ) + ] diff --git a/tools/splat/segtypes/common/group.py b/tools/splat/segtypes/common/group.py new file mode 100644 index 0000000..fa263de --- /dev/null +++ b/tools/splat/segtypes/common/group.py @@ -0,0 +1,146 @@ +from typing import List, Optional + +from util import log + +from segtypes.common.segment import CommonSegment +from segtypes.segment import Segment + + +class CommonSegGroup(CommonSegment): + def __init__( + self, + rom_start: Optional[int], + rom_end: Optional[int], + type: str, + name: str, + vram_start: Optional[int], + args: list, + yaml, + ): + super().__init__( + rom_start, + rom_end, + type, + name, + vram_start, + args=args, + yaml=yaml, + ) + + self.subsegments: List[Segment] = self.parse_subsegments(yaml) + + def get_next_seg_start(self, i, subsegment_yamls): + return ( + self.rom_end + if i == len(subsegment_yamls) - 1 + else Segment.parse_segment_start(subsegment_yamls[i + 1]) + ) + + def parse_subsegments(self, yaml) -> List[Segment]: + ret: List[Segment] = [] + + if not yaml or "subsegments" not in yaml: + return ret + + prev_start: Optional[int] = -1 + last_rom_end = 0 + + for i, subsection_yaml in enumerate(yaml["subsegments"]): + # endpos marker + if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1: + continue + + typ = Segment.parse_segment_type(subsection_yaml) + start = Segment.parse_segment_start(subsection_yaml) + + segment_class = Segment.get_class_for_type(typ) + + end = self.get_next_seg_start(i, yaml["subsegments"]) + + if ( + isinstance(start, int) + and isinstance(prev_start, int) + and start < prev_start + ): + log.error( + f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})" + ) + + vram = None + if start is not None: + assert isinstance(start, int) + vram = self.get_most_parent().rom_to_ram(start) + + if segment_class.is_noload(): + # Pretend bss's rom address is after the last actual rom segment + start = last_rom_end + # and it has a rom size of zero + end = last_rom_end + + segment: Segment = Segment.from_yaml( + segment_class, subsection_yaml, start, end, vram + ) + segment.parent = self + if segment.special_vram_segment: + self.special_vram_segment = True + + ret.append(segment) + prev_start = start + if end is not None: + last_rom_end = end + + return ret + + @property + def needs_symbols(self) -> bool: + for seg in self.subsegments: + if seg.needs_symbols: + return True + return False + + def get_linker_entries(self): + return [entry for sub in self.subsegments for entry in sub.get_linker_entries()] + + def scan(self, rom_bytes): + for sub in self.subsegments: + if sub.should_scan(): + sub.scan(rom_bytes) + + def split(self, rom_bytes): + for sub in self.subsegments: + if sub.should_split(): + sub.split(rom_bytes) + + def should_split(self) -> bool: + return self.extract + + def should_scan(self) -> bool: + return self.extract + + def cache(self): + c = [] + + for sub in self.subsegments: + c.append(sub.cache()) + + return c + + def get_subsegment_for_ram(self, addr: int) -> Optional[Segment]: + for sub in self.subsegments: + if sub.contains_vram(addr): + return sub + return None + + def get_next_subsegment_for_ram(self, addr: int) -> Optional[Segment]: + """ + Returns the first subsegment which comes after the specified address, + or None in case this address belongs to the last subsegment of this group + """ + + for sub in self.subsegments: + if sub.vram_start is None: + continue + assert isinstance(sub.vram_start, int) + if sub.vram_start > addr: + return sub + return None diff --git a/tools/splat/segtypes/common/hasm.py b/tools/splat/segtypes/common/hasm.py new file mode 100644 index 0000000..de7c54c --- /dev/null +++ b/tools/splat/segtypes/common/hasm.py @@ -0,0 +1,24 @@ +from segtypes.common.asm import CommonSegAsm + + +class CommonSegHasm(CommonSegAsm): + def scan(self, rom_bytes: bytes): + if ( + self.rom_start is not None + and self.rom_end is not None + and self.rom_start != self.rom_end + ): + self.scan_code(rom_bytes, is_hasm=True) + + def split(self, rom_bytes: bytes): + if not self.rom_start == self.rom_end and self.spim_section is not None: + out_path = self.out_path() + if out_path and not out_path.exists(): + out_path.parent.mkdir(parents=True, exist_ok=True) + + self.print_file_boundaries() + + with open(out_path, "w", newline="\n") as f: + for line in self.get_file_header(): + f.write(line + "\n") + f.write(self.spim_section.disassemble()) diff --git a/tools/splat/segtypes/common/header.py b/tools/splat/segtypes/common/header.py new file mode 100644 index 0000000..b03b0ae --- /dev/null +++ b/tools/splat/segtypes/common/header.py @@ -0,0 +1,42 @@ +from pathlib import Path + +from util import options + +from segtypes.common.segment import CommonSegment + + +class CommonSegHeader(CommonSegment): + def should_split(self): + return self.extract and options.opts.is_mode_active("code") + + @staticmethod + def get_line(typ, data, comment): + if typ == "ascii": + text = data.decode("ASCII").strip() + text = text.replace("\x00", "\\0") # escape NUL chars + dstr = '"' + text + '"' + else: # .word, .byte + dstr = "0x" + data.hex().upper() + + dstr = dstr.ljust(20 - len(typ)) + + return f".{typ} {dstr} /* {comment} */" + + def out_path(self) -> Path: + return options.opts.asm_path / self.dir / f"{self.name}.s" + + def parse_header(self, rom_bytes): + return [] + + def split(self, rom_bytes): + header_lines = self.parse_header(rom_bytes) + + src_path = self.out_path() + src_path.parent.mkdir(parents=True, exist_ok=True) + with open(src_path, "w", newline="\n") as f: + f.write("\n".join(header_lines)) + self.log(f"Wrote {self.name} to {src_path}") + + @staticmethod + def get_default_name(addr): + return "header" diff --git a/tools/splat/segtypes/common/lib.py b/tools/splat/segtypes/common/lib.py new file mode 100644 index 0000000..51371c1 --- /dev/null +++ b/tools/splat/segtypes/common/lib.py @@ -0,0 +1,53 @@ +from pathlib import Path +from typing import Optional + +from util import log, options + +from segtypes.linker_entry import LinkerEntry +from segtypes.n64.segment import N64Segment + + +class CommonSegLib(N64Segment): + def __init__( + self, + rom_start: Optional[int], + rom_end: Optional[int], + type: str, + name: str, + vram_start: Optional[int], + args: list, + yaml, + ): + super().__init__( + rom_start, + rom_end, + type, + name, + vram_start, + args=args, + yaml=yaml, + ) + + if isinstance(yaml, dict): + log.error("Error: 'dict' not currently supported for 'lib' segment") + return + if len(args) < 1: + log.error(f"Error: {self.name} is missing object file") + return + + self.extract = False + + if len(args) > 1: + self.object, self.section = args[0], args[1] + else: + self.object, self.section = args[0], ".text" + + def get_linker_section(self) -> str: + return self.section + + def get_linker_entries(self): + path = options.opts.lib_path / self.name + + object_path = Path(f"{path}.a:{self.object}.o") + + return [LinkerEntry(self, [path], object_path, self.get_linker_section())] diff --git a/tools/splat/segtypes/common/rdata.py b/tools/splat/segtypes/common/rdata.py new file mode 100644 index 0000000..37adbef --- /dev/null +++ b/tools/splat/segtypes/common/rdata.py @@ -0,0 +1,6 @@ +from segtypes.common.rodata import CommonSegRodata + + +class CommonSegRdata(CommonSegRodata): + def get_linker_section(self) -> str: + return ".rdata" diff --git a/tools/splat/segtypes/common/rodata.py b/tools/splat/segtypes/common/rodata.py new file mode 100644 index 0000000..a189671 --- /dev/null +++ b/tools/splat/segtypes/common/rodata.py @@ -0,0 +1,97 @@ +from typing import Optional, Set, Tuple +import spimdisasm +from segtypes.segment import Segment +from util import log, options, symbols + +from segtypes.common.data import CommonSegData + + +class CommonSegRodata(CommonSegData): + def get_linker_section(self) -> str: + return ".rodata" + + @staticmethod + def is_rodata() -> bool: + return True + + def get_possible_text_subsegment_for_symbol( + self, rodata_sym: spimdisasm.mips.symbols.SymbolBase + ) -> Optional[Tuple[Segment, spimdisasm.common.ContextSymbol]]: + # Check if this rodata segment does not have a corresponding code file, try to look for one + + if self.sibling is not None or not options.opts.pair_rodata_to_text: + return None + + if not rodata_sym.shouldMigrate(): + return None + + if len(rodata_sym.contextSym.referenceFunctions) != 1: + return None + + func = list(rodata_sym.contextSym.referenceFunctions)[0] + text_segment = self.parent.get_subsegment_for_ram(func.vram) + + if text_segment is None or not text_segment.is_text(): + return None + return text_segment, func + + def disassemble_data(self, rom_bytes): + if not isinstance(self.rom_start, int): + log.error( + f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" + ) + + # Supposedly logic error, not user error + assert isinstance(self.rom_end, int), self.rom_end + + # Supposedly logic error, not user error + segment_rom_start = self.get_most_parent().rom_start + assert isinstance(segment_rom_start, int), segment_rom_start + + if not isinstance(self.vram_start, int): + log.error( + f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" + ) + + self.spim_section = spimdisasm.mips.sections.SectionRodata( + symbols.spim_context, + self.rom_start, + self.rom_end, + self.vram_start, + self.name, + rom_bytes, + segment_rom_start, + self.get_exclusive_ram_id(), + ) + + # Set rodata string encoding + # First check the global configuration + if options.opts.string_encoding is not None: + self.spim_section.stringEncoding = options.opts.string_encoding + + # Then check the per-segment configuration in case we want to override the global one + if self.str_encoding is not None: + self.spim_section.stringEncoding = self.str_encoding + + self.spim_section.analyze() + self.spim_section.setCommentOffset(self.rom_start) + + possible_text_segments: Set[Segment] = set() + + for symbol in self.spim_section.symbolList: + generated_symbol = symbols.create_symbol_from_spim_symbol( + self.get_most_parent(), symbol.contextSym + ) + generated_symbol.linker_section = self.get_linker_section() + + possible_text = self.get_possible_text_subsegment_for_symbol(symbol) + if possible_text is not None: + text_segment, refenceeFunction = possible_text + if text_segment not in possible_text_segments: + print( + f"\nRodata segment '{self.name}' may belong to the text segment '{text_segment.name}'" + ) + print( + f" Based on the usage from the function {refenceeFunction.getName()} to the symbol {symbol.getName()}" + ) + possible_text_segments.add(text_segment) diff --git a/tools/splat/segtypes/common/sbss.py b/tools/splat/segtypes/common/sbss.py new file mode 100644 index 0000000..cbd0046 --- /dev/null +++ b/tools/splat/segtypes/common/sbss.py @@ -0,0 +1,6 @@ +from segtypes.common.data import CommonSegData + + +class CommonSegSbss(CommonSegData): + def get_linker_section(self) -> str: + return ".sbss" diff --git a/tools/splat/segtypes/common/sdata.py b/tools/splat/segtypes/common/sdata.py new file mode 100644 index 0000000..a3a0e52 --- /dev/null +++ b/tools/splat/segtypes/common/sdata.py @@ -0,0 +1,6 @@ +from segtypes.common.data import CommonSegData + + +class CommonSegSdata(CommonSegData): + def get_linker_section(self) -> str: + return ".sdata" diff --git a/tools/splat/segtypes/common/segment.py b/tools/splat/segtypes/common/segment.py new file mode 100644 index 0000000..10e2b85 --- /dev/null +++ b/tools/splat/segtypes/common/segment.py @@ -0,0 +1,5 @@ +from segtypes.segment import Segment + + +class CommonSegment(Segment): + pass diff --git a/tools/splat/segtypes/gc/apploader.py b/tools/splat/segtypes/gc/apploader.py new file mode 100644 index 0000000..810e822 --- /dev/null +++ b/tools/splat/segtypes/gc/apploader.py @@ -0,0 +1,8 @@ +import struct +from pathlib import Path + +from segtypes.gc.segment import GCSegment + + +class GcSegApploader(GCSegment): + pass diff --git a/tools/splat/segtypes/gc/bi2.py b/tools/splat/segtypes/gc/bi2.py new file mode 100644 index 0000000..2bf6a97 --- /dev/null +++ b/tools/splat/segtypes/gc/bi2.py @@ -0,0 +1,67 @@ +import struct +from pathlib import Path + +from util import options + +from segtypes.gc.segment import GCSegment + + +class GcSegBi2(GCSegment): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def split(self, bi2_bytes): + lines = [] + + # Gathering variables + debug_monitor_size = struct.unpack_from(">I", bi2_bytes, 0x00)[0] + simulated_memory_size = struct.unpack_from(">I", bi2_bytes, 0x04)[0] + + argument_offset = struct.unpack_from(">I", bi2_bytes, 0x08)[0] + + debug_flag = struct.unpack_from(">I", bi2_bytes, 0x0C)[0] + + track_offset = struct.unpack_from(">I", bi2_bytes, 0x10)[0] + track_size = struct.unpack_from(">I", bi2_bytes, 0x14)[0] + + country_code_bi2 = struct.unpack_from(">I", bi2_bytes, 0x18)[0] + + unk_int = struct.unpack_from(">I", bi2_bytes, 0x1C)[0] + unk_int_2 = struct.unpack_from(">I", bi2_bytes, 0x20)[0] + + # Outputting .s file + lines.append(f"# GameCube disc image bi2 data, located at 0x440 in the disc.\n") + lines.append(f"# Generated by splat.\n\n") + + lines.append(f".section .data\n\n") + + lines.append(f"debug_monitor_size: .long 0x{debug_monitor_size:08X}\n") + lines.append(f"simulated_memory_size: .long 0x{simulated_memory_size:08X}\n\n") + + lines.append(f"argument_offset: .long 0x{argument_offset:08X}\n\n") + + lines.append(f"debug_flag: .long 0x{debug_flag:08X}\n\n") + + lines.append(f"track_offset: .long 0x{track_offset:08X}\n") + lines.append(f"track_size: .long 0x{track_size:08X}\n\n") + + lines.append(f"country_code_bi2: .long 0x{country_code_bi2:08X}\n\n") + + lines.append(f"ukn_int_bi2: .long 0x{unk_int:08X}\n") + lines.append(f"ukn_int_bi2_2: .long 0x{unk_int_2:08X}\n\n") + + lines.append(f".fill 0x00001FDC\n\n") + + out_path = self.out_path() + out_path.parent.mkdir(parents=True, exist_ok=True) + + with open(out_path, "w", encoding="utf-8") as f: + f.writelines(lines) + + return + + def should_split(self) -> bool: + return True + + def out_path(self) -> Path: + return options.opts.asm_path / "sys" / "bi2.s" diff --git a/tools/splat/segtypes/gc/bootinfo.py b/tools/splat/segtypes/gc/bootinfo.py new file mode 100644 index 0000000..28a93fd --- /dev/null +++ b/tools/splat/segtypes/gc/bootinfo.py @@ -0,0 +1,117 @@ +import struct +from pathlib import Path + +from util import options + +from segtypes.gc.segment import GCSegment + + +class GcSegBootinfo(GCSegment): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def split(self, iso_bytes): + lines = [] + + gc_dvd_magic = struct.unpack_from(">I", iso_bytes, 0x1C)[0] + assert gc_dvd_magic == 0xC2339F3D + + # Gathering variables + system_code = chr(iso_bytes[0x00]) + game_code = iso_bytes[0x01:0x03].decode("utf-8") + region_code = chr(iso_bytes[0x03]) + publisher_code = iso_bytes[0x04:0x06].decode("utf-8") + + disc_id = iso_bytes[0x06] + game_version = iso_bytes[0x07] + audio_streaming = iso_bytes[0x08] + stream_buffer_size = iso_bytes[0x09] + + name = iso_bytes[0x20:0x400].decode("utf-8").strip("\x00") + name_padding_len = 0x3E0 - len(name) + + # The following is from YAGCD, don't know what they were for: + # https://web.archive.org/web/20220528011846/http://hitmen.c02.at/files/yagcd/yagcd/chap13.html#sec13.1 + apploader_size = struct.unpack_from(">I", iso_bytes, 0x400)[0] + debug_monitor_address = struct.unpack_from(">I", iso_bytes, 0x404)[0] + + # These on the other hand are easy to understand + dol_offset = struct.unpack_from(">I", iso_bytes, 0x420)[0] + fst_offset = struct.unpack_from(">I", iso_bytes, 0x424)[0] + fst_size = struct.unpack_from(">I", iso_bytes, 0x428)[0] + fst_max_size = struct.unpack_from(">I", iso_bytes, 0x42C)[0] + + user_position = struct.unpack_from(">I", iso_bytes, 0x430)[0] + user_length = struct.unpack_from(">I", iso_bytes, 0x434)[0] + unk_int = struct.unpack_from(">I", iso_bytes, 0x438)[0] + + # Outputting .s file + lines.append(f"# GameCube disc image boot data, located at 0x00 in the disc.\n") + lines.append(f"# Generated by splat.\n\n") + + lines.append(f".section .data\n\n") + + # Game ID stuff + lines.append(f'system_code: .ascii "{system_code}"\n') + lines.append(f'game_code: .ascii "{game_code}"\n') + lines.append(f'region_code: .ascii "{region_code}"\n') + lines.append(f'publisher_code: .ascii "{publisher_code}"\n\n') + + lines.append(f"disc_id: .byte {disc_id:X}\n") + lines.append(f"game_version: .byte {game_version:X}\n") + lines.append(f"audio_streaming: .byte {audio_streaming:X}\n") + lines.append(f"stream_buffer_size: .byte {stream_buffer_size:X}\n\n") + + # padding + lines.append(f".fill 0x12\n\n") + + # GC magic number + lines.append(f"gc_magic: .long 0xC2339F3D\n\n") + + # Long game name + lines.append(f'game_name: .ascii "{name}"\n') + lines.append(f".org 0x400\n\n") + + lines.append(f"apploader_size: .long 0x{apploader_size:08X}\n\n") + + # Unknown stuff gleaned from YAGCD + lines.append(f"debug_monitor_address: .long 0x{debug_monitor_address:08X}\n\n") + + # More padding + lines.append(f".fill 0x18\n\n") + + # DOL and FST data + lines.append(f"dol_offset: .long 0x{dol_offset:08X}\n") + lines.append(f"fst_offset: .long 0x{fst_offset:08X}\n\n") + + lines.append( + f"# The FST is only allocated once per game boot, even in games with multiple disks. fst_max_size is used to ensure that\n" + ) + lines.append( + f"# there is enough space allocated for the FST in the event that a game spans multiple disks, and one disk has a larger FST than another.\n" + ) + lines.append(f"fst_size: .long 0x{fst_size:08X}\n") + lines.append(f"fst_max_size: .long 0x{fst_max_size:08X}\n\n") + + # Honestly not sure what this data is for + lines.append(f"# Not even YAGCD knows what these are for.\n") + lines.append(f"user_position: .long 0x{user_position:08X}\n") + lines.append(f"user_length: .long 0x{user_length:08X}\n") + lines.append(f"unk_int: .long 0x{unk_int:08X}\n\n") + + # Final padding + lines.append(f".word 0\n") + out_path = self.out_path() + + out_path.parent.mkdir(parents=True, exist_ok=True) + + with open(out_path, "w", encoding="utf-8") as f: + f.writelines(lines) + + return + + def should_split(self) -> bool: + return True + + def out_path(self) -> Path: + return options.opts.asm_path / "sys" / "boot.s" diff --git a/tools/splat/segtypes/gc/dol.py b/tools/splat/segtypes/gc/dol.py new file mode 100644 index 0000000..3a92bda --- /dev/null +++ b/tools/splat/segtypes/gc/dol.py @@ -0,0 +1,8 @@ +import struct +from pathlib import Path + +from segtypes.gc.segment import GCSegment + + +class GcSegDol(GCSegment): + pass diff --git a/tools/splat/segtypes/gc/dolheader.py b/tools/splat/segtypes/gc/dolheader.py new file mode 100644 index 0000000..f8a060a --- /dev/null +++ b/tools/splat/segtypes/gc/dolheader.py @@ -0,0 +1,68 @@ +from util import options + +from segtypes.common.header import CommonSegHeader + + +class DolSegHeader(CommonSegHeader): + def parse_header(self, dol_bytes): + header_lines = [] + header_lines.append(".section .data\n") + + # Text file offsets + for i in range(0x00, 0x1C, 4): + header_lines.append( + self.get_line("word", dol_bytes[i : i + 4], f"Text {i / 4} Offset") + ) + # Data file offsets + for i in range(0x1C, 0x48, 4): + header_lines.append( + self.get_line("word", dol_bytes[i : i + 4], f"Data {i / 4} Offset") + ) + + # Text RAM addresses + for i in range(0x48, 0x64, 4): + header_lines.append( + self.get_line( + "word", + dol_bytes[i : i + 4], + f"Text {(i - 0x48) / 4} Address", + ) + ) + # Data RAM addresses + for i in range(0x64, 0x90, 4): + header_lines.append( + self.get_line( + "word", + dol_bytes[i : i + 4], + f"Data {(i - 0x64) / 4} Address", + ) + ) + + # Text file sizes + for i in range(0x90, 0xAC, 4): + header_lines.append( + self.get_line( + "word", + dol_bytes[i : i + 4], + f"Text {(i - 0x90) / 4} Size", + ) + ) + # Data file sizes + for i in range(0xAC, 0xD8, 4): + header_lines.append( + self.get_line( + "word", + dol_bytes[i : i + 4], + f"Data {(i - 0xAC) / 4} Size", + ) + ) + + # BSS RAM address + header_lines.append(self.get_line("word", dol_bytes[0xD8:0xDC], "BSS Address")) + # BSS size + header_lines.append(self.get_line("word", dol_bytes[0xDC:0xE0], "BSS Size")) + + # Entry point + header_lines.append(self.get_line("word", dol_bytes[0xE0:0xE4], "Entry Point")) + + return header_lines diff --git a/tools/splat/segtypes/gc/fst.py b/tools/splat/segtypes/gc/fst.py new file mode 100644 index 0000000..be091fe --- /dev/null +++ b/tools/splat/segtypes/gc/fst.py @@ -0,0 +1,8 @@ +import struct +from pathlib import Path + +from segtypes.gc.segment import GCSegment + + +class GcSegFst(GCSegment): + pass diff --git a/tools/splat/segtypes/gc/rarc.py b/tools/splat/segtypes/gc/rarc.py new file mode 100644 index 0000000..43eda14 --- /dev/null +++ b/tools/splat/segtypes/gc/rarc.py @@ -0,0 +1,321 @@ +import struct +from enum import IntEnum +from pathlib import Path + +from typing import List, Optional + +from util import options +from util.gc.gcutil import read_string_from_bytes +from util.n64.Yay0decompress import Yay0Decompressor + +from segtypes.gc.segment import GCSegment + + +# Represents the RARC archive format used by first-party Nintendo games. +class GcSegRarc(GCSegment): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def split(self, file_bytes): + assert self.file_path is not None + + archive = GCRARCArchive(self.file_path, file_bytes) + archive.build_hierarchy(file_bytes) + + archive.emit(file_bytes) + + def should_split(self) -> bool: + return True + + +class GCRARCArchive: + def __init__(self, file_path: Path, file_bytes): + self.file_path = file_path + self.compression = "none" + file_bytes = self.try_decompress_archive(file_bytes) + + self.magic = struct.unpack_from(">I", file_bytes, 0x0000)[0] + self.file_size = struct.unpack_from(">I", file_bytes, 0x0004)[0] + self.data_header_offset = struct.unpack_from(">I", file_bytes, 0x0008)[0] + self.file_data_offset = struct.unpack_from(">I", file_bytes, 0x000C)[0] + 0x0020 + self.total_file_data_size = struct.unpack_from(">I", file_bytes, 0x0010)[0] + + self.mram_preload_size = struct.unpack_from(">I", file_bytes, 0x0014)[0] + self.aram_preload_size = struct.unpack_from(">I", file_bytes, 0x0018)[0] + + self.data_header = GCRARCDataHeader(self.data_header_offset, file_bytes) + self.nodes: List[GCRARCNode] = [] + + def try_decompress_archive(self, file_bytes): + compression_scheme = struct.unpack_from(">I", file_bytes, 0x0000)[0] + + # Yaz0 + if compression_scheme == 0x59617A30: + self.compression = "yaz0" + return file_bytes + # Yay0 + elif compression_scheme == 0x59617930: + self.compression = "yay0" + return Yay0Decompressor().decompress(file_bytes) + # Not compressed! + else: + return file_bytes + + def build_hierarchy(self, file_bytes): + string_table_offset = self.data_header.string_table_offset + string_table_size = self.data_header.string_table_size + + string_table_bytes = file_bytes[ + string_table_offset : string_table_offset + string_table_size + ] + + # Load the file entries into their corresponding nodes. + for i in range(self.data_header.node_count): + offset = self.data_header.node_offset + i * 0x10 + + new_node = GCRARCNode(offset, file_bytes, string_table_bytes) + new_node.get_entries( + self.data_header.file_entry_offset, file_bytes, string_table_bytes + ) + + self.nodes.append(new_node) + + # Now, organize the nodes into a hierarchy. + for n in self.nodes: + for e in n.entries: + # We're only looking for directory nodes, so ignore files. + if e.flags & int(GCRARCFlags.IS_FILE) != 0x00: + continue + + if e.name == "." or e.name == "..": + continue + + # This is the node that the current entry corresponds to. + dir_node = self.nodes[e.data_offset] + + # Set up hierarchy relationship. + dir_node.parent = n + n.children.append(dir_node) + + def emit(self, file_bytes): + assert options.opts.filesystem_path is not None + + rel_path = self.file_path.relative_to(options.opts.filesystem_path / "files") + arc_root_path = options.opts.asset_path / rel_path.with_suffix("") + + self.nodes[0].emit_to_filesystem_recursive( + arc_root_path, self.file_data_offset, file_bytes + ) + self.emit_config(arc_root_path) + + def emit_config(self, config_path: Path): + lines = [] + + lines.append(f'name: "{self.file_path.name}"\n') + + if self.compression != "none": + lines.append(f"compression: {self.compression}\n") + + lines.append(f"next_file_id: 0x{self.data_header.next_free_file_id:04X}\n") + lines.append( + f"sync_file_ids_to_indices: {self.data_header.sync_file_ids_to_indices}\n" + ) + + root_node = self.nodes[0] + + lines.append("root_dir:\n") + lines.append(f' res_type: "{root_node.resource_type}"\n') + lines.append(f' name: "{root_node.name}"\n') + + if len(root_node.entries) != 0: + lines.append(" entries:\n") + for e in root_node.entries: + entry_config = e.emit_config(2) + if entry_config != None: + lines.extend(entry_config) + + if len(root_node.children) != 0: + lines.append(" subdirs:\n") + for n in root_node.children: + node_config = n.emit_config(2) + if node_config != None: + lines.extend(node_config) + + with open(config_path / "arcinfo.yaml", "w", newline="\n") as f: + f.writelines(lines) + + +class GCRARCDataHeader: + def __init__(self, offset, file_bytes): + self.node_count = struct.unpack_from(">I", file_bytes, offset + 0x0000)[0] + self.node_offset = ( + struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] + 0x0020 + ) + + self.file_entry_count = struct.unpack_from(">I", file_bytes, offset + 0x0008)[0] + self.file_entry_offset = ( + struct.unpack_from(">I", file_bytes, offset + 0x000C)[0] + 0x0020 + ) + + self.string_table_size = struct.unpack_from(">I", file_bytes, offset + 0x0010)[ + 0 + ] + self.string_table_offset = ( + struct.unpack_from(">I", file_bytes, offset + 0x0014)[0] + 0x0020 + ) + + self.next_free_file_id = struct.unpack_from(">H", file_bytes, offset + 0x0018)[ + 0 + ] + self.sync_file_ids_to_indices = bool(file_bytes[offset + 0x001A]) + + +class GCRARCNode: + def __init__(self, offset, file_bytes, string_table_bytes): + self.resource_type = file_bytes[offset + 0x0000 : offset + 0x0004].decode( + "utf-8" + ) + self.name_offset = struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] + self.name_hash = struct.unpack_from(">H", file_bytes, offset + 0x0008)[0] + self.file_entry_count = struct.unpack_from(">H", file_bytes, offset + 0x000A)[0] + self.first_file_entry_index = struct.unpack_from( + ">I", file_bytes, offset + 0x000C + )[0] + + self.name = read_string_from_bytes(self.name_offset, string_table_bytes) + self.entries = [] + + self.parent: Optional[GCRARCNode] = None + self.children = [] + + def get_entries(self, file_entry_offset, file_bytes, string_table_bytes): + for i in range(self.file_entry_count): + entry_offset = file_entry_offset + (self.first_file_entry_index + i) * 0x14 + + new_entry = GCRARCFileEntry(entry_offset, file_bytes, string_table_bytes) + new_entry.parent_node = self + + self.entries.append(new_entry) + + def emit_to_filesystem_recursive( + self, root_path: Path, file_data_offset, file_bytes + ): + dir_path = root_path / self.get_full_directory_path() + dir_path.mkdir(parents=True, exist_ok=True) + + for n in self.children: + n.emit_to_filesystem_recursive(root_path, file_data_offset, file_bytes) + + for e in self.entries: + e.emit_to_filesystem(root_path, file_data_offset, file_bytes) + + def emit_config(self, level): + lines = [] + + lines.append(" " * level + f'- res_type: "{self.resource_type}"\n') + lines.append(" " * level + f' name: "{self.name}"\n') + + if len(self.entries) != 0: + lines.append(" " * level + " entries:\n") + for e in self.entries: + entry_config = e.emit_config(level + 1) + if entry_config != None: + lines.extend(entry_config) + + if len(self.children) != 0: + lines.append(" " * level + " subdirs:\n") + for n in self.children: + node_config = n.emit_config(level + 1) + if node_config != None: + lines.extend(node_config) + + return lines + + def print_recursive(self, level): + print((" " * level) + self.name) + + for n in self.children: + n.print_recursive(level + 1) + + def get_full_directory_path(self): + path_components: List[str] = [] + + node: Optional[GCRARCNode] = self + while node is not None: + path_components.insert(0, node.name) + node = node.parent + + return Path(*path_components) + + +class GCRARCFileEntry: + def __init__(self, offset, file_bytes, string_table_bytes): + self.file_id = struct.unpack_from(">H", file_bytes, offset + 0x0000)[0] + self.name_hash = struct.unpack_from(">H", file_bytes, offset + 0x0002)[0] + self.flags = file_bytes[offset + 0x0004] + self.name_offset = ( + struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] & 0x00FFFFFF + ) + self.data_offset = struct.unpack_from(">I", file_bytes, offset + 0x0008)[0] + self.data_size = struct.unpack_from(">I", file_bytes, offset + 0x000C)[0] + + self.name = read_string_from_bytes(self.name_offset, string_table_bytes) + self.parent_node: Optional[GCRARCNode] = None + + def emit_to_filesystem(self, dir_path: Path, file_data_offset, file_bytes): + if self.flags & int(GCRARCFlags.IS_DIR) != 0x00: + return + + file_path = dir_path / self.get_full_file_path() + + file_data = file_bytes[ + file_data_offset + + self.data_offset : file_data_offset + + self.data_offset + + self.data_size + ] + with open(file_path, "wb") as f: + f.write(file_data) + + def emit_config(self, level): + if self.flags & int(GCRARCFlags.IS_DIR) != 0x00: + return + + lines = [] + + lines.append(" " * level + f' - name: "{self.name}"\n') + lines.append(" " * level + f" file_id: 0x{self.file_id:04X}\n") + + if self.flags & int(GCRARCFlags.IS_COMPRESSED) != 0x00: + if self.flags & int(GCRARCFlags.IS_YAZ0_COMPRESSED) != 0x00: + lines.append(" " * level + f" compression: yaz0\n") + else: + lines.append(" " * level + f" compression: yay0\n") + + if self.flags & int(GCRARCFlags.PRELOAD_TO_MRAM) == 0x00: + if self.flags & int(GCRARCFlags.PRELOAD_TO_ARAM) != 0x00: + lines.append(" " * level + f" preload_type: aram\n") + else: + lines.append(" " * level + f" preload_type: dvd\n") + + return lines + + def get_full_file_path(self): + path_components = [self.name] + + node = self.parent_node + while node is not None: + path_components.insert(0, node.name) + node = node.parent + + return Path("/".join(path_components)) + + +class GCRARCFlags(IntEnum): + IS_FILE = 0x01 + IS_DIR = 0x02 + IS_COMPRESSED = 0x04 + PRELOAD_TO_MRAM = 0x10 + PRELOAD_TO_ARAM = 0x20 + LOAD_FROM_DVD = 0x40 + IS_YAZ0_COMPRESSED = 0x80 diff --git a/tools/splat/segtypes/gc/relheader.py b/tools/splat/segtypes/gc/relheader.py new file mode 100644 index 0000000..e92fc63 --- /dev/null +++ b/tools/splat/segtypes/gc/relheader.py @@ -0,0 +1,114 @@ +from util import options + +from segtypes.common.header import CommonSegHeader + + +class RelSegHeader(CommonSegHeader): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if isinstance(self.yaml, dict): + self.version: int = self.yaml.get("version", 0) + + def parse_header(self, rel_bytes): + header_lines = [] + header_lines.append(".section .data\n") + + # Module ID + header_lines.append(self.get_line("word", rel_bytes[0x00:0x04], "Module ID")) + + # Next module (filled at runtime) + header_lines.append(self.get_line("word", rel_bytes[0x04:0x08], "Next Module")) + # Last module (filled at runtime) + header_lines.append(self.get_line("word", rel_bytes[0x08:0x0C], "Last Module")) + + # Section count + header_lines.append( + self.get_line("word", rel_bytes[0x0C:0x10], "Section Count") + ) + # Section table offset + header_lines.append( + self.get_line("word", rel_bytes[0x10:0x14], "Section Table Offset") + ) + + # Module name offset (might be null) + header_lines.append( + self.get_line("word", rel_bytes[0x14:0x18], "Module Name Offset") + ) + # Module name length + header_lines.append( + self.get_line("word", rel_bytes[0x18:0x1C], "Module Name Length") + ) + + # REL format version + header_lines.append( + self.get_line("word", rel_bytes[0x1C:0x20], "REL Format Version") + ) + + # BSS size + header_lines.append(self.get_line("word", rel_bytes[0x20:0x24], "BSS Size")) + + # Relocation table offset + header_lines.append( + self.get_line("word", rel_bytes[0x24:0x28], "Relocation Table Offset") + ) + # Import table offset + header_lines.append( + self.get_line("word", rel_bytes[0x28:0x2C], "Import Table Offset") + ) + # Import table size + header_lines.append( + self.get_line("word", rel_bytes[0x2C:0x30], "Import Table Size") + ) + + # Prolog section index + header_lines.append( + self.get_line("byte", rel_bytes[0x30:0x31], "Prolog Section Index") + ) + # Epilog section index + header_lines.append( + self.get_line("byte", rel_bytes[0x31:0x32], "Epilog Section Index") + ) + # Unresolved section index + header_lines.append( + self.get_line("byte", rel_bytes[0x32:0x33], "Unresolved Section Index") + ) + # BSS section index (filled at runtime) + header_lines.append( + self.get_line("byte", rel_bytes[0x33:0x34], "BSS Section Index") + ) + + # Prolog function offset + header_lines.append( + self.get_line("word", rel_bytes[0x34:0x38], "Prolog Function Offset") + ) + # Epilog function offset + header_lines.append( + self.get_line("word", rel_bytes[0x38:0x3C], "Epilog Function Offset") + ) + # Unresolved function offset + header_lines.append( + self.get_line("word", rel_bytes[0x3C:0x40], "Unresolved Function Offset") + ) + + # Version 1 is only 0x40 bytes long + if self.version <= 1: + return header_lines + + # Alignment constraint + header_lines.append( + self.get_line("word", rel_bytes[0x40:0x44], "Alignment Constraint") + ) + # BSS alignment constraint + header_lines.append( + self.get_line("word", rel_bytes[0x44:0x48], "BSS Alignment Constraint") + ) + + # Version 2 is only 0x48 bytes long + if self.version <= 2: + return header_lines + + # Fix size + header_lines.append(self.get_line("word", rel_bytes[0x48:0x4C], "Fix Size")) + + return header_lines diff --git a/tools/splat/segtypes/gc/segment.py b/tools/splat/segtypes/gc/segment.py new file mode 100644 index 0000000..8e4d9bc --- /dev/null +++ b/tools/splat/segtypes/gc/segment.py @@ -0,0 +1,5 @@ +from segtypes.segment import Segment + + +class GCSegment(Segment): + pass diff --git a/tools/splat/segtypes/linker_entry.py b/tools/splat/segtypes/linker_entry.py new file mode 100644 index 0000000..3750ae6 --- /dev/null +++ b/tools/splat/segtypes/linker_entry.py @@ -0,0 +1,399 @@ +import os +import re +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Dict, List, OrderedDict, Set, Tuple, Union +from segtypes.n64.palette import N64SegPalette + +from util import options + +from segtypes.segment import Segment +from util.symbols import to_cname + + +# clean 'foo/../bar' to 'bar' +@lru_cache(maxsize=None) +def clean_up_path(path: Path) -> Path: + path_resolved = path.resolve() + base_resolved = options.opts.base_path.resolve() + try: + return path_resolved.relative_to(base_resolved) + except ValueError: + pass + + # If the path wasn't relative to the splat file, use the working directory instead + cwd = Path(os.getcwd()) + try: + return path_resolved.relative_to(cwd) + except ValueError: + pass + + # If it wasn't relative to that too, then just return the path as-is + return path + + +def path_to_object_path(path: Path) -> Path: + path = clean_up_path(path) + if options.opts.use_o_as_suffix: + full_suffix = ".o" + else: + full_suffix = path.suffix + ".o" + return clean_up_path(options.opts.build_path / path.with_suffix(full_suffix)) + + +def write_file_if_different(path: Path, new_content: str): + if path.exists(): + old_content = path.read_text() + else: + old_content = "" + + if old_content != new_content: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w") as f: + f.write(new_content) + + +def segment_cname(segment: Segment) -> str: + name = segment.name + if segment.parent: + name = segment.parent.name + "_" + name + + if isinstance(segment, N64SegPalette): + name += "_pal" + + return to_cname(name) + + +def get_segment_vram_end_symbol_name(segment: Segment) -> str: + return segment_cname(segment) + "_VRAM_END" + + +@dataclass +class LinkerSection: + name: str + started: bool = False + ended: bool = False + + @property + def section_type(self) -> str: + if self.name == ".rdata": + return ".rodata" + return self.name + + +class LinkerEntry: + def __init__( + self, segment: Segment, src_paths: List[Path], object_path: Path, section: str + ): + self.segment = segment + self.src_paths = [clean_up_path(p) for p in src_paths] + self.section = section + if self.section == "linker" or self.section == "linker_offset": + self.object_path = None + elif self.segment.type == "lib": + self.object_path = object_path + else: + self.object_path = path_to_object_path(object_path) + + @property + def section_type(self) -> str: + if self.section == ".rdata": + return ".rodata" + else: + return self.section + + +class LinkerWriter: + def __init__(self): + self.linker_discard_section: bool = options.opts.ld_discard_section + # Used to store all the linker entries - build tools may want this information + self.entries: List[LinkerEntry] = [] + + self.buffer: List[str] = [] + self.symbols: List[str] = [] + + self._indent_level = 0 + + self._writeln("SECTIONS") + self._begin_block() + self._writeln("__romPos = 0;") + + if options.opts.gp is not None: + self._writeln("_gp = " + f"0x{options.opts.gp:X};") + + # Write a series of statements which compute a symbol that represents the highest address among a list of segments' end addresses + def write_max_vram_end_sym(self, symbol: str, overlays: List[Segment]): + for segment in overlays: + if segment == overlays[0]: + self._writeln( + f"{symbol} = {get_segment_vram_end_symbol_name(segment)};" + ) + else: + self._writeln( + f"{symbol} = MAX({symbol}, {get_segment_vram_end_symbol_name(segment)});" + ) + + # Adds all the entries of a segment to the linker script buffer + def add(self, segment: Segment, max_vram_syms: List[Tuple[str, List[Segment]]]): + entries = segment.get_linker_entries() + self.entries.extend(entries) + + seg_name = segment_cname(segment) + + for sym, segs in max_vram_syms: + self.write_max_vram_end_sym(sym, segs) + + section_labels: OrderedDict[str, LinkerSection] = OrderedDict( + { + l: LinkerSection(l) + for l in options.opts.section_order + if l in options.opts.ld_section_labels + } + ) + + # Start the first linker section + + self._write_symbol(f"{seg_name}_ROM_START", "__romPos") + + if entries[0].section_type == ".bss": + self._begin_bss_segment(segment, is_first=True) + self._write_symbol(f"{seg_name}_BSS_START", ".") + if ".bss" in section_labels: + section_labels[".bss"].started = True + else: + self._begin_segment(segment) + + last_seen_sections: Dict[LinkerEntry, str] = {} + + # Find where sections are last seen + for entry in reversed(entries): + if ( + entry.section_type in section_labels.keys() + and entry.section_type not in last_seen_sections.values() + ): + last_seen_sections[entry] = entry.section_type + + cur_section = None + prev_section = None + for entry in entries: + entering_bss = False + leaving_bss = False + cur_section = entry.section_type + + if cur_section == "linker_offset": + self._write_symbol(f"{segment_cname(entry.segment)}_OFFSET", ".") + continue + + for i, section in enumerate(section_labels.values()): + # If we haven't seen this section yet + if not section.started and section.section_type == entry.section_type: + if prev_section == ".bss": + leaving_bss = True + elif cur_section == ".bss": + entering_bss = True + + if not ( + entering_bss or leaving_bss + ): # Don't write a START symbol if we are about to end the section + self._write_symbol( + f"{seg_name}{entry.section_type.upper()}_START", "." + ) + section_labels[entry.section_type].started = True + + if ( + entry.object_path + and cur_section == ".data" + and entry.segment.type != "lib" + ): + path_cname = re.sub( + r"[^0-9a-zA-Z_]", + "_", + str(entry.segment.dir / entry.segment.name) + + ".".join(entry.object_path.suffixes[:-1]), + ) + self._write_symbol(path_cname, ".") + + wildcard = "*" if options.opts.ld_wildcard_sections else "" + + # Create new linker section for BSS + if entering_bss or leaving_bss: + # If this is the last entry of its type, add the END marker for the section we're ending + if ( + entry in last_seen_sections + and section_labels[entry.section_type].started + ): + seg_name_section = to_cname( + f"{seg_name}{last_seen_sections[entry].upper()}" + ) + self._write_symbol(f"{seg_name_section}_END", ".") + self._write_symbol( + f"{seg_name_section}_SIZE", + f"ABSOLUTE({seg_name_section}_END - {seg_name_section}_START)", + ) + section_labels[last_seen_sections[entry]].ended = True + + self._end_block() + + if entering_bss: + self._begin_bss_segment(segment) + else: + self._begin_segment(segment) + + self._write_symbol(f"{seg_name}{entry.section_type.upper()}_START", ".") + section_labels[cur_section].started = True + + # Write THIS linker entry + self._writeln(f"{entry.object_path}({entry.section}{wildcard});") + else: + # Write THIS linker entry + if entry.section == ".bss" and entry.segment.bss_contains_common: + self._writeln(f"{entry.object_path}(.bss COMMON .scommon);") + else: + self._writeln(f"{entry.object_path}({entry.section}{wildcard});") + + # If this is the last entry of its type, add the END marker for the section we're ending + if entry in last_seen_sections: + seg_name_section = to_cname(f"{seg_name}{cur_section.upper()}") + self._write_symbol(f"{seg_name_section}_END", ".") + self._write_symbol( + f"{seg_name_section}_SIZE", + f"ABSOLUTE({seg_name_section}_END - {seg_name_section}_START)", + ) + section_labels[cur_section].ended = True + + prev_section = cur_section + + # End all un-ended sections + for section in section_labels.values(): + if section.started and not section.ended: + seg_name_section = to_cname(f"{seg_name}{section.name.upper()}") + self._write_symbol(f"{seg_name_section}_END", ".") + self._write_symbol( + f"{seg_name_section}_SIZE", + f"ABSOLUTE({seg_name_section}_END - {seg_name_section}_START)", + ) + + all_bss = all(e.section == ".bss" for e in entries) + self._end_segment(segment, all_bss) + + def save_linker_script(self): + if self.linker_discard_section: + self._writeln("/DISCARD/ :") + self._begin_block() + self._writeln("*(*);") + self._end_block() + + self._end_block() # SECTIONS + + assert self._indent_level == 0 + + write_file_if_different( + options.opts.ld_script_path, "\n".join(self.buffer) + "\n" + ) + + def save_symbol_header(self): + path = options.opts.ld_symbol_header_path + + if path: + write_file_if_different( + path, + "#ifndef _HEADER_SYMBOLS_H_\n" + "#define _HEADER_SYMBOLS_H_\n" + "\n" + '#include "common.h"\n' + "\n" + + "".join(f"extern Addr {symbol};\n" for symbol in self.symbols) + + "\n" + "#endif\n", + ) + + def _writeln(self, line: str): + if len(line) == 0: + self.buffer.append(line) + else: + self.buffer.append(" " * self._indent_level + line) + + def _begin_block(self): + self._writeln("{") + self._indent_level += 1 + + def _end_block(self): + self._indent_level -= 1 + self._writeln("}") + + def _write_symbol(self, symbol: str, value: Union[str, int]): + symbol = to_cname(symbol) + + if isinstance(value, int): + value = f"0x{value:X}" + + self._writeln(f"{symbol} = {value};") + + if symbol not in self.symbols: + self.symbols.append(symbol) + + def _begin_segment(self, segment: Segment): + if options.opts.ld_use_follows and segment.vram_of_symbol: + vram_str = segment.vram_of_symbol + " " + else: + vram_str = ( + f"0x{segment.vram_start:X} " + if isinstance(segment.vram_start, int) + else "" + ) + + name = segment_cname(segment) + + self._write_symbol(f"{name}_VRAM", f"ADDR(.{name})") + + line = f".{name} {vram_str}: AT({name}_ROM_START)" + if segment.subalign != None: + line += f" SUBALIGN({segment.subalign})" + + self._writeln(line) + self._begin_block() + + def _begin_bss_segment(self, segment: Segment, is_first: bool = False): + if options.opts.ld_use_follows and segment.vram_of_symbol: + vram_str = segment.vram_of_symbol + " " + else: + vram_str = ( + f"0x{segment.vram_start:X} " + if isinstance(segment.vram_start, int) + else "" + ) + + name = segment_cname(segment) + "_bss" + + self._write_symbol(f"{name}_VRAM", f"ADDR(.{name})") + + if is_first: + addr_str = vram_str + "(NOLOAD)" + else: + addr_str = "(NOLOAD)" + + line = f".{name} {addr_str} :" + if segment.subalign != None: + line += f" SUBALIGN({segment.subalign})" + + self._writeln(line) + self._begin_block() + + def _end_segment(self, segment: Segment, all_bss=False): + self._end_block() + + name = segment_cname(segment) + + if not all_bss: + self._writeln(f"__romPos += SIZEOF(.{name});") + + # Align directive + if segment.align: + self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});") + + self._write_symbol(f"{name}_ROM_END", "__romPos") + + self._write_symbol(get_segment_vram_end_symbol_name(segment), ".") + + self._writeln("") diff --git a/tools/splat/segtypes/n64/__init__.py b/tools/splat/segtypes/n64/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/splat/segtypes/n64/__init__.py diff --git a/tools/splat/segtypes/n64/asm.py b/tools/splat/segtypes/n64/asm.py new file mode 100644 index 0000000..4b93b2c --- /dev/null +++ b/tools/splat/segtypes/n64/asm.py @@ -0,0 +1,28 @@ +from util import options + +from segtypes.common.asm import CommonSegAsm + + +class N64SegAsm(CommonSegAsm): + @staticmethod + def get_file_header(): + ret = [] + + ret.append('.include "macro.inc"') + ret.append("") + ret.append("/* assembler directives */") + ret.append(".set noat /* allow manual use of $at */") + ret.append(".set noreorder /* don't insert nops after branches */") + if options.opts.add_set_gp_64: + ret.append( + ".set gp=64 /* allow use of 64-bit general purpose registers */" + ) + ret.append("") + preamble = options.opts.generated_s_preamble + if preamble: + ret.append(preamble) + ret.append("") + ret.append('.section .text, "ax"') + ret.append("") + + return ret diff --git a/tools/splat/segtypes/n64/ci.py b/tools/splat/segtypes/n64/ci.py new file mode 100644 index 0000000..436908f --- /dev/null +++ b/tools/splat/segtypes/n64/ci.py @@ -0,0 +1,39 @@ +from typing import Optional, TYPE_CHECKING + +from util import log + +from segtypes.n64.img import N64SegImg + +if TYPE_CHECKING: + from segtypes.n64.palette import N64SegPalette + + +# Base class for CI4/CI8 +class N64SegCi(N64SegImg): + def parse_palette_name(self, yaml, args) -> str: + ret = self.name + if isinstance(yaml, dict): + if "palette" in yaml: + ret = yaml["palette"] + elif len(args) > 2: + ret = args[2] + + return ret + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.palette: "Optional[N64SegPalette]" = None + self.palette_name = self.parse_palette_name(self.yaml, self.args) + + def split(self, rom_bytes): + if self.palette is None: + # TODO: output with blank palette + log.error( + f"no palette sibling segment exists\n(hint: add a segment with type 'palette' and name '{self.name}')" + ) + assert self.palette is not None + self.palette.extract = False + self.n64img.palette = self.palette.parse_palette(rom_bytes) + + super().split(rom_bytes) diff --git a/tools/splat/segtypes/n64/ci4.py b/tools/splat/segtypes/n64/ci4.py new file mode 100644 index 0000000..8325881 --- /dev/null +++ b/tools/splat/segtypes/n64/ci4.py @@ -0,0 +1,8 @@ +import n64img.image + +from segtypes.n64.ci import N64SegCi + + +class N64SegCi4(N64SegCi): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, img_cls=n64img.image.CI4) diff --git a/tools/splat/segtypes/n64/ci8.py b/tools/splat/segtypes/n64/ci8.py new file mode 100644 index 0000000..dbf66cd --- /dev/null +++ b/tools/splat/segtypes/n64/ci8.py @@ -0,0 +1,8 @@ +import n64img.image + +from segtypes.n64.ci import N64SegCi + + +class N64SegCi8(N64SegCi): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, img_cls=n64img.image.CI8) diff --git a/tools/splat/segtypes/n64/gfx.py b/tools/splat/segtypes/n64/gfx.py new file mode 100644 index 0000000..62d0965 --- /dev/null +++ b/tools/splat/segtypes/n64/gfx.py @@ -0,0 +1,260 @@ +""" +N64 f3dex display list splitter +Dumps out Gfx[] as a .inc.c file. +""" + +import re +from typing import Optional + +from pathlib import Path + +from pygfxd import ( + gfxd_buffer_to_string, + gfxd_cimg_callback, + gfxd_dl_callback, + gfxd_endian, + gfxd_execute, + gfxd_input_buffer, + gfxd_light_callback, + gfxd_lookat_callback, + gfxd_macro_dflt, + gfxd_macro_fn, + gfxd_mtx_callback, + gfxd_output_buffer, + gfxd_printf, + gfxd_puts, + gfxd_target, + gfxd_timg_callback, + gfxd_tlut_callback, + gfxd_vp_callback, + gfxd_vtx_callback, + gfxd_zimg_callback, + GfxdEndian, + gfxd_f3d, + gfxd_f3db, + gfxd_f3dex, + gfxd_f3dexb, + gfxd_f3dex2, +) + +from util import log, options +from util.log import error + +from segtypes.common.codesubsegment import CommonSegCodeSubsegment + +LIGHTS_RE = re.compile(r"\*\(Lightsn \*\)0x[0-9A-F]{8}") + + +class N64SegGfx(CommonSegCodeSubsegment): + def __init__( + self, + rom_start: Optional[int], + rom_end: Optional[int], + type: str, + name: str, + vram_start: Optional[int], + args: list, + yaml, + ): + super().__init__( + rom_start, + rom_end, + type, + name, + vram_start, + args=args, + yaml=yaml, + ) + self.file_text = None + self.data_only = isinstance(yaml, dict) and yaml.get("data_only", False) + + def format_sym_name(self, sym) -> str: + return sym.name + + def get_linker_section(self) -> str: + return ".data" + + def out_path(self) -> Path: + return options.opts.asset_path / self.dir / f"{self.name}.gfx.inc.c" + + def scan(self, rom_bytes: bytes): + self.file_text = self.disassemble_data(rom_bytes) + + def get_gfxd_target(self): + opt = options.opts.gfx_ucode + + if opt == "f3d": + return gfxd_f3d + elif opt == "f3db": + return gfxd_f3db + elif opt == "f3dex": + return gfxd_f3dex + elif opt == "f3dexb": + return gfxd_f3dexb + elif opt == "f3dex2": + return gfxd_f3dex2 + else: + log.error(f"Unknown target {opt}") + + def tlut_handler(self, addr, idx, count): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(self.format_sym_name(sym)) + return 1 + + def timg_handler(self, addr, fmt, size, width, height, pal): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(self.format_sym_name(sym)) + return 1 + + def cimg_handler(self, addr, fmt, size, width): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(self.format_sym_name(sym)) + return 1 + + def zimg_handler(self, addr): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(self.format_sym_name(sym)) + return 1 + + def dl_handler(self, addr): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(self.format_sym_name(sym)) + return 1 + + def mtx_handler(self, addr): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(f"&{self.format_sym_name(sym)}") + return 1 + + def lookat_handler(self, addr, count): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(self.format_sym_name(sym)) + return 1 + + def light_handler(self, addr, count): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(self.format_sym_name(sym)) + return 1 + + def vtx_handler(self, addr, count): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True, search_ranges=True + ) + index = int((addr - sym.vram_start) / 0x10) + gfxd_printf(f"&{self.format_sym_name(sym)}[{index}]") + return 1 + + def vp_handler(self, addr): + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + gfxd_printf(self.format_sym_name(sym)) + return 1 + + def macro_fn(self): + gfxd_puts(" ") + gfxd_macro_dflt() + gfxd_puts(",\n") + return 0 + + def disassemble_data(self, rom_bytes): + assert isinstance(self.rom_start, int) + assert isinstance(self.rom_end, int) + assert isinstance(self.vram_start, int) + + gfx_data = rom_bytes[self.rom_start : self.rom_end] + segment_length = len(gfx_data) + if (segment_length) % 8 != 0: + error( + f"Error: gfx segment {self.name} length ({segment_length}) is not a multiple of 8!" + ) + + out_str = "" if self.data_only else options.opts.generated_c_preamble + "\n\n" + + sym = self.create_symbol( + addr=self.vram_start, in_segment=True, type="data", define=True + ) + + gfxd_input_buffer(gfx_data) + + # TODO terrible guess at the size we'll need - improve this + outb = bytes([0] * segment_length * 100) + outbuf = gfxd_output_buffer(outb, len(outb)) + + gfxd_target(self.get_gfxd_target()) + gfxd_endian( + GfxdEndian.big if options.opts.endianness == "big" else GfxdEndian.little, 4 + ) + + # Callbacks + gfxd_macro_fn(self.macro_fn) + + gfxd_tlut_callback(self.tlut_handler) + gfxd_timg_callback(self.timg_handler) + gfxd_cimg_callback(self.cimg_handler) + gfxd_zimg_callback(self.zimg_handler) + gfxd_dl_callback(self.dl_handler) + gfxd_mtx_callback(self.mtx_handler) + gfxd_lookat_callback(self.lookat_handler) + gfxd_light_callback(self.light_handler) + # gfxd_seg_callback ? + gfxd_vtx_callback(self.vtx_handler) + gfxd_vp_callback(self.vp_handler) + # gfxd_uctext_callback ? + # gfxd_ucdata_callback ? + # gfxd_dram_callback ? + + gfxd_execute() + + if self.data_only: + out_str += gfxd_buffer_to_string(outbuf) + else: + out_str += "Gfx " + self.format_sym_name(sym) + "[] = {\n" + out_str += gfxd_buffer_to_string(outbuf) + out_str += "};\n" + + # Poor man's light fix until we get my libgfxd PR merged + def light_sub_func(match): + light = match.group(0) + addr = int(light[12:], 0) + sym = self.create_symbol( + addr=addr, in_segment=True, type="data", reference=True + ) + return self.format_sym_name(sym) + + out_str = re.sub(LIGHTS_RE, light_sub_func, out_str) + + return out_str + + def split(self, rom_bytes: bytes): + if self.file_text and self.out_path(): + self.out_path().parent.mkdir(parents=True, exist_ok=True) + + with open(self.out_path(), "w", newline="\n") as f: + f.write(self.file_text) + + def should_scan(self) -> bool: + return ( + options.opts.is_mode_active("gfx") + and self.rom_start is not None + and self.rom_end is not None + ) + + def should_split(self) -> bool: + return self.extract and options.opts.is_mode_active("gfx") diff --git a/tools/splat/segtypes/n64/hasm.py b/tools/splat/segtypes/n64/hasm.py new file mode 100644 index 0000000..18def92 --- /dev/null +++ b/tools/splat/segtypes/n64/hasm.py @@ -0,0 +1,28 @@ +from util import options + +from segtypes.common.hasm import CommonSegHasm + + +class N64SegHasm(CommonSegHasm): + @staticmethod + def get_file_header(): + ret = [] + + ret.append('.include "macro.inc"') + ret.append("") + ret.append("/* assembler directives */") + ret.append(".set noat /* allow manual use of $at */") + ret.append(".set noreorder /* don't insert nops after branches */") + if options.opts.add_set_gp_64: + ret.append( + ".set gp=64 /* allow use of 64-bit general purpose registers */" + ) + ret.append("") + preamble = options.opts.generated_s_preamble + if preamble: + ret.append(preamble) + ret.append("") + ret.append('.section .text, "ax"') + ret.append("") + + return ret diff --git a/tools/splat/segtypes/n64/header.py b/tools/splat/segtypes/n64/header.py new file mode 100644 index 0000000..96c59d5 --- /dev/null +++ b/tools/splat/segtypes/n64/header.py @@ -0,0 +1,50 @@ +from util import options + +from segtypes.common.header import CommonSegHeader + + +class N64SegHeader(CommonSegHeader): + def parse_header(self, rom_bytes): + encoding = options.opts.header_encoding + + header_lines = [] + header_lines.append(".section .data\n") + header_lines.append( + self.get_line("word", rom_bytes[0x00:0x04], "PI BSB Domain 1 register") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x04:0x08], "Clockrate setting") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x08:0x0C], "Entrypoint address") + ) + header_lines.append(self.get_line("word", rom_bytes[0x0C:0x10], "Revision")) + header_lines.append(self.get_line("word", rom_bytes[0x10:0x14], "Checksum 1")) + header_lines.append(self.get_line("word", rom_bytes[0x14:0x18], "Checksum 2")) + header_lines.append(self.get_line("word", rom_bytes[0x18:0x1C], "Unknown 1")) + header_lines.append(self.get_line("word", rom_bytes[0x1C:0x20], "Unknown 2")) + + if encoding != "word": + header_lines.append( + '.ascii "' + + rom_bytes[0x20:0x34].decode(encoding).strip().ljust(20) + + '" /* Internal name */' + ) + else: + for i in range(0x20, 0x34, 4): + header_lines.append( + self.get_line("word", rom_bytes[i : i + 4], "Internal name") + ) + + header_lines.append(self.get_line("word", rom_bytes[0x34:0x38], "Unknown 3")) + header_lines.append(self.get_line("word", rom_bytes[0x38:0x3C], "Cartridge")) + header_lines.append( + self.get_line("ascii", rom_bytes[0x3C:0x3E], "Cartridge ID") + ) + header_lines.append( + self.get_line("ascii", rom_bytes[0x3E:0x3F], "Country code") + ) + header_lines.append(self.get_line("byte", rom_bytes[0x3F:0x40], "Version")) + header_lines.append("") + + return header_lines diff --git a/tools/splat/segtypes/n64/i1.py b/tools/splat/segtypes/n64/i1.py new file mode 100644 index 0000000..c293099 --- /dev/null +++ b/tools/splat/segtypes/n64/i1.py @@ -0,0 +1,9 @@ +import n64img.image
+
+from segtypes.n64.img import N64SegImg
+
+
+class N64SegI1(N64SegImg):
+ def __init__(self, *args, **kwargs):
+ kwargs["img_cls"] = n64img.image.I1
+ super().__init__(*args, **kwargs)
diff --git a/tools/splat/segtypes/n64/i4.py b/tools/splat/segtypes/n64/i4.py new file mode 100644 index 0000000..16a68c5 --- /dev/null +++ b/tools/splat/segtypes/n64/i4.py @@ -0,0 +1,9 @@ +import n64img.image + +from segtypes.n64.img import N64SegImg + + +class N64SegI4(N64SegImg): + def __init__(self, *args, **kwargs): + kwargs["img_cls"] = n64img.image.I4 + super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/i8.py b/tools/splat/segtypes/n64/i8.py new file mode 100644 index 0000000..c2364f2 --- /dev/null +++ b/tools/splat/segtypes/n64/i8.py @@ -0,0 +1,9 @@ +import n64img.image + +from segtypes.n64.img import N64SegImg + + +class N64SegI8(N64SegImg): + def __init__(self, *args, **kwargs): + kwargs["img_cls"] = n64img.image.I8 + super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia16.py b/tools/splat/segtypes/n64/ia16.py new file mode 100644 index 0000000..f288472 --- /dev/null +++ b/tools/splat/segtypes/n64/ia16.py @@ -0,0 +1,9 @@ +import n64img.image + +from segtypes.n64.img import N64SegImg + + +class N64SegIa16(N64SegImg): + def __init__(self, *args, **kwargs): + kwargs["img_cls"] = n64img.image.IA16 + super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia4.py b/tools/splat/segtypes/n64/ia4.py new file mode 100644 index 0000000..0eef15b --- /dev/null +++ b/tools/splat/segtypes/n64/ia4.py @@ -0,0 +1,9 @@ +import n64img.image + +from segtypes.n64.img import N64SegImg + + +class N64SegIa4(N64SegImg): + def __init__(self, *args, **kwargs): + kwargs["img_cls"] = n64img.image.IA4 + super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia8.py b/tools/splat/segtypes/n64/ia8.py new file mode 100644 index 0000000..186be5e --- /dev/null +++ b/tools/splat/segtypes/n64/ia8.py @@ -0,0 +1,9 @@ +import n64img.image + +from segtypes.n64.img import N64SegImg + + +class N64SegIa8(N64SegImg): + def __init__(self, *args, **kwargs): + kwargs["img_cls"] = n64img.image.IA8 + super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/img.py b/tools/splat/segtypes/n64/img.py new file mode 100644 index 0000000..7c4bf80 --- /dev/null +++ b/tools/splat/segtypes/n64/img.py @@ -0,0 +1,83 @@ +from pathlib import Path +from typing import Type, Optional + +from n64img.image import Image +from util import log, options + +from segtypes.n64.segment import N64Segment + + +class N64SegImg(N64Segment): + def __init__( + self, + rom_start: Optional[int], + rom_end: Optional[int], + type: str, + name: str, + vram_start: Optional[int], + args: list, + yaml, + img_cls: Type[Image], + ): + super().__init__( + rom_start, + rom_end, + type, + name, + vram_start, + args=args, + yaml=yaml, + ) + + self.n64img: Image = img_cls(None, 0, 0) + + if isinstance(yaml, dict): + if self.extract: + self.width = yaml["width"] + self.height = yaml["height"] + + self.n64img.flip_h = bool(yaml.get("flip_x", False)) + self.n64img.flip_v = bool(yaml.get("flip_y", False)) + else: + if self.extract: + if len(yaml) < 5: + log.error( + f"Error: {self.name} is missing width and height parameters" + ) + self.width = yaml[3] + self.height = yaml[4] + + self.n64img.width = self.width + self.n64img.height = self.height + + self.check_len() + + def check_len(self) -> None: + if self.extract: + expected_len = int(self.n64img.size()) + assert isinstance(self.rom_start, int) + assert isinstance(self.rom_end, int) + assert isinstance(self.subalign, int) + actual_len = self.rom_end - self.rom_start + if actual_len > expected_len and actual_len - expected_len > self.subalign: + log.error( + f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)" + ) + + def out_path(self) -> Path: + return options.opts.asset_path / self.dir / f"{self.name}.png" + + def should_split(self) -> bool: + return self.extract and options.opts.is_mode_active("img") + + def split(self, rom_bytes): + path = self.out_path() + path.parent.mkdir(parents=True, exist_ok=True) + + assert isinstance(self.rom_start, int) + assert isinstance(self.rom_end, int) + + self.n64img.data = rom_bytes[self.rom_start : self.rom_end] + self.n64img.write(path) + + self.log(f"Wrote {self.name} to {path}") diff --git a/tools/splat/segtypes/n64/ipl3.py b/tools/splat/segtypes/n64/ipl3.py new file mode 100644 index 0000000..6a76cb3 --- /dev/null +++ b/tools/splat/segtypes/n64/ipl3.py @@ -0,0 +1,9 @@ +from segtypes.common.code import CommonSegCode +from segtypes.common.hasm import CommonSegHasm + + +class N64SegIpl3(CommonSegHasm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.special_vram_segment = True diff --git a/tools/splat/segtypes/n64/linker_offset.py b/tools/splat/segtypes/n64/linker_offset.py new file mode 100644 index 0000000..ac4d945 --- /dev/null +++ b/tools/splat/segtypes/n64/linker_offset.py @@ -0,0 +1,10 @@ +from pathlib import Path + +from segtypes.n64.segment import N64Segment + + +class N64SegLinker_offset(N64Segment): + def get_linker_entries(self): + from segtypes.linker_entry import LinkerEntry + + return [LinkerEntry(self, [], Path(self.name), "linker_offset")] diff --git a/tools/splat/segtypes/n64/mio0.py b/tools/splat/segtypes/n64/mio0.py new file mode 100644 index 0000000..5ad4635 --- /dev/null +++ b/tools/splat/segtypes/n64/mio0.py @@ -0,0 +1,10 @@ +from util.n64.Mio0decompress import Mio0Decompressor + +from segtypes.common.decompressor import CommonSegDecompressor + + +class N64SegMio0(CommonSegDecompressor): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.decompressor = Mio0Decompressor() + self.compression_type = "Mio0" diff --git a/tools/splat/segtypes/n64/palette.py b/tools/splat/segtypes/n64/palette.py new file mode 100644 index 0000000..eabd472 --- /dev/null +++ b/tools/splat/segtypes/n64/palette.py @@ -0,0 +1,99 @@ +from itertools import zip_longest +from pathlib import Path +from typing import List, Optional, Tuple, TYPE_CHECKING + +from util import log, options +from util.color import unpack_color + +from segtypes.n64.segment import N64Segment + +if TYPE_CHECKING: + from segtypes.n64.ci import N64SegCi as Raster + + +def iter_in_groups(iterable, n, fillvalue=None): + args = [iter(iterable)] * n + return zip_longest(*args, fillvalue=fillvalue) + + +class N64SegPalette(N64Segment): + require_unique_name = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.raster: "Optional[Raster]" = None + + # palette segments must be named as one of the following: + # 1) same as the relevant raster segment name (max. 1 palette) + # 2) relevant raster segment name + "." + unique palette name + # 3) unique, referencing the relevant raster segment using `raster_name` + self.raster_name = ( + self.yaml.get("raster_name", self.name.split(".")[0]) + if isinstance(self.yaml, dict) + else self.name.split(".")[0] + ) + + if self.extract: + if self.rom_end is None: + log.error( + f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" + ) + + if self.max_length() and isinstance(self.rom_end, int): + expected_len = int(self.max_length()) + assert isinstance(self.rom_end, int) + assert isinstance(self.rom_start, int) + assert isinstance(self.subalign, int) + actual_len = self.rom_end - self.rom_start + if ( + actual_len > expected_len + and actual_len - expected_len > self.subalign + ): + log.error( + f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)" + ) + + def split(self, rom_bytes): + if self.raster is None: + # TODO: output with no raster + log.error(f"orphaned palette segment: {self.name} lacks ci4/ci8 sibling") + + assert self.raster is not None + self.raster.n64img.palette = self.parse_palette(rom_bytes) # type: ignore + + self.raster.n64img.write(self.out_path()) + self.raster.extract = False + + def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]: + assert isinstance(self.rom_start, int) + assert isinstance(self.rom_end, int) + + data = rom_bytes[self.rom_start : self.rom_end] + palette = [] + + for a, b in iter_in_groups(data, 2): + palette.append(unpack_color([a, b])) + + return palette + + def max_length(self): + return 256 * 2 + + def out_path(self) -> Path: + return options.opts.asset_path / self.dir / f"{self.name}.png" + + def should_split(self) -> bool: + return self.extract and options.opts.is_mode_active("img") + + def get_linker_entries(self): + from segtypes.linker_entry import LinkerEntry + + return [ + LinkerEntry( + self, + [options.opts.asset_path / self.dir / f"{self.name}.png"], + options.opts.asset_path / self.dir / f"{self.name}.pal", + self.get_linker_section(), + ) + ] diff --git a/tools/splat/segtypes/n64/rgba16.py b/tools/splat/segtypes/n64/rgba16.py new file mode 100644 index 0000000..4b6f4fd --- /dev/null +++ b/tools/splat/segtypes/n64/rgba16.py @@ -0,0 +1,9 @@ +import n64img.image + +from segtypes.n64.img import N64SegImg + + +class N64SegRgba16(N64SegImg): + def __init__(self, *args, **kwargs): + kwargs["img_cls"] = n64img.image.RGBA16 + super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/rgba32.py b/tools/splat/segtypes/n64/rgba32.py new file mode 100644 index 0000000..ae3d538 --- /dev/null +++ b/tools/splat/segtypes/n64/rgba32.py @@ -0,0 +1,9 @@ +import n64img.image + +from segtypes.n64.img import N64SegImg + + +class N64SegRgba32(N64SegImg): + def __init__(self, *args, **kwargs): + kwargs["img_cls"] = n64img.image.RGBA32 + super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/rsp.py b/tools/splat/segtypes/n64/rsp.py new file mode 100644 index 0000000..f69123a --- /dev/null +++ b/tools/splat/segtypes/n64/rsp.py @@ -0,0 +1,10 @@ +import rabbitizer + +from segtypes.common.hasm import CommonSegHasm + + +class N64SegRsp(CommonSegHasm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.instr_category = rabbitizer.InstrCategory.RSP diff --git a/tools/splat/segtypes/n64/segment.py b/tools/splat/segtypes/n64/segment.py new file mode 100644 index 0000000..8f9c3cc --- /dev/null +++ b/tools/splat/segtypes/n64/segment.py @@ -0,0 +1,5 @@ +from segtypes.segment import Segment + + +class N64Segment(Segment): + pass diff --git a/tools/splat/segtypes/n64/vtx.py b/tools/splat/segtypes/n64/vtx.py new file mode 100644 index 0000000..8d961b2 --- /dev/null +++ b/tools/splat/segtypes/n64/vtx.py @@ -0,0 +1,106 @@ +""" +N64 Vtx struct splitter +Dumps out Vtx as a .inc.c file. + +Originally written by Mark Street (https://github.com/mkst) +""" + +import struct +from pathlib import Path +from typing import Optional + +from util import options, log + +from segtypes.common.codesubsegment import CommonSegCodeSubsegment + + +class N64SegVtx(CommonSegCodeSubsegment): + def __init__( + self, + rom_start: Optional[int], + rom_end: Optional[int], + type: str, + name: str, + vram_start: Optional[int], + args: list, + yaml, + ): + super().__init__( + rom_start, + rom_end, + type, + name, + vram_start, + args=args, + yaml=yaml, + ) + self.file_text = None + self.data_only = isinstance(yaml, dict) and yaml.get("data_only", False) + + def format_sym_name(self, sym) -> str: + return sym.name + + def get_linker_section(self) -> str: + return ".data" + + def out_path(self) -> Path: + return options.opts.asset_path / self.dir / f"{self.name}.vtx.inc.c" + + def scan(self, rom_bytes: bytes): + self.file_text = self.disassemble_data(rom_bytes) + + def disassemble_data(self, rom_bytes): + assert isinstance(self.rom_start, int) + assert isinstance(self.rom_end, int) + assert isinstance(self.vram_start, int) + + vertex_data = rom_bytes[self.rom_start : self.rom_end] + segment_length = len(vertex_data) + if (segment_length) % 16 != 0: + log.error( + f"Error: Vtx segment {self.name} length ({segment_length}) is not a multiple of 16!" + ) + + lines = [] + if not self.data_only: + lines.append(options.opts.generated_c_preamble) + lines.append("") + + vertex_count = segment_length // 16 + sym = self.create_symbol( + addr=self.vram_start, in_segment=True, type="data", define=True + ) + + if not self.data_only: + lines.append(f"Vtx {self.format_sym_name(sym)}[{vertex_count}] = {{") + + for vtx in struct.iter_unpack(">hhhHhhBBBB", vertex_data): + x, y, z, flg, t, c, r, g, b, a = vtx + vtx_string = f" {{{{{{ {x:5}, {y:5}, {z:5} }}, {flg}, {{ {t:5}, {c:5} }}, {{ {r:3}, {g:3}, {b:3}, {a:3} }}}}}}," + if flg != 0: + self.warn(f"Non-zero flag found in vertex data {self.name}!") + lines.append(vtx_string) + + if not self.data_only: + lines.append("};") + + # enforce newline at end of file + lines.append("") + return "\n".join(lines) + + def split(self, rom_bytes: bytes): + if self.file_text and self.out_path(): + self.out_path().parent.mkdir(parents=True, exist_ok=True) + + with open(self.out_path(), "w", newline="\n") as f: + f.write(self.file_text) + + def should_scan(self) -> bool: + return ( + options.opts.is_mode_active("vtx") + and self.rom_start is not None + and self.rom_end is not None + ) + + def should_split(self) -> bool: + return self.extract and options.opts.is_mode_active("vtx") diff --git a/tools/splat/segtypes/n64/yay0.py b/tools/splat/segtypes/n64/yay0.py new file mode 100644 index 0000000..210f761 --- /dev/null +++ b/tools/splat/segtypes/n64/yay0.py @@ -0,0 +1,10 @@ +from util.n64.Yay0decompress import Yay0Decompressor + +from segtypes.common.decompressor import CommonSegDecompressor + + +class N64SegYay0(CommonSegDecompressor): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.decompressor = Yay0Decompressor() + self.compression_type = "Yay0" diff --git a/tools/splat/segtypes/psx/__init__.py b/tools/splat/segtypes/psx/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/splat/segtypes/psx/__init__.py diff --git a/tools/splat/segtypes/psx/asm.py b/tools/splat/segtypes/psx/asm.py new file mode 100644 index 0000000..2e9ff5e --- /dev/null +++ b/tools/splat/segtypes/psx/asm.py @@ -0,0 +1,23 @@ +from util import options + +from segtypes.common.asm import CommonSegAsm + + +class PsxSegAsm(CommonSegAsm): + @staticmethod + def get_file_header(): + ret = [] + + ret.append('.include "macro.inc"') + ret.append("") + ret.append(".set noat") + ret.append(".set noreorder") + ret.append("") + preamble = options.opts.generated_s_preamble + if preamble: + ret.append(preamble) + ret.append("") + ret.append('.section .text, "ax"') + ret.append("") + + return ret diff --git a/tools/splat/segtypes/psx/header.py b/tools/splat/segtypes/psx/header.py new file mode 100644 index 0000000..aeda3ee --- /dev/null +++ b/tools/splat/segtypes/psx/header.py @@ -0,0 +1,61 @@ +from segtypes.common.header import CommonSegHeader + + +class PsxSegHeader(CommonSegHeader): + # little endian so reverse words, TODO: use struct.unpack("<i",...) ? + # breakdown from https://psx-spx.consoledev.net/cdromdrive/#filenameexe-general-purpose-executable + def parse_header(self, rom_bytes): + header_lines = [] + header_lines.append(".section .data\n") + header_lines.append( + self.get_line("ascii", rom_bytes[0x00:0x08], "Magic number") + ) + header_lines.append(self.get_line("word", rom_bytes[0x08:0x0C], "Zerofilled")) + header_lines.append(self.get_line("word", rom_bytes[0x0C:0x10], "Zerofilled")) + header_lines.append( + self.get_line("word", rom_bytes[0x10:0x14][::-1], "Initial PC") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x14:0x18][::-1], "Initial $gp/r28") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x18:0x1C][::-1], ".text start") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x1C:0x20][::-1], ".text size") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x20:0x24][::-1], ".data start") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x24:0x28][::-1], ".data size") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x28:0x2C][::-1], ".bss start") + ) + header_lines.append( + self.get_line("word", rom_bytes[0x2C:0x30][::-1], ".bss size") + ) + header_lines.append( + self.get_line( + "word", rom_bytes[0x30:0x34][::-1], "Initial $sp/r29 & $fp/r30 base" + ) + ) + header_lines.append( + self.get_line( + "word", rom_bytes[0x34:0x38][::-1], "Initial $sp/r29 & $fp/r30 offset" + ) + ) + header_lines.append(self.get_line("word", rom_bytes[0x38:0x3C], "Reserved")) + header_lines.append(self.get_line("word", rom_bytes[0x3C:0x40], "Reserved")) + header_lines.append(self.get_line("word", rom_bytes[0x40:0x44], "Reserved")) + header_lines.append(self.get_line("word", rom_bytes[0x44:0x48], "Reserved")) + header_lines.append(self.get_line("word", rom_bytes[0x48:0x4C], "Reserved")) + assert isinstance(self.rom_end, int) + header_lines.append( + self.get_line("ascii", rom_bytes[0x4C : self.rom_end], "Sony Inc") + ) + + header_lines.append("") + + return header_lines diff --git a/tools/splat/segtypes/psx/segment.py b/tools/splat/segtypes/psx/segment.py new file mode 100644 index 0000000..eff2875 --- /dev/null +++ b/tools/splat/segtypes/psx/segment.py @@ -0,0 +1,5 @@ +from segtypes.segment import Segment + + +class PsxSegment(Segment): + pass diff --git a/tools/splat/segtypes/segment.py b/tools/splat/segtypes/segment.py new file mode 100644 index 0000000..2650b9d --- /dev/null +++ b/tools/splat/segtypes/segment.py @@ -0,0 +1,615 @@ +import importlib +import importlib.util +from pathlib import Path + +from typing import Dict, List, Optional, Set, Type, TYPE_CHECKING, Union + +from intervaltree import Interval, IntervalTree + +from util import log, options, symbols +from util.symbols import Symbol, to_cname + +# circular import +if TYPE_CHECKING: + from segtypes.linker_entry import LinkerEntry + + +def parse_segment_vram(segment: Union[dict, list]) -> Optional[int]: + if isinstance(segment, dict) and "vram" in segment: + return int(segment["vram"]) + else: + return None + + +def parse_segment_align(segment: Union[dict, list]) -> Optional[int]: + if isinstance(segment, dict) and "align" in segment: + return int(segment["align"]) + return None + + +def parse_segment_subalign(segment: Union[dict, list]) -> int: + default = options.opts.subalign + if isinstance(segment, dict): + subalign = segment.get("subalign", default) + if subalign != None: + subalign = int(subalign) + return subalign + return default + + +def parse_segment_section_order(segment: Union[dict, list]) -> List[str]: + default = options.opts.section_order + if isinstance(segment, dict): + return segment.get("section_order", default) + return default + + +def parse_segment_follows_vram(segment: Union[dict, list]) -> Optional[str]: + if isinstance(segment, dict): + return segment.get("follows_vram", None) + return None + + +def parse_segment_vram_of_symbol(segment: Union[dict, list]) -> Optional[str]: + if isinstance(segment, dict): + return segment.get("vram_of_symbol", segment.get("follows_vram_symbol", None)) + return None + + +class Segment: + require_unique_name = True + + @staticmethod + def get_class_for_type(seg_type) -> Type["Segment"]: + # so .data loads SegData, for example + if seg_type.startswith("."): + seg_type = seg_type[1:] + + segment_class = Segment.get_base_segment_class(seg_type) + if segment_class == None: + # Look in extensions + segment_class = Segment.get_extension_segment_class(seg_type) + return segment_class + + @staticmethod + def get_base_segment_class(seg_type): + platform = options.opts.platform + is_platform_seg = False + + # heirarchy is platform -> common -> fail + try: + segmodule = importlib.import_module(f"segtypes.{platform}.{seg_type}") + is_platform_seg = True + except ModuleNotFoundError: + try: + segmodule = importlib.import_module(f"segtypes.common.{seg_type}") + except ModuleNotFoundError: + return None + + seg_prefix = platform.capitalize() if is_platform_seg else "Common" + return getattr(segmodule, f"{seg_prefix}Seg{seg_type.capitalize()}") + + @staticmethod + def get_extension_segment_class(seg_type): + platform = options.opts.platform + + ext_path = options.opts.extensions_path + if not ext_path: + log.error( + f"could not load presumed extended segment type '{seg_type}' because no extensions path is configured" + ) + assert ext_path is not None + + try: + ext_spec = importlib.util.spec_from_file_location( + f"{platform}.segtypes.{seg_type}", ext_path / f"{seg_type}.py" + ) + assert ext_spec is not None + ext_mod = importlib.util.module_from_spec(ext_spec) + assert ext_spec.loader is not None + ext_spec.loader.exec_module(ext_mod) + except Exception as err: + log.write(err, status="error") + log.error( + f"could not load segment type '{seg_type}'\n(hint: confirm your extension directory is configured correctly)" + ) + + return getattr( + ext_mod, f"{platform.upper()}Seg{seg_type[0].upper()}{seg_type[1:]}" + ) + + @staticmethod + def parse_segment_start(segment: Union[dict, list]) -> Optional[int]: + if isinstance(segment, dict): + s = segment.get("start", "auto") + else: + s = segment[0] + + if s == "auto": + return None + elif s == "...": + return None + else: + return int(s) + + @staticmethod + def parse_segment_type(segment: Union[dict, list]) -> str: + if isinstance(segment, dict): + return str(segment["type"]) + else: + return str(segment[1]) + + @staticmethod + def parse_segment_name(cls, rom_start, segment: Union[dict, list]) -> str: + if isinstance(segment, dict) and "name" in segment: + return str(segment["name"]) + elif isinstance(segment, dict) and "dir" in segment: + return str(segment["dir"]) + elif isinstance(segment, list) and len(segment) >= 3: + return str(segment[2]) + else: + return str(cls.get_default_name(rom_start)) + + @staticmethod + def parse_segment_symbol_name_format(segment: Union[dict, list]) -> str: + if isinstance(segment, dict) and "symbol_name_format" in segment: + return str(segment["symbol_name_format"]) + else: + return options.opts.symbol_name_format + + @staticmethod + def parse_segment_symbol_name_format_no_rom(segment: Union[dict, list]) -> str: + if isinstance(segment, dict) and "symbol_name_format_no_rom" in segment: + return str(segment["symbol_name_format_no_rom"]) + else: + return options.opts.symbol_name_format_no_rom + + @staticmethod + def parse_segment_file_path(segment: Union[dict, list]) -> Optional[Path]: + if isinstance(segment, dict) and "path" in segment: + return Path(segment["path"]) + return None + + @staticmethod + def parse_segment_bss_contains_common(segment: Union[dict, list]) -> bool: + if isinstance(segment, dict) and "bss_contains_common" in segment: + return bool(segment["bss_contains_common"]) + else: + return False + + def __init__( + self, + rom_start: Optional[int], + rom_end: Optional[int], + type: str, + name: str, + vram_start: Optional[int], + args: list, + yaml, + ): + self.rom_start = rom_start + self.rom_end = rom_end + self.type = type + self.name = name + self.vram_start: Optional[int] = vram_start + + self.align: Optional[int] = None + self.given_subalign: int = options.opts.subalign + self.exclusive_ram_id: Optional[str] = None + self.given_dir: Path = Path() + + # Symbols known to be in this segment + self.given_seg_symbols: Dict[int, List[Symbol]] = {} + + # Ranges for faster symbol lookup + self.symbol_ranges_ram: IntervalTree = IntervalTree() + self.symbol_ranges_rom: IntervalTree = IntervalTree() + + self.given_section_order: List[str] = options.opts.section_order + self.given_follows_vram: Optional[str] = None + self.vram_of_symbol: Optional[str] = None + + self.given_symbol_name_format: str = options.opts.symbol_name_format + self.given_symbol_name_format_no_rom: str = ( + options.opts.symbol_name_format_no_rom + ) + + self.parent: Optional[Segment] = None + self.sibling: Optional[Segment] = None + self.rodata_sibling: Optional[Segment] = None + self.file_path: Optional[Path] = None + + self.args: List[str] = args + self.yaml = yaml + + self.extract: bool = True + if self.rom_start is None: + self.extract = False + elif self.type.startswith("."): + self.extract = False + + self.warnings: List[str] = [] + self.did_run = False + self.bss_contains_common = Segment.parse_segment_bss_contains_common(yaml) + + # For segments which are not in the usual VRAM segment space, like N64's IPL3 which lives in 0xA4... + self.special_vram_segment: bool = False + + if isinstance(self.rom_start, int) and isinstance(self.rom_end, int): + if self.rom_start > self.rom_end: + log.error( + f"Error: segments out of order - ({self.name} starts at 0x{self.rom_start:X}, but next segment starts at 0x{self.rom_end:X})" + ) + + @staticmethod + def from_yaml( + cls: Type["Segment"], + yaml: Union[dict, list], + rom_start: Optional[int], + rom_end: Optional[int], + vram=None, + ): + type = Segment.parse_segment_type(yaml) + name = Segment.parse_segment_name(cls, rom_start, yaml) + vram_start = vram if vram is not None else parse_segment_vram(yaml) + + args: List[str] = [] if isinstance(yaml, dict) else yaml[3:] + + ret = cls( + rom_start=rom_start, + rom_end=rom_end, + type=type, + name=name, + vram_start=vram_start, + args=args, + yaml=yaml, + ) + ret.given_section_order = parse_segment_section_order(yaml) + ret.given_subalign = parse_segment_subalign(yaml) + if isinstance(yaml, dict): + ret.extract = bool(yaml.get("extract", ret.extract)) + ret.exclusive_ram_id = yaml.get("exclusive_ram_id") + ret.given_dir = Path(yaml.get("dir", "")) + ret.given_symbol_name_format = Segment.parse_segment_symbol_name_format(yaml) + ret.given_symbol_name_format_no_rom = ( + Segment.parse_segment_symbol_name_format_no_rom(yaml) + ) + ret.file_path = Segment.parse_segment_file_path(yaml) + + ret.bss_contains_common = Segment.parse_segment_bss_contains_common(yaml) + if not ret.given_follows_vram: + ret.given_follows_vram = parse_segment_follows_vram(yaml) + + if not ret.vram_of_symbol: + ret.vram_of_symbol = parse_segment_vram_of_symbol(yaml) + + if not ret.align: + ret.align = parse_segment_align(yaml) + return ret + + # For executable segments (.text); like c, asm or hasm + @staticmethod + def is_text() -> bool: + return False + + # For readonly segments (.rodata); like rodata or rdata + @staticmethod + def is_rodata() -> bool: + return False + + # For segments which does not take space in ROM; like bss + @staticmethod + def is_noload() -> bool: + return False + + @property + def needs_symbols(self) -> bool: + return False + + @property + def dir(self) -> Path: + if self.parent: + return self.parent.dir / self.given_dir + else: + return self.given_dir + + @property + def symbol_name_format(self) -> str: + return self.given_symbol_name_format + + @property + def symbol_name_format_no_rom(self) -> str: + return self.given_symbol_name_format_no_rom + + @property + def subalign(self) -> int: + if self.parent: + return self.parent.subalign + else: + return self.given_subalign + + def get_exclusive_ram_id(self) -> Optional[str]: + if self.parent: + return self.parent.get_exclusive_ram_id() + return self.exclusive_ram_id + + def add_symbol(self, symbol: Symbol): + if symbol.vram_start not in self.given_seg_symbols: + self.given_seg_symbols[symbol.vram_start] = [] + self.given_seg_symbols[symbol.vram_start].append(symbol) + + # For larger symbols, add their ranges to interval trees for faster lookup + if symbol.size > 4: + self.symbol_ranges_ram.addi(symbol.vram_start, symbol.vram_end, symbol) + if symbol.rom and isinstance(symbol.rom, int): + self.symbol_ranges_rom.addi(symbol.rom, symbol.rom_end, symbol) + + @property + def seg_symbols(self) -> Dict[int, List[Symbol]]: + if self.parent: + return self.parent.seg_symbols + else: + return self.given_seg_symbols + + @property + def size(self) -> Optional[int]: + if isinstance(self.rom_start, int) and isinstance(self.rom_end, int): + return self.rom_end - self.rom_start + else: + return None + + @property + def vram_end(self) -> Optional[int]: + if self.vram_start is not None and self.size is not None: + return self.vram_start + self.size + else: + return None + + @property + def section_order(self) -> List[str]: + return self.given_section_order + + @property + def rodata_follows_data(self) -> bool: + if ".rodata" not in self.section_order or ".data" not in self.section_order: + return False + return ( + self.section_order.index(".rodata") - self.section_order.index(".data") == 1 + ) + + def contains_vram(self, vram: int) -> bool: + if self.vram_start is not None and self.vram_end is not None: + return vram >= self.vram_start and vram < self.vram_end + else: + return False + + def contains_rom(self, rom: int) -> bool: + if isinstance(self.rom_start, int) and isinstance(self.rom_end, int): + return rom >= self.rom_start and rom < self.rom_end + else: + return False + + def rom_to_ram(self, rom_addr: int) -> Optional[int]: + if not self.contains_rom(rom_addr) and rom_addr != self.rom_end: + return None + + if self.vram_start is not None and isinstance(self.rom_start, int): + return self.vram_start + rom_addr - self.rom_start + else: + return None + + def ram_to_rom(self, ram_addr: int) -> Optional[int]: + if not self.contains_vram(ram_addr) and ram_addr != self.vram_end: + return None + + if self.vram_start is not None and isinstance(self.rom_start, int): + return self.rom_start + ram_addr - self.vram_start + else: + return None + + def should_scan(self) -> bool: + return self.should_split() + + def should_split(self) -> bool: + return self.extract and options.opts.is_mode_active(self.type) + + def scan(self, rom_bytes: bytes): + pass + + def split(self, rom_bytes: bytes): + pass + + def cache(self): + return (self.yaml, self.rom_end) + + def get_linker_section(self) -> str: + return ".data" + + def out_path(self) -> Optional[Path]: + return None + + def get_most_parent(self) -> "Segment": + seg = self + + while seg.parent: + seg = seg.parent + + return seg + + def get_linker_entries(self) -> "List[LinkerEntry]": + from segtypes.linker_entry import LinkerEntry + + path = self.out_path() + + if path: + return [LinkerEntry(self, [path], path, self.get_linker_section())] + else: + return [] + + def log(self, msg): + if options.opts.verbose: + log.write(f"{self.type} {self.name}: {msg}") + + def warn(self, msg: str): + self.warnings.append(msg) + + def max_length(self): + return None + + def is_name_default(self): + return self.name == self.get_default_name(self.rom_start) + + def unique_id(self): + if self.parent: + s = self.parent.unique_id() + "_" + else: + s = "" + + return s + self.type + "_" + self.name + + def status(self): + if len(self.warnings) > 0: + return "warn" + elif self.did_run: + return "ok" + else: + return "skip" + + @staticmethod + def get_default_name(addr) -> str: + return f"{addr:X}" + + @staticmethod + def visible_ram(seg1: "Segment", seg2: "Segment") -> bool: + if seg1.get_most_parent() == seg2.get_most_parent(): + return True + if seg1.get_exclusive_ram_id() is None or seg2.get_exclusive_ram_id() is None: + return True + return seg1.get_exclusive_ram_id() != seg2.get_exclusive_ram_id() + + def retrieve_symbol( + self, syms: Dict[int, List[Symbol]], addr: int + ) -> Optional[Symbol]: + if addr not in syms: + return None + + items = syms[addr] + + # Filter out symbols that are in different top-level segments with the same unique_ram_id + items = [ + i + for i in items + if i.segment is None or Segment.visible_ram(self, i.segment) + ] + + if len(items) > 1: + # print(f"Trying to retrieve {addr:X} from symbol dict but there are {len(items)} entries to pick from - picking the first") + pass + if len(items) == 0: + return None + return items[0] + + def get_symbol( + self, + addr: int, + in_segment: bool = False, + type: Optional[str] = None, + create: bool = False, + define: bool = False, + reference: bool = False, + search_ranges: bool = False, + local_only: bool = False, + dead: bool = True, + ) -> Optional[Symbol]: + ret: Optional[Symbol] = None + rom: Optional[int] = None + + most_parent = self.get_most_parent() + + if in_segment: + # If the vram address is within this segment, we can calculate the symbol's rom address + rom = most_parent.ram_to_rom(addr) + ret = most_parent.retrieve_symbol(most_parent.seg_symbols, addr) + + if not ret and search_ranges: + # Search ranges first, starting with rom + if rom is not None: + cands: Set[Interval] = most_parent.symbol_ranges_rom[rom] + if cands: + ret = cands.pop().data + # and then vram if we can't find a rom match + if not ret: + cands = most_parent.symbol_ranges_ram[addr] + if cands: + ret = cands.pop().data + elif not local_only: + ret = most_parent.retrieve_symbol(symbols.all_symbols_dict, addr) + + if not ret and search_ranges: + cands = symbols.all_symbols_ranges[addr] + if cands: + ret = cands.pop().data + + # Reject dead symbols unless we allow them + if not dead and ret and ret.dead: + ret = None + + # Create the symbol if it doesn't exist + if not ret and create: + ret = Symbol(addr, rom=rom, type=type) + symbols.add_symbol(ret) + + if in_segment: + ret.segment = most_parent + if addr not in most_parent.seg_symbols: + most_parent.seg_symbols[addr] = [] + most_parent.seg_symbols[addr].append(ret) + + if ret: + if define: + ret.defined = True + if reference: + ret.referenced = True + if ret.type is None: + ret.type = type + if ret.rom is None: + ret.rom = rom + if in_segment: + if ret.segment is None: + ret.segment = most_parent + + return ret + + def create_symbol( + self, + addr: int, + in_segment: bool, + type: Optional[str] = None, + define: bool = False, + reference: bool = False, + search_ranges: bool = False, + local_only: bool = False, + dead: bool = True, + ) -> Symbol: + ret = self.get_symbol( + addr, + in_segment=in_segment, + type=type, + create=True, + define=define, + reference=reference, + search_ranges=search_ranges, + local_only=local_only, + dead=dead, + ) + assert ret is not None + + return ret + + def get_func_for_addr(self, addr) -> Optional[Symbol]: + for syms in self.seg_symbols.values(): + for sym in syms: + if sym.type == "func" and sym.contains_vram(addr): + return sym + + return None diff --git a/tools/splat/split.py b/tools/splat/split.py new file mode 100755 index 0000000..18ab2c8 --- /dev/null +++ b/tools/splat/split.py @@ -0,0 +1,562 @@ +#! /usr/bin/env python3 + +import argparse +import hashlib +import importlib +import pickle +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +import rabbitizer +import spimdisasm +import tqdm +import yaml +from colorama import Fore, Style +from intervaltree import Interval, IntervalTree + +from segtypes.linker_entry import ( + LinkerWriter, + get_segment_vram_end_symbol_name, + to_cname, +) +from segtypes.segment import Segment +from util import compiler, log, options, palettes, symbols, relocs + +from util.symbols import Symbol + +VERSION = "0.13.5" +# This value should be kept in sync with the version listed on requirements.txt +SPIMDISASM_MIN = (1, 12, 0) + +parser = argparse.ArgumentParser( + description="Split a rom given a rom, a config, and output directory" +) +parser.add_argument("config", help="path to a compatible config .yaml file", nargs="+") +parser.add_argument("--modes", nargs="+", default="all") +parser.add_argument("--verbose", action="store_true", help="Enable debug logging") +parser.add_argument( + "--use-cache", action="store_true", help="Only split changed segments in config" +) +parser.add_argument( + "--skip-version-check", + action="store_true", + help="Skips the disassembler's version check", +) + +linker_writer: LinkerWriter +config: Dict[str, Any] + +segment_roms: IntervalTree = IntervalTree() +segment_rams: IntervalTree = IntervalTree() + + +def fmt_size(size): + if size > 1000000: + return str(size // 1000000) + " MB" + elif size > 1000: + return str(size // 1000) + " KB" + else: + return str(size) + " B" + + +def initialize_segments(config_segments: Union[dict, list]) -> List[Segment]: + global segment_roms + global segment_rams + + segment_roms = IntervalTree() + segment_rams = IntervalTree() + + segments_by_name: Dict[str, Segment] = {} + ret = [] + + last_rom_end = 0 + + for i, seg_yaml in enumerate(config_segments): + # end marker + if isinstance(seg_yaml, list) and len(seg_yaml) == 1: + continue + + seg_type = Segment.parse_segment_type(seg_yaml) + + segment_class = Segment.get_class_for_type(seg_type) + + this_start = Segment.parse_segment_start(seg_yaml) + + if i == len(config_segments) - 1 and Segment.parse_segment_file_path(seg_yaml): + next_start: Optional[int] = 0 + else: + next_start = Segment.parse_segment_start(config_segments[i + 1]) + + if segment_class.is_noload(): + # Pretend bss's rom address is after the last actual rom segment + this_start = last_rom_end + # and it has a rom size of zero + next_start = last_rom_end + + segment: Segment = Segment.from_yaml( + segment_class, seg_yaml, this_start, next_start + ) + + if segment.require_unique_name: + if segment.name in segments_by_name: + log.error(f"segment name '{segment.name}' is not unique") + + segments_by_name[segment.name] = segment + + ret.append(segment) + if ( + isinstance(segment.rom_start, int) + and isinstance(segment.rom_end, int) + and segment.rom_start != segment.rom_end + ): + segment_roms.addi(segment.rom_start, segment.rom_end, segment) + if ( + isinstance(segment.vram_start, int) + and isinstance(segment.vram_end, int) + and segment.vram_start != segment.vram_end + ): + segment_rams.addi(segment.vram_start, segment.vram_end, segment) + + if next_start is not None: + last_rom_end = next_start + + for segment in ret: + if segment.given_follows_vram: + if segment.given_follows_vram not in segments_by_name: + log.error( + f"segment '{segment.given_follows_vram}', the 'follows_vram' value for segment '{segment.name}', does not exist" + ) + segment.vram_of_symbol = get_segment_vram_end_symbol_name( + segments_by_name[segment.given_follows_vram] + ) + + return ret + + +def assign_symbols_to_segments(): + for symbol in symbols.all_symbols: + if symbol.segment: + continue + + if symbol.rom: + cands: Set[Interval] = segment_roms[symbol.rom] + if len(cands) > 1: + log.error("multiple segments rom overlap symbol", symbol) + elif len(cands) == 0: + log.error("no segment rom overlaps symbol", symbol) + else: + cand: Interval = cands.pop() + seg: Segment = cand.data + seg.add_symbol(symbol) + else: + cands = segment_rams[symbol.vram_start] + segs: List[Segment] = [cand.data for cand in cands] + for seg in segs: + if not seg.get_exclusive_ram_id(): + seg.add_symbol(symbol) + + +def do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached): + unk_size = seg_sizes.get("unk", 0) + rest_size = 0 + total_size = len(rom_bytes) + + for typ in seg_sizes: + if typ != "unk": + rest_size += seg_sizes[typ] + + known_ratio = rest_size / total_size + unk_ratio = unk_size / total_size + + log.write(f"Split {fmt_size(rest_size)} ({known_ratio:.2%}) in defined segments") + for typ in seg_sizes: + if typ != "unk": + tmp_size = seg_sizes[typ] + tmp_ratio = tmp_size / total_size + log.write( + f"{typ:>20}: {fmt_size(tmp_size):>8} ({tmp_ratio:.2%}) {Fore.GREEN}{seg_split[typ]} split{Style.RESET_ALL}, {Style.DIM}{seg_cached[typ]} cached" + ) + log.write( + f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files" + ) + + +def merge_configs(main_config, additional_config): + # Merge rules are simple + # For each key in the dictionary + # - If list then append to list + # - If a dictionary then repeat merge on sub dictionary entries + # - Else assume string or number and replace entry + + for curkey in additional_config: + if curkey not in main_config: + main_config[curkey] = additional_config[curkey] + elif type(main_config[curkey]) != type(additional_config[curkey]): + log.error(f"Type for key {curkey} in configs does not match") + else: + # keys exist and match, see if a list to append + if type(main_config[curkey]) == list: + main_config[curkey] += additional_config[curkey] + elif type(main_config[curkey]) == dict: + # need to merge sub areas + main_config[curkey] = merge_configs( + main_config[curkey], additional_config[curkey] + ) + else: + # not a list or dictionary, must be a number or string, overwrite + main_config[curkey] = additional_config[curkey] + + return main_config + + +def configure_disassembler(): + # Configure spimdisasm + spimdisasm.common.GlobalConfig.PRODUCE_SYMBOLS_PLUS_OFFSET = True + spimdisasm.common.GlobalConfig.TRUST_USER_FUNCTIONS = True + spimdisasm.common.GlobalConfig.TRUST_JAL_FUNCTIONS = True + spimdisasm.common.GlobalConfig.GLABEL_ASM_COUNT = False + + if options.opts.rom_address_padding: + spimdisasm.common.GlobalConfig.ASM_COMMENT_OFFSET_WIDTH = 6 + else: + spimdisasm.common.GlobalConfig.ASM_COMMENT_OFFSET_WIDTH = 0 + + # spimdisasm is not performing any analyzis on non-text sections so enabling this options is pointless + spimdisasm.common.GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_SECTION_TYPE = False + spimdisasm.common.GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_DATA_TYPE = False + + spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False + + rabbitizer.config.regNames_userFpcCsr = False + rabbitizer.config.regNames_vr4300Cop0NamedRegisters = False + + rabbitizer.config.misc_opcodeLJust = options.opts.mnemonic_ljust - 1 + + rabbitizer.config.regNames_gprAbiNames = rabbitizer.Abi.fromStr( + options.opts.mips_abi_gpr + ) + rabbitizer.config.regNames_fprAbiNames = rabbitizer.Abi.fromStr( + options.opts.mips_abi_float_regs + ) + + if options.opts.endianness == "big": + spimdisasm.common.GlobalConfig.ENDIAN = spimdisasm.common.InputEndian.BIG + else: + spimdisasm.common.GlobalConfig.ENDIAN = spimdisasm.common.InputEndian.LITTLE + + rabbitizer.config.pseudos_pseudoMove = False + + selected_compiler = options.opts.compiler + if selected_compiler == compiler.SN64: + rabbitizer.config.regNames_namedRegisters = False + rabbitizer.config.toolchainTweaks_sn64DivFix = True + rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = True + spimdisasm.common.GlobalConfig.ASM_COMMENT = False + spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False + spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.SN64 + elif selected_compiler == compiler.GCC: + rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = True + spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.GCC + elif selected_compiler == compiler.IDO: + spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.IDO + + spimdisasm.common.GlobalConfig.GP_VALUE = options.opts.gp + + spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL = options.opts.asm_function_macro + spimdisasm.common.GlobalConfig.ASM_JTBL_LABEL = options.opts.asm_jtbl_label_macro + spimdisasm.common.GlobalConfig.ASM_DATA_LABEL = options.opts.asm_data_macro + spimdisasm.common.GlobalConfig.ASM_TEXT_END_LABEL = options.opts.asm_end_label + + if spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL == ".globl": + spimdisasm.common.GlobalConfig.ASM_TEXT_ENT_LABEL = ".ent" + spimdisasm.common.GlobalConfig.ASM_TEXT_FUNC_AS_LABEL = True + + if spimdisasm.common.GlobalConfig.ASM_DATA_LABEL == ".globl": + spimdisasm.common.GlobalConfig.ASM_DATA_SYM_AS_LABEL = True + + spimdisasm.common.GlobalConfig.LINE_ENDS = options.opts.c_newline + + spimdisasm.common.GlobalConfig.ALLOW_ALL_ADDENDS_ON_DATA = ( + options.opts.allow_data_addends + ) + + +def brief_seg_name(seg: Segment, limit: int, ellipsis="…") -> str: + s = seg.name.strip() + if len(s) > limit: + return s[:limit].strip() + ellipsis + return s + + +def main(config_path, modes, verbose, use_cache=True, skip_version_check=False): + global config + + if not skip_version_check and spimdisasm.__version_info__ < SPIMDISASM_MIN: + log.error( + f"splat {VERSION} requires as minimum spimdisasm {SPIMDISASM_MIN}, but the installed version is {spimdisasm.__version_info__}" + ) + + log.write(f"splat {VERSION} (powered by spimdisasm {spimdisasm.__version__})") + + # Load config + config = {} + for entry in config_path: + with open(entry) as f: + additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader) + config = merge_configs(config, additional_config) + + options.initialize(config, config_path, modes, verbose) + + with options.opts.target_path.open("rb") as f2: + rom_bytes = f2.read() + + if "sha1" in config: + sha1 = hashlib.sha1(rom_bytes).hexdigest() + e_sha1 = config["sha1"].lower() + if e_sha1 != sha1: + log.error(f"sha1 mismatch: expected {e_sha1}, was {sha1}") + + # Create main output dir + options.opts.base_path.mkdir(parents=True, exist_ok=True) + + processed_segments: List[Segment] = [] + + seg_sizes: Dict[str, int] = {} + seg_split: Dict[str, int] = {} + seg_cached: Dict[str, int] = {} + + # Load cache + if use_cache: + try: + with options.opts.cache_path.open("rb") as f3: + cache = pickle.load(f3) + + if verbose: + log.write(f"Loaded cache ({len(cache.keys())} items)") + except Exception: + cache = {} + else: + cache = {} + + # invalidate entire cache if options change + if use_cache and cache.get("__options__") != config.get("options"): + if verbose: + log.write("Options changed, invalidating cache") + + cache = { + "__options__": config.get("options"), + } + + configure_disassembler() + + platform_module = importlib.import_module(f"platforms.{options.opts.platform}") + platform_init = getattr(platform_module, "init") + platform_init(rom_bytes) + + # Initialize segments + all_segments = initialize_segments(config["segments"]) + + # Load and process symbols + symbols.initialize(all_segments) + relocs.initialize() + + # Assign symbols to segments + assign_symbols_to_segments() + + if options.opts.is_mode_active("code"): + symbols.initialize_spim_context(all_segments) + relocs.initialize_spim_context() + + # Resolve raster/palette siblings + if options.opts.is_mode_active("img"): + palettes.initialize(all_segments) + + # Scan + scan_bar = tqdm.tqdm(all_segments, total=len(all_segments)) + for segment in scan_bar: + assert isinstance(segment, Segment) + scan_bar.set_description(f"Scanning {brief_seg_name(segment, 20)}") + typ = segment.type + if segment.type == "bin" and segment.is_name_default(): + typ = "unk" + + if typ not in seg_sizes: + seg_sizes[typ] = 0 + seg_split[typ] = 0 + seg_cached[typ] = 0 + seg_sizes[typ] += 0 if segment.size is None else segment.size + + if segment.should_scan(): + # Check cache but don't write anything + if use_cache: + if segment.cache() == cache.get(segment.unique_id()): + continue + + segment.did_run = True + segment.scan(rom_bytes) + + processed_segments.append(segment) + + seg_split[typ] += 1 + + symbols.mark_c_funcs_as_defined() + + # Split + split_bar = tqdm.tqdm( + all_segments, + total=len(all_segments), + ) + for segment in split_bar: + split_bar.set_description(f"Splitting {brief_seg_name(segment, 20)}") + + if use_cache: + cached = segment.cache() + + if cached == cache.get(segment.unique_id()): + # Cache hit + if segment.type not in seg_cached: + seg_cached[segment.type] = 0 + seg_cached[segment.type] += 1 + continue + else: + # Cache miss; split + cache[segment.unique_id()] = cached + + if segment.should_split(): + segment_bytes = rom_bytes + if segment.file_path: + with open(segment.file_path, "rb") as segment_input_file: + segment_bytes = segment_input_file.read() + segment.split(segment_bytes) + + if ( + options.opts.is_mode_active("ld") and options.opts.platform != "gc" + ): # TODO move this to platform initialization when it gets implemented + # Calculate list of segments for which we need to find the largest so we can safely place the symbol after it + max_vram_end_syms: Dict[str, List[Segment]] = {} + for sym in symbols.appears_after_overlays_syms: + max_vram_end_syms[sym.name] = [ + seg + for seg in all_segments + if isinstance(seg.vram_start, int) + and seg.vram_start == sym.appears_after_overlays_addr + ] + max_vram_end_sym_names: Set[str] = set(max_vram_end_syms.keys()) + + max_vram_end_insertion_points: Dict[ + Segment, List[Tuple[str, List[Segment]]] + ] = {} + # Find the last segment whose vram_of_symbol is one of the max_vram_end_syms + for segment in reversed(all_segments): + vram_of_sym = segment.vram_of_symbol + if vram_of_sym is not None and vram_of_sym in max_vram_end_sym_names: + if segment not in max_vram_end_insertion_points: + max_vram_end_insertion_points[segment] = [] + max_vram_end_insertion_points[segment].append( + (vram_of_sym, max_vram_end_syms[vram_of_sym]) + ) + max_vram_end_sym_names.remove(vram_of_sym) + + global linker_writer + linker_writer = LinkerWriter() + linker_bar = tqdm.tqdm( + all_segments, + total=len(all_segments), + ) + + for segment in linker_bar: + linker_bar.set_description(f"Linker script {brief_seg_name(segment, 20)}") + linker_writer.add(segment, max_vram_end_insertion_points.get(segment, [])) + linker_writer.save_linker_script() + linker_writer.save_symbol_header() + + # write elf_sections.txt - this only lists the generated sections in the elf, not subsections + # that the elf combines into one section + if options.opts.elf_section_list_path: + section_list = "" + for segment in all_segments: + section_list += "." + to_cname(segment.name) + "\n" + with open(options.opts.elf_section_list_path, "w", newline="\n") as f: + f.write(section_list) + + # Write undefined_funcs_auto.txt + if options.opts.create_undefined_funcs_auto: + to_write = [ + s + for s in symbols.all_symbols + if s.referenced and not s.defined and not s.dead and s.type == "func" + ] + to_write.sort(key=lambda x: x.vram_start) + + with open(options.opts.undefined_funcs_auto_path, "w", newline="\n") as f: + for symbol in to_write: + f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n") + + # write undefined_syms_auto.txt + if options.opts.create_undefined_syms_auto: + to_write = [ + s + for s in symbols.all_symbols + if s.referenced + and not s.defined + and not s.dead + and s.type not in {"func", "label", "jtbl_label"} + ] + to_write.sort(key=lambda x: x.vram_start) + + with open(options.opts.undefined_syms_auto_path, "w", newline="\n") as f: + for symbol in to_write: + f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n") + + # print warnings during split + for segment in all_segments: + if len(segment.warnings) > 0: + log.write( + f"{Style.DIM}0x{segment.rom_start:06X}{Style.RESET_ALL} {segment.type} {Style.BRIGHT}{segment.name}{Style.RESET_ALL}:" + ) + + for warn in segment.warnings: + log.write("warning: " + warn, status="warn") + + log.write("") # empty line + + # Statistics + do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached) + + # Save cache + if cache != {} and use_cache: + if verbose: + log.write("Writing cache") + with open(options.opts.cache_path, "wb") as f4: + pickle.dump(cache, f4) + + if options.opts.dump_symbols and options.opts.is_mode_active("code"): + from pathlib import Path + + splat_hidden_folder = Path(".splat/") + splat_hidden_folder.mkdir(exist_ok=True) + + with open(splat_hidden_folder / "splat_symbols.csv", "w") as f: + f.write( + "vram_start,given_name,name,type,given_size,size,rom,defined,user_declared,referenced,dead,extract\n" + ) + for s in sorted(symbols.all_symbols, key=lambda x: x.vram_start): + f.write(f"{s.vram_start:X},{s.given_name},{s.name},{s.type},") + if s.given_size is not None: + f.write(f"0x{s.given_size:X},") + else: + f.write("None,") + f.write(f"{s.size},") + if s.rom is not None: + f.write(f"0x{s.rom:X},") + else: + f.write("None,") + f.write( + f"{s.defined},{s.user_declared},{s.referenced},{s.dead},{s.extract}\n" + ) + + symbols.spim_context.saveContextToFile(splat_hidden_folder / "spim_context.csv") + + +if __name__ == "__main__": + args = parser.parse_args() + main(args.config, args.modes, args.verbose, args.use_cache, args.skip_version_check) diff --git a/tools/splat/stubs/colorama.pyi b/tools/splat/stubs/colorama.pyi new file mode 100644 index 0000000..5d13e50 --- /dev/null +++ b/tools/splat/stubs/colorama.pyi @@ -0,0 +1,8 @@ +from typing import Any + +def init(**kwargs): ... + +Fore: Any +Back: Any +Style: Any +Cursor: Any diff --git a/tools/splat/util/__init__.py b/tools/splat/util/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/splat/util/__init__.py diff --git a/tools/splat/util/color.py b/tools/splat/util/color.py new file mode 100644 index 0000000..6d75cbd --- /dev/null +++ b/tools/splat/util/color.py @@ -0,0 +1,19 @@ +from math import ceil + +from util import options + + +# RRRRRGGG GGBBBBBA +def unpack_color(data): + s = int.from_bytes(data[0:2], byteorder=options.opts.endianness) + + r = (s >> 11) & 0x1F + g = (s >> 6) & 0x1F + b = (s >> 1) & 0x1F + a = (s & 1) * 0xFF + + r = ceil(0xFF * (r / 31)) + g = ceil(0xFF * (g / 31)) + b = ceil(0xFF * (b / 31)) + + return r, g, b, a diff --git a/tools/splat/util/compiler.py b/tools/splat/util/compiler.py new file mode 100644 index 0000000..c89c35c --- /dev/null +++ b/tools/splat/util/compiler.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass + + +@dataclass +class Compiler: + name: str + asm_function_macro: str = "glabel" + asm_jtbl_label_macro: str = "glabel" + asm_data_macro: str = "glabel" + asm_end_label: str = "" + c_newline: str = "\n" + asm_inc_header: str = "" + include_macro_inc: bool = True + + +GCC = Compiler( + "GCC", + asm_inc_header=".set noat /* allow manual use of $at */\n.set noreorder /* don't insert nops after branches */\n\n", +) + +SN64 = Compiler( + "SN64", + asm_function_macro=".globl", + asm_jtbl_label_macro=".globl", + asm_data_macro=".globl", + asm_end_label=".end", + c_newline="\r\n", + include_macro_inc=False, +) + +IDO = Compiler("IDO") + +compiler_for_name = {"GCC": GCC, "SN64": SN64, "IDO": IDO} + + +def for_name(name: str) -> Compiler: + name = name.upper() + if name in compiler_for_name: + return compiler_for_name[name] + return Compiler(name) diff --git a/tools/splat/util/floats.py b/tools/splat/util/floats.py new file mode 100644 index 0000000..009725e --- /dev/null +++ b/tools/splat/util/floats.py @@ -0,0 +1,63 @@ +import math +import struct + + +# From mips_to_c: https://github.com/matt-kempster/mips_to_c/blob/d208400cca045113dada3e16c0d59c50cdac4529/src/translate.py#L2085 +def format_f32_imm(num: int) -> str: + packed = struct.pack(">I", num & (2**32 - 1)) + value = struct.unpack(">f", packed)[0] + + if not value or value == 4294967296.0: + # Zero, negative zero, nan, or INT_MAX. + return str(value) + + # Write values smaller than 1e-7 / greater than 1e7 using scientific notation, + # and values in between using fixed point. + if abs(math.log10(abs(value))) > 6.9: + fmt_char = "e" + elif abs(value) < 1: + fmt_char = "f" + else: + fmt_char = "g" + + def fmt(prec: int) -> str: + """Format 'value' with 'prec' significant digits/decimals, in either scientific + or regular notation depending on 'fmt_char'.""" + ret = ("{:." + str(prec) + fmt_char + "}").format(value) + if fmt_char == "e": + return ret.replace("e+", "e").replace("e0", "e").replace("e-0", "e-") + if "e" in ret: + # The "g" format character can sometimes introduce scientific notation if + # formatting with too few decimals. If this happens, return an incorrect + # value to prevent the result from being used. + # + # Since the value we are formatting is within (1e-7, 1e7) in absolute + # value, it will at least be possible to format with 7 decimals, which is + # less than float precision. Thus, this annoying Python limitation won't + # lead to us outputting numbers with more precision than we really have. + return "0" + return ret + + # 20 decimals is more than enough for a float. Start there, then try to shrink it. + prec = 20 + while prec > 0: + prec -= 1 + value2 = float(fmt(prec)) + if struct.pack(">f", value2) != packed: + prec += 1 + break + + if prec == 20: + # Uh oh, even the original value didn't format correctly. Fall back to str(), + # which ought to work. + return str(value) + + ret = fmt(prec) + if "." not in ret and "e" not in ret: + ret += ".0" + return ret + + +def format_f64_imm(num: int) -> str: + (value,) = struct.unpack(">d", struct.pack(">Q", num & (2**64 - 1))) + return str(value) diff --git a/tools/splat/util/gc/gcfst.py b/tools/splat/util/gc/gcfst.py new file mode 100644 index 0000000..e50420f --- /dev/null +++ b/tools/splat/util/gc/gcfst.py @@ -0,0 +1,181 @@ +import struct +from pathlib import Path +from typing import List, Optional + +from segtypes.gc.segment import GCSegment + +from util import options +from util.gc.gcutil import read_string_from_bytes + + +# Represents the info for either a directory or a file within a GameCube disc image's file system. +class GCFSTEntry: + def __init__(self, flags: bool, name_offset, offset, length): + self.flags = flags + self.name_offset = name_offset + self.offset = offset + self.length = length + + self.name = "" + self.parent: Optional[GCFSTEntry] = None + self.children: List[GCFSTEntry] = [] + + def populate_children_recursive( + self, root_dir: "GCFSTEntry", current_node_offset, fst_bytes, string_table_bytes + ): + self.parent = root_dir + self.name = read_string_from_bytes(self.name_offset, string_table_bytes) + + # This node is a file, so we don't do anything but return that we read 1 node. + if self.flags == False: + return 1 + + nodes_read = 1 + next_child_offset = current_node_offset + 0x0C + + # Directory nodes contain the index of the next node that is NOT its child, meaning the index of their next sibling node. + # We can figure out when we're done reading child nodes by comparing the offset of the next node to read to the + # offset of the next sibling node. We stop reading when the next node offset is >= the offset of the next sibling node. + while next_child_offset < self.length * 0x0C: + new_entry = GCFSTEntry( + bool(fst_bytes[next_child_offset + 0x0000]), + struct.unpack_from( + ">I", fst_bytes[next_child_offset : next_child_offset + 0x0004] + )[0] + & 0x00FFFFFF, + struct.unpack_from(">I", fst_bytes, next_child_offset + 0x0004)[0], + struct.unpack_from(">I", fst_bytes, next_child_offset + 0x0008)[0], + ) + + self.children.append(new_entry) + nodes_read += new_entry.populate_children_recursive( + self, next_child_offset, fst_bytes, string_table_bytes + ) + + next_child_offset = current_node_offset + nodes_read * 0x0C + + return nodes_read + + # Builds this entry's full path within the filesystem from its parents' names. + def get_full_name(self): + path_components: List[str] = [] + + entry = self + while entry.parent != None: + path_components.insert(0, entry.name) + + if entry.parent is None: + break + entry = entry.parent + + return Path(*path_components) + + # Emits this entry to the filesystem. + def emit(self, filesystem_dir: Path, iso_bytes): + full_path = filesystem_dir / self.get_full_name() + + # If this is a directory, we just need to make the directory on disk. + if self.flags == True: + full_path.mkdir(parents=True, exist_ok=True) + return + + file_bytes = iso_bytes[self.offset : self.offset + self.length] + with open(full_path, "wb") as f: + f.write(file_bytes) + + def emit_recursive(self, filesystem_dir: Path, iso_bytes): + # Don't emit if this is the root directory. + if self.parent != None: + self.emit(filesystem_dir, iso_bytes) + + for e in self.children: + e.emit_recursive(filesystem_dir, iso_bytes) + + +# Splits the ISO into its component parts - header info, apploader, DOL, FST metadata, and the individual files in the filesystem. +def split_iso(iso_bytes): + split_sys_info(iso_bytes) + split_content(iso_bytes) + + +# Splits the header info, apploader, DOL, and FST metadata from the ISO. +def split_sys_info(iso_bytes): + assert options.opts.filesystem_path is not None + + sys_path = options.opts.filesystem_path / "sys" + sys_path.mkdir(parents=True, exist_ok=True) + + # Split boot.info. Always at 0x0000 and 0x0440 bytes long. + with open(sys_path / "boot.bin", "wb") as f: + f.write(iso_bytes[0x0000:0x0440]) + + # Split bi2.info. Always at 0x0440 and 0x2000 bytes long. + with open(sys_path / "bi2.bin", "wb") as f: + f.write(iso_bytes[0x0440:0x2440]) + + # Split apploader.img. Always at 0x2440 and size is listed at 0x0400. + apploader_size = struct.unpack_from(">I", iso_bytes, 0x0400)[0] + with open(sys_path / "apploader.img", "wb") as f: + f.write(iso_bytes[0x2440 : 0x2440 + apploader_size]) + + # Split main.dol. Offset specified explicitly at 0x0420, but size must be calculated. + dol_offset = struct.unpack_from(">I", iso_bytes, 0x0420)[0] + fst_offset = struct.unpack_from(">I", iso_bytes, 0x0424)[0] + + dol_size = fst_offset - dol_offset + with open(sys_path / "main.dol", "wb") as f: + f.write(iso_bytes[dol_offset : dol_offset + dol_size]) + + # Split fst.bin. Offset specified at 0x0424 and size specified at 0x402C. + fst_size = struct.unpack_from(">I", iso_bytes, 0x0428)[0] + with open(sys_path / "fst.bin", "wb") as f: + f.write(iso_bytes[fst_offset : fst_offset + fst_size]) + + +# Splits the ISO's filesystem into individual files. +def split_content(iso_bytes): + assert options.opts.filesystem_path is not None + + fst_path = options.opts.filesystem_path / "sys" / "fst.bin" + assert fst_path.is_file() + + fst_bytes = fst_path.read_bytes() + fst_root_entry = populate_filesystem(fst_bytes) + + files_path = options.opts.filesystem_path / "files" + files_path.mkdir(parents=True, exist_ok=True) + fst_root_entry.emit_recursive(files_path, iso_bytes) + + +# Loads the FST data needed to split the filesystem. +def populate_filesystem(fst_bytes): + root_dir = GCFSTEntry( + bool(fst_bytes[0x0000]), + struct.unpack_from(">I", fst_bytes, 0x0000)[0] & 0x00FFFFFF, + struct.unpack_from(">I", fst_bytes, 0x0004)[0], + struct.unpack_from(">I", fst_bytes, 0x0008)[0], + ) + + string_table_bytes = fst_bytes[root_dir.length * 0x0C : len(fst_bytes)] + + # Parsing the filesystem is a bit tricky. The root directory's length property is the total number of nodes in the FST. + # So, we initialize nodes_read to 1, since the root is included in the number of nodes. + # We will rely on each directory and file on the root directory to tell us how many nodes were read while parsing them. + # We can stop reading the FST when our total number of nodes read is >= the number of nodes in the FST. + nodes_read = 1 + while nodes_read < root_dir.length: + current_offset = nodes_read * 0x0C + + new_entry = GCFSTEntry( + bool(fst_bytes[current_offset + 0x0000]), + struct.unpack_from(">I", fst_bytes, current_offset)[0] & 0x00FFFFFF, + struct.unpack_from(">I", fst_bytes, current_offset + 0x0004)[0], + struct.unpack_from(">I", fst_bytes, current_offset + 0x0008)[0], + ) + + root_dir.children.append(new_entry) + nodes_read += new_entry.populate_children_recursive( + root_dir, current_offset, fst_bytes, string_table_bytes + ) + + return root_dir diff --git a/tools/splat/util/gc/gcinfo.py b/tools/splat/util/gc/gcinfo.py new file mode 100644 index 0000000..78130df --- /dev/null +++ b/tools/splat/util/gc/gcinfo.py @@ -0,0 +1,88 @@ +#! /usr/bin/env python3 + + +import argparse + +import hashlib + +from pathlib import Path +from typing import Optional + +parser = argparse.ArgumentParser( + description="Gives information on GameCube disc images" +) +parser.add_argument("iso", help="path to a GameCube disc image") + +system_codes = { + "D": "GameCube Demo", + "G": "GameCube", + "P": "GameCube Promotional", + "R": "Early Wii", + "S": "Later Wii", +} + +region_codes = {"E": "NTSC-U", "J": "NTSC-J", "P": "PAL"} + +publisher_codes = {"01": "Nintendo", "08": "Capcom", "8P": "Sega", "E9": "Natsume"} + + +def get_info(iso_path: Path, iso_bytes: Optional[bytes] = None): + if iso_bytes is None: + iso_bytes = iso_path.read_bytes() + + return get_info_bytes(iso_bytes) + + +def get_info_bytes(iso_bytes: bytes): + system_code = chr(iso_bytes[0x00]) + game_code = iso_bytes[0x01:0x03].decode("utf-8") + region_code = chr(iso_bytes[0x03]) + publisher_code = iso_bytes[0x04:0x06].decode("utf-8") + + name = str(iso_bytes[0x20:0x400], "utf-8").strip("\x00") + root = "filesystem" + + compiler = "mwcc" + sha1 = hashlib.sha1(iso_bytes).hexdigest() + + return GCIso( + name, + root, + system_code, + game_code, + region_code, + publisher_code, + compiler, + sha1, + ) + + +class GCIso: + def __init__( + self, + name: str, + root: str, + system_code, + game_code, + region_code, + publisher_code, + compiler, + sha1, + ): + self.name = name + self.root = root + self.system_code = system_code + self.game_code = game_code + self.region_code = region_code + self.publisher_code = publisher_code + self.compiler = compiler + self.sha1 = sha1 + + def get_system_name(self): + return system_codes[self.system_code] + + def get_publisher_name(self): + return publisher_codes[self.publisher_code] + + def get_region_name(self): + return region_codes[self.region_code] diff --git a/tools/splat/util/gc/gcutil.py b/tools/splat/util/gc/gcutil.py new file mode 100644 index 0000000..ba78bc2 --- /dev/null +++ b/tools/splat/util/gc/gcutil.py @@ -0,0 +1,11 @@ +def read_string_from_bytes(name_offset, string_table_bytes): + bytes = bytearray() + + for offset in range(len(string_table_bytes) - name_offset): + cur_byte = string_table_bytes[name_offset + offset] + if cur_byte == 0x00: + break + + bytes.append(cur_byte) + + return bytes.decode("shift-jis") diff --git a/tools/splat/util/log.py b/tools/splat/util/log.py new file mode 100644 index 0000000..df6ef53 --- /dev/null +++ b/tools/splat/util/log.py @@ -0,0 +1,45 @@ +import sys +from typing import NoReturn, Optional + +from colorama import Fore, init, Style + +init(autoreset=True) + +newline = True + +Status = Optional[str] + + +def write(*args, status=None, **kwargs): + global newline + + if not newline: + print("") + newline = True + + print(status_to_ansi(status) + str(args[0]), *args[1:], **kwargs) + + +def error(*args, **kwargs) -> NoReturn: + write(*args, **kwargs, status="error") + sys.exit(2) + + +# The line_num is expected to be zero-indexed +def parsing_error_preamble(path, line_num, line): + write("") + write(f"error reading {path}, line {line_num + 1}:", status="error") + write(f"\t{line}") + + +def status_to_ansi(status: Status): + if status == "ok": + return Fore.GREEN + elif status == "warn": + return Fore.YELLOW + Style.BRIGHT + elif status == "error": + return Fore.RED + Style.BRIGHT + elif status == "skip": + return Style.DIM + else: + return "" diff --git a/tools/splat/util/n64/Mio0decompress.py b/tools/splat/util/n64/Mio0decompress.py new file mode 100644 index 0000000..64a059e --- /dev/null +++ b/tools/splat/util/n64/Mio0decompress.py @@ -0,0 +1,108 @@ +import argparse +import struct +import sys + +try: + from .. import log + from .decompressor import Decompressor +except ImportError: + print(f"Run as python3 -m util.n64.Miodecompress") + sys.exit(1) + + +class GenericMio0Decompressor(Decompressor): + def __init__( + self, unpacked_offset, compressed_offset, uncompressed_offset, header_length + ): + self.unpacked_offset = unpacked_offset + self.compressed_offset = compressed_offset + self.uncompressed_offset = uncompressed_offset + self.header_length = header_length + + @staticmethod + def read_word(data, offset): + (res,) = struct.unpack(">I", data[offset : offset + 4]) + return res + + @staticmethod + def read_short(data, offset): + (res,) = struct.unpack(">H", data[offset : offset + 2]) + return res + + def decompress(self, in_bytes, byte_order="big") -> bytearray: + magic = in_bytes[0:4] + if magic != b"MIO0": + log.error(f"MIO0 magic is incorrect: {magic}") + + unpacked_size = self.read_word(in_bytes, self.unpacked_offset) + comp_offset = self.read_word(in_bytes, self.compressed_offset) + uncomp_offset = self.read_word(in_bytes, self.uncompressed_offset) + + layout_data = struct.iter_unpack(">I", in_bytes[self.header_length :]) + uncompressed_data = struct.iter_unpack(">B", in_bytes[uncomp_offset:]) + compressed_data = struct.iter_unpack(">H", in_bytes[comp_offset:]) + + idx = 0 + ret = bytearray(unpacked_size) + + mask_bit_counter = 0 + while idx < unpacked_size: + if mask_bit_counter == 0: + (current_mask,) = next(layout_data) + mask_bit_counter = 32 + + if current_mask & 0x80000000: + (ud,) = next(uncompressed_data) + ret[idx] = ud + idx += 1 + else: + (length_offset,) = next(compressed_data) + + length = (length_offset >> 12) + 3 + index = (length_offset & 0xFFF) + 1 + offset = idx - index + + if not (3 <= length <= 18): + log.error(f"Invalid length: {length}, corrupt data?") + + if not (1 <= index <= 4096): + log.error(f"Invalid index: {index}, corrupt data?") + + for i in range(length): + ret[idx] = ret[offset + i] + idx += 1 + + current_mask <<= 1 + mask_bit_counter -= 1 + + return ret + + +class Mio0Decompressor(GenericMio0Decompressor): + def __init__(self): + super().__init__( + unpacked_offset=4, + compressed_offset=8, + uncompressed_offset=12, + header_length=16, + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("infile") + parser.add_argument("outfile") + args = parser.parse_args() + + with open(args.infile, "rb") as f: + raw_bytes = f.read() + + miodecompress = Mio0Decompressor() + decompressed = miodecompress.decompress(raw_bytes) + + with open(args.outfile, "wb") as f: + f.write(decompressed) + + +if __name__ == "__main__": + main() diff --git a/tools/splat/util/n64/Yay0decompress.c b/tools/splat/util/n64/Yay0decompress.c new file mode 100644 index 0000000..250f347 --- /dev/null +++ b/tools/splat/util/n64/Yay0decompress.c @@ -0,0 +1,44 @@ +#include <stdio.h> +#include <stdint.h> +#include <stdbool.h> +#include <stdlib.h> + +typedef struct { + uint32_t magic; + uint32_t uncompressedLength; + uint32_t opPtr; + uint32_t dataPtr; +} Yay0Header; + +void decompress(Yay0Header* hdr, uint8_t* srcPtr, uint8_t* dstPtr, bool isBigEndian) { + uint8_t byte = 0, mask = 0; + uint8_t* ctrl, * ops, * data; + uint16_t copy, op; + uint32_t written = 0; + + ctrl = srcPtr + sizeof(Yay0Header); + ops = srcPtr + hdr->opPtr; + data = srcPtr + hdr->dataPtr; + + while (written < hdr->uncompressedLength) { + if ((mask >>= 1) == 0) { + byte = *ctrl++; + mask = 0x80; + } + + if (byte & mask) { + *dstPtr++ = *data++; + written++; + } else { + op = isBigEndian ? (ops[0] << 8) | ops[1] : (ops[1] << 8) | ops[0]; + ops += 2; + + written += copy = (op >> 12) ? (2 + (op >> 12)) : (18 + *data++); + + while (copy--) { + *dstPtr = dstPtr[-(op & 0xfff) - 1]; + dstPtr++; + } + } + } +} diff --git a/tools/splat/util/n64/Yay0decompress.py b/tools/splat/util/n64/Yay0decompress.py new file mode 100644 index 0000000..ca24c23 --- /dev/null +++ b/tools/splat/util/n64/Yay0decompress.py @@ -0,0 +1,162 @@ +import argparse +import os +import sys +from ctypes import * +from struct import pack, unpack_from +from typing import Optional + +try: + from .. import log + from .decompressor import Decompressor +except ImportError: + print(f"Run as python3 -m util.n64.Yay0decompress") + sys.exit(1) + +tried_loading = False +lib: Optional[CDLL] = None + + +def setup_lib(): + global lib + global tried_loading + if lib: + return True + if tried_loading: + return False + try: + tried_loading = True + lib = cdll.LoadLibrary( + os.path.dirname(os.path.realpath(__file__)) + "/Yay0decompress" + ) + return True + except Exception: + log.write( + "Failed to load Yay0 C library; falling back to Python method", + status="warn", + ) + tried_loading = True + return False + + +class Yay0Decompressor(Decompressor): + def decompress(self, in_bytes, byte_order="big") -> bytearray: + # attempt to load the library only once per execution + global lib + if not setup_lib(): + return self.decompress_python(in_bytes, byte_order) + assert lib is not None + + class Yay0(Structure): + _fields_ = [ + ("magic", c_uint32), + ("uncompressedLength", c_uint32), + ("opPtr", c_uint32), + ("dataPtr", c_uint32), + ] + + # read the file header + bigEndian = byte_order == "big" + if bigEndian: + # the struct is only a view, so when passed to C it will keep + # its BigEndian values and crash. Explicitly convert them here to little + hdr = Yay0.from_buffer_copy( + pack("<IIII", *unpack_from(">IIII", in_bytes, 0)) + ) + else: + hdr = Yay0.from_buffer_copy(in_bytes, 0) + + magic = getattr(hdr, hdr._fields_[0][0]) + if magic != int.from_bytes(str.encode("Yay0"), byteorder="big"): + log.error(f"Yay0 magic is incorrect: {magic}") + + # create the input/output buffers, copying data to in + src = (c_uint8 * len(in_bytes)).from_buffer_copy(in_bytes, 0) + dst = (c_uint8 * hdr.uncompressedLength)() + + # call decompress, equivilant to, in C: + # decompress(&hdr, &src, &dst, bigEndian) + lib.decompress(byref(hdr), byref(src), byref(dst), c_bool(bigEndian)) + + # other functions want the results back as a non-ctypes type + return bytearray(dst) + + @staticmethod + def decompress_python(in_bytes, byte_order="big"): + if in_bytes[:4] != b"Yay0": + log.error("Input file is not Yay0") + + decompressed_size = int.from_bytes(in_bytes[4:8], byteorder=byte_order) + link_table_offset = int.from_bytes(in_bytes[8:12], byteorder=byte_order) + chunk_offset = int.from_bytes(in_bytes[12:16], byteorder=byte_order) + + link_table_idx = link_table_offset + chunk_idx = chunk_offset + other_idx = 16 + + mask_bit_counter = 0 + current_mask = 0 + + # preallocate result and index into it + idx = 0 + ret = bytearray(decompressed_size) + + while idx < decompressed_size: + # If we're out of bits, get the next mask + if mask_bit_counter == 0: + current_mask = int.from_bytes( + in_bytes[other_idx : other_idx + 4], byteorder=byte_order + ) + other_idx += 4 + mask_bit_counter = 32 + + if current_mask & 0x80000000: + ret[idx] = in_bytes[chunk_idx] + idx += 1 + chunk_idx += 1 + else: + link = int.from_bytes( + in_bytes[link_table_idx : link_table_idx + 2], byteorder=byte_order + ) + link_table_idx += 2 + + offset = idx - (link & 0xFFF) + + count = link >> 12 + + if count == 0: + count_modifier = in_bytes[chunk_idx] + chunk_idx += 1 + count = count_modifier + 18 + else: + count += 2 + + # Copy the block + for i in range(count): + ret[idx] = ret[offset + i - 1] + idx += 1 + + current_mask <<= 1 + mask_bit_counter -= 1 + + return ret + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("infile") + parser.add_argument("outfile") + parser.add_argument("--byte-order", default="big", choices=["big", "little"]) + args = parser.parse_args() + + with open(args.infile, "rb") as f: + raw_bytes = f.read() + + yay0decompressor = Yay0Decompressor() + decompressed = yay0decompressor.decompress(raw_bytes, args.byte_order) + + with open(args.outfile, "wb") as f: + f.write(decompressed) + + +if __name__ == "__main__": + main() diff --git a/tools/splat/util/n64/__init__.py b/tools/splat/util/n64/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/splat/util/n64/__init__.py diff --git a/tools/splat/util/n64/decompressor.py b/tools/splat/util/n64/decompressor.py new file mode 100644 index 0000000..cc5ddb8 --- /dev/null +++ b/tools/splat/util/n64/decompressor.py @@ -0,0 +1,7 @@ +from abc import ABC, abstractmethod + + +class Decompressor(ABC): + @abstractmethod + def decompress(self, in_bytes, byte_order="big") -> bytearray: + pass diff --git a/tools/splat/util/n64/find_code_length.py b/tools/splat/util/n64/find_code_length.py new file mode 100755 index 0000000..1f9053c --- /dev/null +++ b/tools/splat/util/n64/find_code_length.py @@ -0,0 +1,63 @@ +#! /usr/bin/env python3 + +import argparse + +import rabbitizer +import spimdisasm + + +def int_any_base(x): + return int(x, 0) + + +parser = argparse.ArgumentParser( + description="Given a rom and start offset, find where the code ends" +) +parser.add_argument("rom", help="path to a .z64 rom") +parser.add_argument("start", help="start offset", type=int_any_base) +parser.add_argument("--end", help="end offset", default=None, type=int_any_base) +parser.add_argument( + "--vram", + help="vram address to start disassembly at", + default="0x80000000", + type=int_any_base, +) + + +def run(rom_bytes, start_offset, vram, end_offset=None): + rom_addr = start_offset + last_return = rom_addr + + wordList = spimdisasm.common.Utils.bytesToBEWords(rom_bytes[start_offset:]) + for word in wordList: + insn = rabbitizer.Instruction(word) + if not insn.isImplemented(): + break + + if insn.isJrRa(): + last_return = rom_addr + rom_addr += 4 + if end_offset and rom_addr >= end_offset: + break + + # align to next 0x10 boundary + end = last_return + 0x10 + end -= end % 0x10 + return end + + +def main(): + args = parser.parse_args() + + with open(args.rom, "rb") as f: + rom_bytes = f.read() + + start = args.start + end = args.end + vram = args.vram + + print(f"0x{run(rom_bytes, start, vram, end):X}") + + +if __name__ == "__main__": + main() diff --git a/tools/splat/util/n64/rominfo.py b/tools/splat/util/n64/rominfo.py new file mode 100755 index 0000000..fae1fee --- /dev/null +++ b/tools/splat/util/n64/rominfo.py @@ -0,0 +1,322 @@ +#! /usr/bin/env python3 + +import argparse + +import hashlib +import itertools +import struct + +import sys +import zlib +from dataclasses import dataclass + +from pathlib import Path +from typing import Optional + +import rabbitizer +import spimdisasm + +parser = argparse.ArgumentParser(description="Gives information on N64 roms") +parser.add_argument("rom", help="path to an N64 rom") +parser.add_argument( + "--header-encoding", + dest="header_encoding", + help=( + "Text encoding the game header is using;" + " see docs.python.org/3/library/codecs.html#standard-encodings for valid encodings" + ), +) + +country_codes = { + 0x00: "Unknown", + 0x37: "Beta", + 0x41: "Asian (NTSC)", + 0x42: "Brazillian", + 0x43: "Chiniese", + 0x44: "German", + 0x45: "North America", + 0x46: "French", + 0x47: "Gateway 64 (NTSC)", + 0x48: "Dutch", + 0x49: "Italian", + 0x4A: "Japanese", + 0x4B: "Korean", + 0x4C: "Gateway 64 (PAL)", + 0x4E: "Canadian", + 0x50: "European (basic spec.)", + 0x53: "Spanish", + 0x55: "Australian", + 0x57: "Scandanavian", + 0x58: "European", + 0x59: "European", +} + + +@dataclass +class CIC: + ntsc_name: str + pal_name: str + offset: int + + +crc_to_cic = { + 0x6170A4A1: CIC("6101", "7102", 0x000000), + 0x90BB6CB5: CIC("6102", "7101", 0x000000), + 0x0B050EE0: CIC("6103", "7103", 0x100000), + 0x98BC2C86: CIC("6105", "7105", 0x000000), + 0xACC8580A: CIC("6106", "7106", 0x200000), +} +unknown_cic = CIC("unknown", "unknown", 0x0000000) + + +@dataclass +class N64EntrypointInfo: + entry_size: int + bss_start_address: Optional[int] + bss_size: Optional[int] + main_address: Optional[int] + stack_top: int + + @staticmethod + def parse_rom_bytes( + rom_bytes, offset: int = 0x1000, size: int = 0x60 + ) -> "N64EntrypointInfo": + word_list = spimdisasm.common.Utils.bytesToWords( + rom_bytes, offset, offset + size + ) + nops_count = 0 + + register_values = [0 for _ in range(32)] + + register_bss_address: Optional[int] = None + register_bss_size: Optional[int] = None + register_main_address: Optional[int] = None + + size = 0 + for word in word_list: + insn = rabbitizer.Instruction(word) + if not insn.isImplemented(): + break + + if insn.isNop(): + nops_count += 1 + elif nops_count >= 3: + break + elif insn.canBeHi(): + register_values[insn.rt.value] = insn.getProcessedImmediate() << 16 + elif insn.canBeLo(): + if insn.isLikelyHandwritten(): + # Try to skip this instructions: + # addi $t0, $t0, 0x8 + # addi $t1, $t1, -0x8 + pass + elif insn.modifiesRt(): + register_values[insn.rt.value] = ( + register_values[insn.rs.value] + insn.getProcessedImmediate() + ) + elif insn.doesStore(): + if insn.rt == rabbitizer.RegGprO32.zero: + # Try to detect the zero-ing bss algorithm + # sw $zero, 0x0($t0) + register_bss_address = insn.rs.value + elif insn.isBranch(): + # lui $t1, 0x2 + # addiu $t1, $t1, -0x7220 + # ... + # addi $t1, $t1, -0x8 + # ... + # bnez $t1, label + register_bss_size = insn.rs.value + + elif insn.isJumptableJump() or insn.isReturn(): + # lui $t2, 0x8000 + # addiu $t2, $t2, 0x494 + # ... + # jr $t2 + register_main_address = insn.rs.value + + # print(f"{word:08X}", insn) + size += 4 + + # for i, val in enumerate(register_values): + # print(i, f"{val:08X}") + + bss_address = ( + register_values[register_bss_address] + if register_bss_address is not None + else None + ) + bss_size = ( + register_values[register_bss_size] + if register_bss_size is not None + else None + ) + main_address = ( + register_values[register_main_address] + if register_main_address is not None + else None + ) + stack_top = register_values[rabbitizer.RegGprO32.sp.value] + return N64EntrypointInfo(size, bss_address, bss_size, main_address, stack_top) + + +@dataclass +class N64Rom: + name: str + header_encoding: str + country_code: int + libultra_version: str + checksum: str + cic: CIC + entry_point: int + size: int + compiler: str + sha1: str + entrypoint_info: N64EntrypointInfo + + def get_country_name(self) -> str: + return country_codes[self.country_code] + + +def swap_bytes(data): + return bytes( + itertools.chain.from_iterable( + struct.pack(">H", x) for (x,) in struct.iter_unpack("<H", data) + ) + ) + + +def read_rom(rom_path: Path): + rom_bytes = rom_path.read_bytes() + + if rom_path.suffix.lower() == ".n64": + print("Warning: Input file has .n64 suffix, byte-swapping!") + rom_bytes = swap_bytes(rom_bytes) + as_z64 = rom_path.with_suffix(".z64") + if not as_z64.exists(): + print(f"Writing down {as_z64}") + as_z64.write_bytes(rom_bytes) + return rom_bytes + + +def get_cic(rom_bytes: bytes): + ipl3_crc = zlib.crc32(rom_bytes[0x40:0x1000]) + + return crc_to_cic.get(ipl3_crc, unknown_cic) + + +def get_entry_point(program_counter: int, cic: CIC): + return program_counter - cic.offset + + +def guess_header_encoding(rom_bytes: bytes): + header = rom_bytes[0x20:0x34] + encodings = ["ASCII", "shift_jis", "euc-jp"] + for encoding in encodings: + try: + header.decode(encoding) + return encoding + except UnicodeDecodeError: + # we guessed wrong... + pass + + sys.exit("Unknown header encoding, please raise an Issue with us") + + +def get_info( + rom_path: Path, rom_bytes: Optional[bytes] = None, header_encoding=None +) -> N64Rom: + if rom_bytes is None: + rom_bytes = read_rom(rom_path) + + if header_encoding is None: + header_encoding = guess_header_encoding(rom_bytes) + + return get_info_bytes(rom_bytes, header_encoding) + + +def get_info_bytes(rom_bytes: bytes, header_encoding: str) -> N64Rom: + (program_counter,) = struct.unpack(">I", rom_bytes[0x8:0xC]) + libultra_version = chr(rom_bytes[0xF]) + checksum = rom_bytes[0x10:0x18].hex().upper() + + try: + name = rom_bytes[0x20:0x34].decode(header_encoding).strip() + except: + sys.exit( + "splat could not decode the game name;" + " try using a different encoding by passing the --header-encoding argument" + " (see docs.python.org/3/library/codecs.html#standard-encodings for valid encodings)" + ) + + country_code = rom_bytes[0x3E] + + cic = get_cic(rom_bytes) + entry_point = get_entry_point(program_counter, cic) + + compiler = get_compiler_info(rom_bytes, entry_point, print_result=False) + + sha1 = hashlib.sha1(rom_bytes).hexdigest() + + entrypoint_info = N64EntrypointInfo.parse_rom_bytes(rom_bytes) + + return N64Rom( + name, + header_encoding, + country_code, + libultra_version, + checksum, + cic, + entry_point, + len(rom_bytes), + compiler, + sha1, + entrypoint_info, + ) + + +def get_compiler_info(rom_bytes, entry_point, print_result=True): + jumps = 0 + branches = 0 + + word_list = spimdisasm.common.Utils.bytesToWords(rom_bytes[0x1000:]) + for word in word_list: + insn = rabbitizer.Instruction(word) + if not insn.isImplemented(): + break + + if insn.uniqueId == rabbitizer.InstrId.cpu_j: + jumps += 1 + elif insn.uniqueId == rabbitizer.InstrId.cpu_b: + branches += 1 + + compiler = "IDO" if branches > jumps else "GCC" + if print_result: + print( + f"{branches} branches and {jumps} jumps detected in the first code segment." + f" Compiler is most likely {compiler}" + ) + return compiler + + +def main(): + rabbitizer.config.pseudos_pseudoB = True + + args = parser.parse_args() + rom_bytes = read_rom(Path(args.rom)) + rom = get_info(Path(args.rom), rom_bytes, args.header_encoding) + + print("Image name: " + rom.name) + print("Country code: " + chr(rom.country_code) + " - " + rom.get_country_name()) + print("Libultra version: " + rom.libultra_version) + print("Checksum: " + rom.checksum) + print("CIC: " + rom.cic.ntsc_name + " / " + rom.cic.pal_name) + print("RAM entry point: " + hex(rom.entry_point)) + print("Header encoding: " + rom.header_encoding) + print("") + + get_compiler_info(rom_bytes, rom.entry_point) + + +if __name__ == "__main__": + main() diff --git a/tools/splat/util/options.py b/tools/splat/util/options.py new file mode 100644 index 0000000..a8a3eba --- /dev/null +++ b/tools/splat/util/options.py @@ -0,0 +1,422 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import cast, Dict, List, Literal, Mapping, Optional, Set, Type, TypeVar + +from util import compiler +from util.compiler import Compiler + + +@dataclass +class SplatOpts: + # Debug / logging + verbose: bool + dump_symbols: bool + modes: List[str] + + # Project configuration + + # Determines the base path of the project. Everything is relative to this path + base_path: Path + # Determines the path to the target binary + target_path: Path + # Determines the platform of the target binary + platform: str + # Determines the compiler used to compile the target binary + compiler: Compiler + # Determines the endianness of the target binary + endianness: Literal["big", "little"] + # Determines the default section order of the target binary + # this can be overridden per-segment + section_order: List[str] + # Determines the code that is inserted by default in generated .c files + generated_c_preamble: str + # Determines the code that is inserted by default in generated .s files + generated_s_preamble: str + # Determines whether to use .o as the suffix for all binary files?... TODO document + use_o_as_suffix: bool + # the value of the $gp register to correctly calculate offset to %gp_rel relocs + gp: Optional[int] + + # Paths + asset_path: Path + # Determines the path to the symbol addresses file(s) + # A symbol_addrs file is to be updated/curated manually and contains addresses of symbols + # as well as optional metadata such as rom address, type, and more + # + # It's possible to use more than one file by supplying a list instead of a string + symbol_addrs_paths: List[Path] + reloc_addrs_paths: List[Path] + # Determines the path to the project build directory + build_path: Path + # Determines the path to the source code directory + src_path: Path + # Determines the path to the asm code directory + asm_path: Path + # Determines the path to the asm data directory + data_path: Path + # Determines the path to the asm nonmatchings directory + nonmatchings_path: Path + # Determines the path to the cache file (used when supplied --use-cache via the CLI) + cache_path: Path + + # Determines whether to create an automatically-generated undefined functions file + # this file stores all functions that are referenced in the code but are not defined as seen by splat + create_undefined_funcs_auto: bool + # Determines the path to the undefined_funcs_auto file + undefined_funcs_auto_path: Path + + # Determines whether to create an automatically-generated undefined symbols file + # this file stores all symbols that are referenced in the code but are not defined as seen by splat + create_undefined_syms_auto: bool + # Determines the path to the undefined_symbols_auto file + undefined_syms_auto_path: Path + + # Determines the path in which to search for custom splat extensions + extensions_path: Optional[Path] + + # Determines the path to library files that are to be linked into the target binary + lib_path: Path + + # TODO document + elf_section_list_path: Optional[Path] + + # Linker script + # Determines the default subalign value to be specified in the generated linker script + subalign: int + # The following option determines whether to automatically configure the linker script to link against + # specified sections for all "base" (asm/c) files when the yaml doesn't have manual configurations + # for these sections. + auto_all_sections: List[str] + # Determines the desired path to the linker script that splat will generate + ld_script_path: Path + # Determines the desired path to the linker symbol header, + # which exposes externed definitions for all segment ram/rom start/end locations + ld_symbol_header_path: Optional[Path] + # Determines whether to add a discard section to the linker script + ld_discard_section: bool + # Determines the list of section labels that are to be added to the linker script + ld_section_labels: List[str] + # Determines whether to add wildcards for section linking in the linker script (.rodata* for example) + ld_wildcard_sections: bool + # Determines whether to use use "follows" settings to determine locations of overlays in the linker script. + # If disabled, this effectively ignores "follows" directives in the yaml. + ld_use_follows: bool + + ################################################################################ + # C file options + ################################################################################ + # Determines whether to create new c files if they don't exist + create_c_files: bool + # Determines whether to "auto-decompile" empty functions + auto_decompile_empty_functions: bool + # Determines whether to detect matched/unmatched functions in existing c files + # so we can avoid creating .s files for already-decompiled functions + do_c_func_detection: bool + # Determines the newline char(s) to be used in c files + c_newline: str + + ################################################################################ + # (Dis)assembly-related options + ################################################################################ + # The following options determine the format that symbols should be named by default + symbol_name_format: str + # Same as above but for symbols with no rom address + symbol_name_format_no_rom: str + # Determines whether to detect and hint to the user about likely file splits when disassembling + find_file_boundaries: bool + # Determines whether to detect and hint to the user about possible rodata sections corresponding to a text section + pair_rodata_to_text: bool + # Determines whether to attempt to automatically migrate rodata into functions + # (only works in certain circumstances) + migrate_rodata_to_functions: bool + # Determines the header to be used in every asm file that's included from c files + asm_inc_header: str + # Determines the macro used to declare functions in asm files + asm_function_macro: str + # Determines the macro used to declare jumptable labels in asm files + asm_jtbl_label_macro: str + # Determines the macro used to declare data symbols in asm files + asm_data_macro: str + # Determines the macro used at the end of a function, such as endlabel or .end + asm_end_label: str + # Determines including the macro.inc file on non-migrated rodata variables + include_macro_inc: bool + # Determines the number of characters to left align before the TODO finish documenting + mnemonic_ljust: int + # Determines whether to pad the rom address + rom_address_padding: bool + # Determines which ABI names to use for general purpose registers + mips_abi_gpr: str + # Determines which ABI names to use for floating point registers + # Valid values: 'numeric', 'o32', 'n32', 'n64' + # o32 is highly recommended, as it provides logically named registers for floating point instructions + # For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d + mips_abi_float_regs: str + # Determines whether to ad ".set gp=64 to asm/hasm files" + add_set_gp_64: bool + # Generate .asmproc.d dependency files for each C file which still reference functions in assembly files + create_asm_dependencies: bool + # Global option for rodata string encoding. This can be overriden per segment + string_encoding: Optional[str] + # Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol + allow_data_addends: bool + + ################################################################################ + # N64-specific options + ################################################################################ + # Determines the encoding of the header + header_encoding: str + # Determines the type gfx ucode (used by gfx segments) + # Valid options are ['f3d', 'f3db', 'f3dex', 'f3dexb', 'f3dex2'] + gfx_ucode: str + # Use named libultra symbols by default. Those will need to be added to a linker script manually by the user + libultra_symbols: bool + # Use named hardware register symbols by default. Those will need to be added to a linker script manually by the user + hardware_regs: bool + + ################################################################################ + # Gamecube-specific options + ################################################################################ + # Path where the iso's filesystem will be extracted to + filesystem_path: Optional[Path] + + ################################################################################ + # Compiler-specific options + ################################################################################ + # Determines whether to use a legacy INCLUDE_ASM macro format in c files + # only applies to GCC/SN64 + use_legacy_include_asm: bool + + # Returns whether the given mode is currently enabled + def is_mode_active(self, mode: str) -> bool: + return mode in self.modes or "all" in self.modes + + +opts: SplatOpts + + +T = TypeVar("T") + + +class OptParser: + _read_opts: Set[str] + + def __init__(self, yaml: Mapping[str, object]) -> None: + self._yaml = yaml + self._read_opts = set() + + def parse_opt(self, opt: str, t: Type[T], default: Optional[T] = None) -> T: + if opt not in self._yaml: + if default is not None: + return default + raise ValueError(f"Missing required option {opt}") + self._read_opts.add(opt) + value = self._yaml[opt] + if isinstance(value, t): + return value + if t is float and isinstance(value, int): + return cast(T, float(value)) + raise ValueError(f"Expected {opt} to have type {t}, got {type(value)}") + + def parse_optional_opt(self, opt: str, t: Type[T]) -> Optional[T]: + if opt not in self._yaml: + return None + return self.parse_opt(opt, t) + + def parse_opt_within( + self, opt: str, t: Type[T], within: List[T], default: Optional[T] = None + ) -> T: + value = self.parse_opt(opt, t, default) + if value not in within: + raise ValueError(f"Invalid value for {opt}: {value}") + return value + + def parse_path( + self, base_path: Path, opt: str, default: Optional[str] = None + ) -> Path: + return base_path / Path(self.parse_opt(opt, str, default)) + + def parse_optional_path(self, base_path: Path, opt: str) -> Optional[Path]: + if opt not in self._yaml: + return None + return self.parse_path(base_path, opt) + + def parse_path_list(self, base_path: Path, opt: str, default: str) -> List[Path]: + paths = self.parse_opt(opt, object, default) + + if isinstance(paths, str): + return [base_path / paths] + elif isinstance(paths, list): + return [base_path / path for path in paths] + else: + raise ValueError(f"Expected str or list for '{opt}', got {type(paths)}") + + def check_no_unread_opts(self) -> None: + opts = [opt for opt in self._yaml if opt not in self._read_opts] + if opts: + raise ValueError(f"Unrecognized YAML option(s): {', '.join(opts)}") + + +def _parse_yaml( + yaml: Dict, + config_paths: List[str], + modes: List[str], + verbose: bool = False, +) -> SplatOpts: + p = OptParser(yaml) + + basename = p.parse_opt("basename", str) + platform = p.parse_opt_within("platform", str, ["n64", "psx", "gc", "ps2"]) + comp = compiler.for_name(p.parse_opt("compiler", str, "IDO")) + + base_path = Path(config_paths[0]).parent / p.parse_opt("base_path", str) + asm_path: Path = p.parse_path(base_path, "asm_path", "asm") + + def parse_endianness() -> Literal["big", "little"]: + endianness = p.parse_opt_within( + "endianness", + str, + ["big", "little"], + "little" if platform in ["psx", "ps2"] else "big", + ) + + if endianness == "big": + return "big" + elif endianness == "little": + return "little" + else: + raise ValueError(f"Invalid endianness: {endianness}") + + ret = SplatOpts( + verbose=verbose, + dump_symbols=p.parse_opt("dump_symbols", bool, False), + modes=modes, + base_path=base_path, + target_path=p.parse_path(base_path, "target_path"), + platform=platform, + compiler=comp, + endianness=parse_endianness(), + section_order=p.parse_opt( + "section_order", list, [".text", ".data", ".rodata", ".bss"] + ), + generated_c_preamble=p.parse_opt( + "generated_c_preamble", str, '#include "common.h"' + ), + generated_s_preamble=p.parse_opt("generated_s_preamble", str, ""), + use_o_as_suffix=p.parse_opt("o_as_suffix", bool, False), + gp=p.parse_opt("gp_value", int, 0), + asset_path=p.parse_path(base_path, "asset_path", "assets"), + symbol_addrs_paths=p.parse_path_list( + base_path, "symbol_addrs_path", "symbol_addrs.txt" + ), + reloc_addrs_paths=p.parse_path_list( + base_path, "reloc_addrs_path", "reloc_addrs.txt" + ), + build_path=p.parse_path(base_path, "build_path", "build"), + src_path=p.parse_path(base_path, "src_path", "src"), + asm_path=asm_path, + data_path=p.parse_path(asm_path, "data_path", "data"), + nonmatchings_path=p.parse_path(asm_path, "nonmatchings_path", "nonmatchings"), + cache_path=p.parse_path(base_path, "cache_path", ".splache"), + create_undefined_funcs_auto=p.parse_opt( + "create_undefined_funcs_auto", bool, True + ), + undefined_funcs_auto_path=p.parse_path( + base_path, "undefined_funcs_auto_path", "undefined_funcs_auto.txt" + ), + create_undefined_syms_auto=p.parse_opt( + "create_undefined_syms_auto", bool, True + ), + undefined_syms_auto_path=p.parse_path( + base_path, "undefined_syms_auto_path", "undefined_syms_auto.txt" + ), + extensions_path=p.parse_optional_path(base_path, "extensions_path"), + lib_path=p.parse_path(base_path, "lib_path", "lib"), + elf_section_list_path=p.parse_optional_path(base_path, "elf_section_list_path"), + subalign=p.parse_opt("subalign", int, 16), + auto_all_sections=p.parse_opt( + "auto_all_sections", list, [".data", ".rodata", ".bss"] + ), + ld_script_path=p.parse_path(base_path, "ld_script_path", f"{basename}.ld"), + ld_symbol_header_path=p.parse_optional_path(base_path, "ld_symbol_header_path"), + ld_discard_section=p.parse_opt("ld_discard_section", bool, True), + ld_section_labels=p.parse_opt( + "ld_section_labels", + list, + [".text", ".data", ".rodata", ".bss"], + ), + ld_wildcard_sections=p.parse_opt("ld_wildcard_sections", bool, False), + ld_use_follows=p.parse_opt("ld_use_follows", bool, True), + create_c_files=p.parse_opt("create_c_files", bool, True), + auto_decompile_empty_functions=p.parse_opt( + "auto_decompile_empty_functions", bool, True + ), + do_c_func_detection=p.parse_opt("do_c_func_detection", bool, True), + c_newline=p.parse_opt("c_newline", str, comp.c_newline), + symbol_name_format=p.parse_opt("symbol_name_format", str, "$VRAM"), + symbol_name_format_no_rom=p.parse_opt( + "symbol_name_format_no_rom", str, "$VRAM_$SEG" + ), + find_file_boundaries=p.parse_opt("find_file_boundaries", bool, True), + pair_rodata_to_text=p.parse_opt("pair_rodata_to_text", bool, True), + migrate_rodata_to_functions=p.parse_opt( + "migrate_rodata_to_functions", bool, True + ), + asm_inc_header=p.parse_opt("asm_inc_header", str, comp.asm_inc_header), + asm_function_macro=p.parse_opt( + "asm_function_macro", str, comp.asm_function_macro + ), + asm_jtbl_label_macro=p.parse_opt( + "asm_jtbl_label_macro", str, comp.asm_jtbl_label_macro + ), + asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro), + asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label), + include_macro_inc=p.parse_opt( + "include_macro_inc", bool, comp.include_macro_inc + ), + mnemonic_ljust=p.parse_opt("mnemonic_ljust", int, 11), + rom_address_padding=p.parse_opt("rom_address_padding", bool, False), + mips_abi_gpr=p.parse_opt_within( + "mips_abi_gpr", + str, + ["numeric", "o32", "n32", "n64"], + "o32", + ), + mips_abi_float_regs=p.parse_opt_within( + "mips_abi_float_regs", + str, + ["numeric", "o32", "n32", "n64"], + "numeric", + ), + add_set_gp_64=p.parse_opt("add_set_gp_64", bool, True), + create_asm_dependencies=p.parse_opt("create_asm_dependencies", bool, False), + string_encoding=p.parse_optional_opt("string_encoding", str), + allow_data_addends=p.parse_opt("allow_data_addends", bool, True), + header_encoding=p.parse_opt("header_encoding", str, "ASCII"), + gfx_ucode=p.parse_opt_within( + "gfx_ucode", + str, + ["f3d", "f3db", "f3dex", "f3dexb", "f3dex2"], + "f3dex2", + ), + libultra_symbols=p.parse_opt("libultra_symbols", bool, False), + hardware_regs=p.parse_opt("hardware_regs", bool, False), + use_legacy_include_asm=p.parse_opt("use_legacy_include_asm", bool, True), + filesystem_path=p.parse_optional_path(base_path, "filesystem_path"), + ) + p.check_no_unread_opts() + return ret + + +def initialize( + config: Dict, + config_paths: List[str], + modes: Optional[List[str]] = None, + verbose=False, +): + global opts + + if not modes: + modes = ["all"] + + opts = _parse_yaml(config["options"], config_paths, modes, verbose) diff --git a/tools/splat/util/palettes.py b/tools/splat/util/palettes.py new file mode 100644 index 0000000..8ee1e9a --- /dev/null +++ b/tools/splat/util/palettes.py @@ -0,0 +1,34 @@ +from typing import Dict + +from segtypes.common.group import CommonSegGroup +from segtypes.n64.ci import N64SegCi +from segtypes.n64.palette import N64SegPalette as Palette + + +# Resolve Raster#palette and Palette#raster links +def initialize(all_segments): + def process(segments): + raster_map: Dict[str, N64SegCi] = {} + palette_map: Dict[str, Palette] = {} + + for segment in segments: + if isinstance(segment, Palette): + palette_map[segment.name] = segment + + if isinstance(segment, N64SegCi): + raster_map[segment.name] = segment + + if isinstance(segment, CommonSegGroup): + process(segment.subsegments) + + for raster_name in raster_map: + raster = raster_map[raster_name] + # print(f"{raster_name} -> {raster.palette_name}") + raster.palette = palette_map.get(raster.palette_name) + + for palette_name in palette_map: + palette = palette_map[palette_name] + # print(f"{palette_name} -> {palette.raster_name}") + palette.raster = raster_map.get(palette.raster_name) + + process(all_segments) diff --git a/tools/splat/util/range.py b/tools/splat/util/range.py new file mode 100644 index 0000000..04aea6e --- /dev/null +++ b/tools/splat/util/range.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Range: + start: Optional[int] = None + end: Optional[int] = None + + def has_start(self): + return self.start is not None + + def has_end(self): + return self.end is not None + + def is_complete(self): + return self.has_start() and self.has_end() diff --git a/tools/splat/util/relocs.py b/tools/splat/util/relocs.py new file mode 100644 index 0000000..36cdb65 --- /dev/null +++ b/tools/splat/util/relocs.py @@ -0,0 +1,125 @@ +from dataclasses import dataclass +from typing import Dict + +import spimdisasm +import tqdm +from intervaltree import Interval, IntervalTree + +from util import log, options, symbols + + +@dataclass +class Reloc: + rom_address: int + reloc_type: str + symbol_name: str + + addend: int = 0 + + +all_relocs: Dict[int, Reloc] = {} + + +def add_reloc(reloc: Reloc): + all_relocs[reloc.rom_address] = reloc + + +def initialize(): + global all_relocs + + all_relocs = {} + + for path in options.opts.reloc_addrs_paths: + if not path.exists(): + continue + + with path.open() as f: + sym_addrs_lines = f.readlines() + for line_num, line in enumerate( + tqdm.tqdm(sym_addrs_lines, desc=f"Loading relocs ({path.stem})") + ): + line = line.strip() + # Allow comments + line = line.split("//")[0] + line = line.strip() + + if line == "": + continue + + rom_addr = None + reloc_type = None + symbol_name = None + addend = None + + for info in line.split(" "): + if ":" not in info: + continue + + if info.count(":") > 1: + log.parsing_error_preamble(path, line_num, line) + log.write(f"Too many ':'s in '{info}'") + log.error("") + + attr_name, attr_val = info.split(":") + if attr_name == "": + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Missing attribute name in '{info}', is there extra whitespace?" + ) + log.error("") + if attr_val == "": + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Missing attribute value in '{info}', is there extra whitespace?" + ) + log.error("") + + # Non-Boolean attributes + try: + if attr_name == "rom": + rom_addr = int(attr_val, 0) + continue + if attr_name == "reloc": + reloc_type = attr_val + continue + if attr_name == "symbol": + symbol_name = attr_val + continue + if attr_name == "addend": + addend = int(attr_val, 0) + continue + except: + log.parsing_error_preamble(path, line_num, line) + log.write(f"value of attribute '{attr_name}' could not be read:") + log.write("") + raise + + if rom_addr is None: + log.parsing_error_preamble(path, line_num, line) + log.error(f"Missing required 'rom' attribute for reloc") + if reloc_type is None: + log.parsing_error_preamble(path, line_num, line) + log.error(f"Missing required 'reloc' attribute for reloc") + if symbol_name is None: + log.parsing_error_preamble(path, line_num, line) + log.error(f"Missing required 'symbol' attribute for reloc") + + reloc = Reloc(rom_addr, reloc_type, symbol_name) + if addend is not None: + reloc.addend = addend + + add_reloc(reloc) + + +def initialize_spim_context(): + for rom_address, reloc in all_relocs.items(): + reloc_type = spimdisasm.common.RelocType.fromStr(reloc.reloc_type) + + if reloc_type is None: + log.error( + f"Reloc type '{reloc.reloc_type}' is not valid. Rom address: 0x{rom_address:X}" + ) + + symbols.spim_context.addGlobalReloc( + rom_address, reloc_type, reloc.symbol_name, addend=reloc.addend + ) diff --git a/tools/splat/util/symbols.py b/tools/splat/util/symbols.py new file mode 100644 index 0000000..07db03c --- /dev/null +++ b/tools/splat/util/symbols.py @@ -0,0 +1,640 @@ +from dataclasses import dataclass +import re +from typing import Dict, List, Optional, Set, TYPE_CHECKING + +import spimdisasm +import tqdm +from intervaltree import IntervalTree + +# circular import +if TYPE_CHECKING: + from segtypes.segment import Segment + +from util import log, options + +all_symbols: List["Symbol"] = [] +all_symbols_dict: Dict[int, List["Symbol"]] = {} +all_symbols_ranges = IntervalTree() +ignored_addresses: Set[int] = set() +to_mark_as_defined: Set[str] = set() +appears_after_overlays_syms: List["Symbol"] = [] + +# Initialize a spimdisasm context, used to store symbols and functions +spim_context = spimdisasm.common.Context() + +TRUEY_VALS = ["true", "on", "yes", "y"] +FALSEY_VALS = ["false", "off", "no", "n"] + +splat_sym_types = {"func", "jtbl", "jtbl_label", "label"} + + +def check_valid_type(typename: str) -> bool: + if typename[0].isupper(): + return True + + if typename in splat_sym_types: + return True + + if typename in spimdisasm.common.gKnownTypes: + return True + + return False + + +def is_truey(str: str) -> bool: + return str.lower() in TRUEY_VALS + + +def is_falsey(str: str) -> bool: + return str.lower() in FALSEY_VALS + + +def add_symbol(sym: "Symbol"): + all_symbols.append(sym) + if sym.vram_start is not None: + if sym.vram_start not in all_symbols_dict: + all_symbols_dict[sym.vram_start] = [] + all_symbols_dict[sym.vram_start].append(sym) + + # For larger symbols, add their ranges to interval trees for faster lookup + if sym.size > 4: + all_symbols_ranges.addi(sym.vram_start, sym.vram_end, sym) + + +def to_cname(symbol_name: str) -> str: + symbol_name = re.sub(r"[^0-9a-zA-Z_]", "_", symbol_name) + + if symbol_name[0] in ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]: + symbol_name = "_" + symbol_name + + return symbol_name + + +def initialize(all_segments: "List[Segment]"): + global all_symbols + global all_symbols_dict + global all_symbols_ranges + + all_symbols = [] + all_symbols_dict = {} + all_symbols_ranges = IntervalTree() + + def get_seg_for_name(name: str) -> Optional["Segment"]: + for segment in all_segments: + if segment.name == name: + return segment + return None + + # Manual list of func name / addrs + for path in options.opts.symbol_addrs_paths: + if path.exists(): + with open(path) as f: + sym_addrs_lines = f.readlines() + for line_num, line in enumerate( + tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})") + ): + line = line.strip() + if not line == "" and not line.startswith("//"): + comment_loc = line.find("//") + line_main = line + line_ext = "" + + if comment_loc != -1: + line_ext = line[comment_loc + 2 :].strip() + line_main = line[:comment_loc].strip() + + try: + line_split = line_main.split("=") + name = line_split[0].strip() + addr = int(line_split[1].strip()[:-1], 0) + except: + log.parsing_error_preamble(path, line_num, line) + log.write("Line should be of the form") + log.write( + "<function_name> = <address> // attr0:val0 attr1:val1 [...]" + ) + log.write("with <address> in hex preceded by 0x, or dec") + log.write("") + raise + + sym = Symbol(addr, given_name=name) + + ignore_sym = False + if line_ext: + for info in line_ext.split(" "): + if ":" in info: + if info.count(":") > 1: + log.parsing_error_preamble(path, line_num, line) + log.write(f"Too many ':'s in '{info}'") + log.error("") + + attr_name, attr_val = info.split(":") + if attr_name == "": + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Missing attribute name in '{info}', is there extra whitespace?" + ) + log.error("") + if attr_val == "": + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Missing attribute value in '{info}', is there extra whitespace?" + ) + log.error("") + + # Non-Boolean attributes + try: + if attr_name == "type": + if not check_valid_type(attr_val): + log.parsing_error_preamble( + path, line_num, line + ) + log.write( + f"Unrecognized symbol type in '{info}', it should be one of" + ) + log.write( + [ + *splat_sym_types, + *spimdisasm.common.gKnownTypes, + ] + ) + log.write( + "You may use a custom type that starts with a capital letter" + ) + log.error("") + type = attr_val + sym.type = type + continue + if attr_name == "size": + size = int(attr_val, 0) + sym.given_size = size + continue + if attr_name == "rom": + rom_addr = int(attr_val, 0) + sym.rom = rom_addr + continue + if attr_name == "segment": + seg = get_seg_for_name(attr_val) + if seg is None: + log.parsing_error_preamble( + path, line_num, line + ) + log.write( + f"Cannot find segment '{attr_val}'" + ) + log.error("") + else: + # Add segment to symbol + sym.segment = seg + continue + if attr_name == "name_end": + sym.given_name_end = attr_val + continue + if attr_name == "appears_after_overlays_addr": + sym.appears_after_overlays_addr = int( + attr_val, 0 + ) + appears_after_overlays_syms.append(sym) + continue + except: + log.parsing_error_preamble(path, line_num, line) + log.write( + f"value of attribute '{attr_name}' could not be read:" + ) + log.write("") + raise + + # Boolean attributes + tf_val = ( + True + if is_truey(attr_val) + else False + if is_falsey(attr_val) + else None + ) + if tf_val is None: + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of" + ) + log.write([*TRUEY_VALS, *FALSEY_VALS]) + log.error("") + else: + if attr_name == "dead": + sym.dead = tf_val + continue + if attr_name == "defined": + sym.defined = tf_val + continue + if attr_name == "extract": + sym.extract = tf_val + continue + if attr_name == "ignore": + ignore_sym = tf_val + continue + if attr_name == "force_migration": + sym.force_migration = tf_val + continue + if attr_name == "force_not_migration": + sym.force_not_migration = tf_val + continue + if attr_name == "allow_addend": + sym.allow_addend = tf_val + continue + if attr_name == "dont_allow_addend": + sym.dont_allow_addend = tf_val + continue + if ignore_sym: + if sym.given_size == None or sym.given_size == 0: + ignored_addresses.add(sym.vram_start) + else: + spim_context.addBannedSymbolRangeBySize( + sym.vram_start, sym.given_size + ) + + ignore_sym = False + continue + + if sym.segment: + sym.segment.add_symbol(sym) + + sym.user_declared = True + add_symbol(sym) + + +def initialize_spim_context(all_segments: "List[Segment]") -> None: + global_vrom_start = None + global_vrom_end = None + global_vram_start = None + global_vram_end = None + overlay_segments: Set[spimdisasm.common.SymbolsSegment] = set() + + spim_context.bannedSymbols |= ignored_addresses + + from segtypes.common.code import CommonSegCode + + for segment in all_segments: + if not isinstance(segment, CommonSegCode): + # We only care about the VRAMs of code segments + continue + + if segment.special_vram_segment: + # Special segments which should not be accounted in the global VRAM calculation, like N64's IPL3 + continue + + if ( + not isinstance(segment.vram_start, int) + or not isinstance(segment.vram_end, int) + or not isinstance(segment.rom_start, int) + or not isinstance(segment.rom_end, int) + ): + continue + + ram_id = segment.get_exclusive_ram_id() + if ram_id is None: + if global_vram_start is None: + global_vram_start = segment.vram_start + elif segment.vram_start < global_vram_start: + global_vram_start = segment.vram_start + + if global_vram_end is None: + global_vram_end = segment.vram_end + elif global_vram_end < segment.vram_end: + global_vram_end = segment.vram_end + + if global_vrom_start is None: + global_vrom_start = segment.rom_start + elif segment.rom_start < global_vrom_start: + global_vrom_start = segment.rom_start + + if global_vrom_end is None: + global_vrom_end = segment.rom_end + elif global_vrom_end < segment.rom_end: + global_vrom_end = segment.rom_end + + else: + spim_segment = spim_context.addOverlaySegment( + ram_id, + segment.rom_start, + segment.rom_end, + segment.vram_start, + segment.vram_end, + ) + # Add the segment-specific symbols first + for symbols_list in segment.seg_symbols.values(): + for sym in symbols_list: + add_symbol_to_spim_segment(spim_segment, sym) + + overlay_segments.add(spim_segment) + + if ( + global_vram_start is not None + and global_vram_end is not None + and global_vrom_start is not None + and global_vrom_end is not None + ): + spim_context.changeGlobalSegmentRanges( + global_vrom_start, global_vrom_end, global_vram_start, global_vram_end + ) + + # Check the vram range of the global segment does not overlap with any overlay segment + for ovl_segment in overlay_segments: + assert ( + ovl_segment.vramStart <= ovl_segment.vramEnd + ), f"{ovl_segment.vramStart:X} {ovl_segment.vramEnd:X}" + if ( + ovl_segment.vramEnd > global_vram_start + and global_vram_end > ovl_segment.vramStart + ): + log.write( + f"Warning: the vram range ([0x{ovl_segment.vramStart:X}, 0x{ovl_segment.vramEnd:X}]) of the non-global segment at rom address 0x{ovl_segment.vromStart:X} overlaps with the global vram range ([0x{global_vram_start:X}, 0x{global_vram_end:X}])", + status="warn", + ) + + # pass the global symbols to spimdisasm + for segment in all_segments: + if not isinstance(segment, CommonSegCode): + # We only care about the VRAMs of code segments + continue + + ram_id = segment.get_exclusive_ram_id() + if ram_id is not None: + continue + + for symbols_list in segment.seg_symbols.values(): + for sym in symbols_list: + add_symbol_to_spim_segment(spim_context.globalSegment, sym) + + +def add_symbol_to_spim_segment( + segment: spimdisasm.common.SymbolsSegment, sym: "Symbol" +) -> spimdisasm.common.ContextSymbol: + if sym.type == "func": + context_sym = segment.addFunction( + sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom + ) + elif sym.type == "jtbl": + context_sym = segment.addJumpTable( + sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom + ) + elif sym.type == "jtbl_label": + context_sym = segment.addJumpTableLabel( + sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom + ) + elif sym.type == "label": + context_sym = segment.addBranchLabel( + sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom + ) + else: + context_sym = segment.addSymbol( + sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom + ) + if sym.type is not None: + context_sym.type = sym.type + + if sym.user_declared: + context_sym.isUserDeclared = True + if sym.defined: + context_sym.isDefined = True + if sym.rom is not None: + context_sym.vromAddress = sym.rom + if sym.given_size is not None: + context_sym.size = sym.size + if sym.force_migration: + context_sym.forceMigration = True + if sym.force_not_migration: + context_sym.forceNotMigration = True + if sym.allow_addend: + context_sym.allowedToReferenceAddends = True + if sym.dont_allow_addend: + context_sym.notAllowedToReferenceAddends = True + context_sym.setNameGetCallbackIfUnset(lambda _: sym.name) + if sym.given_name_end: + context_sym.nameEnd = sym.given_name_end + + return context_sym + + +def add_symbol_to_spim_section( + section: spimdisasm.mips.sections.SectionBase, sym: "Symbol" +) -> spimdisasm.common.ContextSymbol: + if sym.type == "func": + context_sym = section.addFunction( + sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom + ) + elif sym.type == "jtbl": + context_sym = section.addJumpTable( + sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom + ) + elif sym.type == "jtbl_label": + context_sym = section.addJumpTableLabel( + sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom + ) + elif sym.type == "label": + context_sym = section.addBranchLabel( + sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom + ) + else: + context_sym = section.addSymbol( + sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom + ) + if sym.type is not None: + context_sym.type = sym.type + + if sym.user_declared: + context_sym.isUserDeclared = True + if sym.defined: + context_sym.isDefined = True + if sym.rom is not None: + context_sym.vromAddress = sym.rom + if sym.given_size is not None: + context_sym.size = sym.size + if sym.force_migration: + context_sym.forceMigration = True + if sym.force_not_migration: + context_sym.forceNotMigration = True + context_sym.setNameGetCallbackIfUnset(lambda _: sym.name) + if sym.given_name_end: + context_sym.nameEnd = sym.given_name_end + + return context_sym + + +def create_symbol_from_spim_symbol( + segment: "Segment", context_sym: spimdisasm.common.ContextSymbol +) -> "Symbol": + in_segment = False + + sym_type = None + if context_sym.type == spimdisasm.common.SymbolSpecialType.jumptable: + in_segment = True + sym_type = "jtbl" + elif context_sym.type == spimdisasm.common.SymbolSpecialType.function: + sym_type = "func" + elif context_sym.type == spimdisasm.common.SymbolSpecialType.branchlabel: + in_segment = True + sym_type = "label" + elif context_sym.type == spimdisasm.common.SymbolSpecialType.jumptablelabel: + in_segment = True + sym_type = "jtbl_label" + + if not in_segment: + if ( + context_sym.overlayCategory is None + and segment.get_exclusive_ram_id() is None + ): + in_segment = segment.contains_vram(context_sym.vram) + elif context_sym.overlayCategory == segment.get_exclusive_ram_id(): + if context_sym.vromAddress is not None: + in_segment = segment.contains_rom(context_sym.vromAddress) + else: + in_segment = segment.contains_vram(context_sym.vram) + + sym = segment.create_symbol( + context_sym.vram, in_segment, type=sym_type, reference=True + ) + + if sym.given_name is None and context_sym.name is not None: + sym.given_name = context_sym.name + + # To keep the symbol name in sync between splat and spimdisasm + context_sym.setNameGetCallback(lambda _: sym.name) + + if context_sym.size is not None: + sym.given_size = context_sym.getSize() + if context_sym.vromAddress is not None: + sym.rom = context_sym.getVrom() + if context_sym.isDefined: + sym.defined = True + if context_sym.referenceCounter > 0: + sym.referenced = True + + return sym + + +def mark_c_funcs_as_defined(): + for symbol in all_symbols: + if len(to_mark_as_defined) == 0: + return + sym_name = symbol.name + if sym_name in to_mark_as_defined: + symbol.defined = True + to_mark_as_defined.remove(sym_name) + + +@dataclass +class Symbol: + vram_start: int + + given_name: Optional[str] = None + given_name_end: Optional[str] = None + rom: Optional[int] = None + type: Optional[str] = None + given_size: Optional[int] = None + segment: Optional["Segment"] = None + + defined: bool = False + referenced: bool = False + dead: bool = False + extract: bool = True + user_declared: bool = False + + force_migration: bool = False + force_not_migration: bool = False + + allow_addend: bool = False + dont_allow_addend: bool = False + + linker_section: Optional[str] = None + + _generated_default_name: Optional[str] = None + _last_type: Optional[str] = None + + appears_after_overlays_addr: Optional[int] = None + + def __str__(self): + return self.name + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Symbol): + return False + return self.vram_start == other.vram_start and self.segment == other.segment + + # https://stackoverflow.com/a/56915493/6292472 + def __hash__(self): + return hash((self.vram_start, self.segment)) + + def format_name(self, format: str) -> str: + ret = format + + ret = ret.replace("$VRAM", f"{self.vram_start:08X}") + + if "$ROM" in ret: + if not isinstance(self.rom, int): + log.error( + f"Attempting to rom-name a symbol with no ROM address: {self.vram_start:X} typed {self.type}" + ) + ret = ret.replace("$ROM", f"{self.rom:X}") + + if "$SEG" in ret: + if self.segment is None: + # This probably is fine - we can't expect every symbol to have a segment. Fall back to just the ram address + return f"{self.vram_start:X}" + assert self.segment is not None + ret = ret.replace("$SEG", self.segment.name) + + return ret + + @property + def default_name(self) -> str: + if self._generated_default_name is not None: + if self.type == self._last_type: + return self._generated_default_name + + if self.segment: + if isinstance(self.rom, int): + suffix = self.format_name(self.segment.symbol_name_format) + else: + suffix = self.format_name(self.segment.symbol_name_format_no_rom) + else: + if isinstance(self.rom, int): + suffix = self.format_name(options.opts.symbol_name_format) + else: + suffix = self.format_name(options.opts.symbol_name_format_no_rom) + + if self.type == "func": + prefix = "func" + elif self.type == "jtbl": + prefix = "jtbl" + elif self.type in {"jtbl_label", "label"}: + return f".L{suffix}" + else: + prefix = "D" + + self._last_type = self.type + self._generated_default_name = f"{prefix}_{suffix}" + return self._generated_default_name + + @property + def rom_end(self): + return None if not self.rom else self.rom + self.size + + @property + def vram_end(self): + return self.vram_start + self.size + + @property + def name(self) -> str: + return self.given_name if self.given_name else self.default_name + + @property + def size(self) -> int: + if self.given_size is not None: + return self.given_size + return 4 + + def contains_vram(self, offset): + return offset >= self.vram_start and offset < self.vram_end + + def contains_rom(self, offset): + return offset >= self.rom and offset < self.rom_end diff --git a/tools/sym_info.py b/tools/sym_info.py new file mode 100755 index 0000000..595faf2 --- /dev/null +++ b/tools/sym_info.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import mapfile_parser +from pathlib import Path + + +def symInfoMain(): + parser = argparse.ArgumentParser(description="Display various information about a symbol or address.") + parser.add_argument("symname", help="symbol name or VROM/VRAM address to lookup") + parser.add_argument("-e", "--expected", dest="use_expected", action="store_true", help="use the map file in expected/build/ instead of build/") + parser.add_argument("-v", "--version", help="Which version should be processed", default="jp") + + args = parser.parse_args() + + BUILTMAP = Path(f"build") / f"animalforest.{args.version}.map" + + mapPath = BUILTMAP + if args.use_expected: + mapPath = "expected" / BUILTMAP + + mapfile_parser.frontends.sym_info.doSymInfo(mapPath, args.symname) + +if __name__ == "__main__": + symInfoMain() |
