diff options
| author | emilybrooks <emilybrooksemilybrooks@gmail.com> | 2024-01-06 15:51:16 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-01-06 15:51:16 -0700 |
| commit | 093b8b7b9b20150a2eb6d2a8ab887f9387298fee (patch) | |
| tree | 10573a1e986d07db373e9f92f770bccbb746f559 /tools | |
| parent | ebd97e9fd1cd8bb02a397ca8b0982bc0e2b076ae (diff) | |
Update splat to library version 0.21.0 (#130)
* build not matching for some reason
* partial fix
* builds properly now
* remove pigment64 from readme
Diffstat (limited to 'tools')
128 files changed, 0 insertions, 9834 deletions
diff --git a/tools/splat/.github/workflows/black.yml b/tools/splat/.github/workflows/black.yml deleted file mode 100644 index 138bd3a..0000000 --- a/tools/splat/.github/workflows/black.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: black - -on: - push: - pull_request: - -jobs: - black_checks: - runs-on: ubuntu-latest - name: black - steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Dependencies - run: | - pip install black - pip install -r requirements.txt - pip install types-PyYAML - - name: black - run: black --check . diff --git a/tools/splat/.github/workflows/mypy.yml b/tools/splat/.github/workflows/mypy.yml deleted file mode 100644 index ba18d96..0000000 --- a/tools/splat/.github/workflows/mypy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: mypy - -on: - push: - pull_request: - -jobs: - mypy_checks: - runs-on: ubuntu-latest - name: mypy - steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Dependencies - run: | - pip install mypy - pip install -r requirements.txt - pip install types-PyYAML - - name: mypy - run: mypy --show-column-numbers --hide-error-context . diff --git a/tools/splat/.github/workflows/unit_tests.yml b/tools/splat/.github/workflows/unit_tests.yml deleted file mode 100644 index 9cdb1f5..0000000 --- a/tools/splat/.github/workflows/unit_tests.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: unit_tests - -on: - push: - pull_request: - -jobs: - unit_tests: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Dependencies - run: pip install -r requirements.txt - - name: Run unit tests - run: sh run_tests.sh diff --git a/tools/splat/.gitignore b/tools/splat/.gitignore deleted file mode 100644 index 38da5f7..0000000 --- a/tools/splat/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -.idea/ -venv/ -.vscode/ -__pycache__/ -.mypy_cache/ -util/n64/Yay0decompress -*.ld -*.n64 -*.yaml -*.z64 diff --git a/tools/splat/.gitrepo b/tools/splat/.gitrepo deleted file mode 100644 index 2cad301..0000000 --- a/tools/splat/.gitrepo +++ /dev/null @@ -1,12 +0,0 @@ -; DO NOT EDIT (unless you know what you are doing) -; -; This subdirectory is a git "subrepo", and this file is maintained by the -; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme -; -[subrepo] - remote = git@github.com:ethteck/splat.git - branch = master - commit = 6ec2b39108ac3891824896dc33a4de56765f962d - parent = f19df74c6ee597f01c3d7def1fdef105d9bdef92 - method = merge - cmdver = 0.4.6 diff --git a/tools/splat/CHANGELOG.md b/tools/splat/CHANGELOG.md deleted file mode 100644 index a72a12e..0000000 --- a/tools/splat/CHANGELOG.md +++ /dev/null @@ -1,486 +0,0 @@ -# splat Release Notes - -### 0.16.9 - -* Add command line argument `--disassemble-all`, which has the same effect as the `disassemble_all` yaml option so will disamble already matched functions as well as migrated data. - * Note: the command line argument takes precedence over the yaml, so will take effect even if the yaml option is set to false. - -### 0.16.8 - -* Avoid ignoring the `align` defined in a segment for `code` segments - -### 0.16.7 - -* Use `pylibyaml` to speed-up yaml parsing - -### 0.16.6 - -* Add option `ld_rom_start`. - * Allows offsetting rom address linker symbols by some arbitrary value. - * Useful for SN64 games which often have rom addresses offset by 0xB0000000. - * Defaults to 0. - -### 0.16.5 - -* Add option `segment_symbols_style`. - * Allows changing the style of the generated segment symbols in the linker script. - * Possible values: - * `splat`: The current style for segment symbols. - * `makerom`: Style that aims to be compatible with makerom generated symbols. - * Defaults to `splat`. - -### 0.16.4 - -* Add `get_section_flags` method to the `Segment` class. - * Useful for providing linker section flags when creating a custom section when making splat extensions. - * This may be necessary for some custom section types, because sections unrecognized by the linker will not link its data properly. - * More info about section flags: <https://sourceware.org/binutils/docs/as/Section.html#ELF-Version> - -### 0.16.3 - -* Add `--stdout-only` flag. Redirects the progress bar output to `stdout` instead of `stderr`. -* Add a check to prevent relocs with duplicated rom addresses. -* Check empty functions only have 2 instructions before autodecompiling them. - -### 0.16.2 - -* Add option `disassemble_all`. If enabled then already matched functions and migrated data will be disassembled to files anyways. - -### 0.16.1 - -* Various changes so that series of image and palette subsegments can have `auto` rom addresses (as long as the first can find its rom address from the parent segment or its own definition) - -### 0.16.0 - -* Add option `detect_redundant_function_end`. It tries to detect redundant and unreferenced functions ends and merge them together. - * This option is ignored if the compiler is not set to IDO. - * This type of codegen is only affected by flags `-g`, `-g1` and `-g2`. - * This option can also be overriden per file. -* Disable `include_macro_inc` by default for IDO projects. -* Disable `asm_emit_size_directive` by default for SN64 projects. -* `spimdisasm` 1.16.0 or above is now required. - -### 0.15.4 - -* Try to assign a segment to an user-declared symbol if the user declared the rom address. - * Helps to disambiguate symbols for same-address overlays. - -### 0.15.3 - -* Disabled `asm_emit_size_directive` by default for IDO projects. - -### 0.15.2 - -* Various cleanup and fixes to support more liberal use of `auto` for rom addresses - -### 0.15.1 - -* Made some modifications such that linker object paths should be simpler in some circumstances - -### 0.15.0 - -* New options: - * `data_string_encoding` can be set at the global level (or `str_encoding` at the segment level) to specify the encoding using when guessing and disassembling strings the the data section. In spimdisasm this value defaults to ASCII. - * `rodata_string_guesser_level` changes the behaviour of the rodata string guesser. A higher value means more agressive guessing, while 0 and negative means no guessing at all. Even if the guesser feature is disabled, symbols manually marked as strings in the symbol_addrs.txt file will still be disassembled as strings. In spimdisasm this value defaults to 1. - * level 0: Completely disable the guessing feature. - * level 1: The most conservative guessing level. Imposes the following restrictions: - * Do not try to guess if the user provided a type for the symbol. - * Do no try to guess if type information for the symbol can be inferred by other means. - * A string symbol must be referenced only once. - * Strings must not be empty. - * level 2: A string no longer needs to be referenced only once to be considered a possible string. This can happen because of a deduplication optimization. - * level 3: Empty strings are allowed. - * level 4: Symbols with autodetected type information but no user type information can still be guessed as strings. - * `data_string_guesser_level` is similar to `rodata_string_guesser_level`, but for the data section instead. In spimdisasm this value defaults to 2. - * `asm_emit_size_directive` toggles the size directived emitted by the disassembler. In spimdisasm this defaults to True. - -### 0.14.1 - -* Fix bug, cod cleanup - -### 0.14.0 - -* Add support for PSX's GTE instruction set - -### 0.13.10 - -* New option `disasm_unknown` (False by default) - * If enabled it tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data - -### 0.13.9 - -* New segment option `linker_entry` (true by default). - * If disabled, this segment will not produce entries in the linker script. - -### 0.13.8 - -* New option `segment_end_before_align`. - * If enabled, the end symbol for each segment will be placed before the alignment directive for the segment - -### 0.13.7 - -* Severely sped-up linker entry writing by using a dict instead of a list. Symbol headers will no longer be in any specific order (which shouldn't matter, because they're headers). - -### 0.13.6 - -* Changed CI image processing so that their data is fetched during the scan phase, supporting palettes that come before CI images. - -### 0.13.5 - -* An error will be produced if a symbol is declared with an unknown type in the symbol_addrs file. - * The current list of known symbols is `'func', 'label', 'jtbl', 'jtbl_label', 's8', 'u8', 's16', 'u16', 's32', 'u32', 's64', 'u64', 'f32', 'f64', 'Vec3f', 'asciz', 'char*', 'char'`. - * Custom types are allowed if they start with a capital letter. - -### 0.13.4 - -* Renamed `follows_vram_symbol` segment option to `vram_of_symbol` to more accurately reflect what it's used for - to set the segment's vram based on a symbol. -* Refactored the `appears_after_overlays_addr` feature so that expressions are written at the latest possible moment in the linker script. This fixes errors and warnings regarding forward references to later symbols. - -### 0.13.3 - -* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieves this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays. - -### 0.13.2 - -* Actually implemented `ld_use_follows`. Oopz - -### 0.13.1 - -* Added `ld_wildcard_sections` option (disabled by default), which adds a wildcard to the linker script for section linking. This can be helpful for modern GCC, which creates additional rodata sections such as ".rodata.xyz". -* Added `ld_use_follows` option (enabled by default), which, if disabled, makes splat ignore follows_vram and follows_symbols. This helps for fixing matching builds while being able to add infrastructure to the yaml for non-matching builds by just re-enabling the option. - -### 0.13.0 - -* Automatically generate `INCLUDE_RODATA`/`#pragma GLOBAL_ASM` directives for non migrated rodata symbols when creating new C files. -* Non migrated rodata symbols will now only be produced if the C file has a corresponding rodata file with the same name and the C file has a `INCLUDE_RODATA`/`#pragma GLOBAL_ASM` directive referencing the symbol, similar to how functions are disassembled. - * Because of this, the `partial_migration` attribute has lost its purpose and has been removed. -* Rodata symbol files are now included in the autogenerated dependency files too. - -### 0.12.14 - -* New option: `pair_rodata_to_text`. - * If enabled, splat will try to find to which text segment an unpaired rodata segment belongs, and it will hint it to the user. - -### 0.12.13 - -* bss segments can now omit the rom offset. - -### 0.12.12 - -* Try to detect and warn to the user if a gap between two migrated rodata symbols is detected and suggest possible solutions to the user. - -### 0.12.11 - -* New disassembly option in the yaml: `allow_data_addends`. - * Allows enabling/disabling using addends on all `.data` symbols. -* Three new options for symbols: `name_end`, `allow_addend` and `dont_allow_addend`. - * `name_end`: allows to provide a closing name for any symbol. Useful for handwritten asm which usually have an "end" name. - * `allow_addend` and `dont_allow_addend`: Allow overriding the global `allow_data_addends` option for allowing addends on data symbols. - -### 0.12.10 - -* Allows passing user-created relocs to the disassembler via the `reloc_addrs.txt` file, allowing to improve the automatic disassembly. -* Multiple reloc_addrs files can be specified in the yaml with the `reloc_addrs_path` option. - -### 0.12.9 - -* Added `format_sym_name()` to the vtx segment so it, too, can be extended - -### 0.12.8 - -* The gfx and vtx segments now have a `data_only` option, which, if enabled, will emit only the plain data for the type and omit the enclosing symbol definition. This mode is useful when you want to manually declare the symbol and then #include the extracted data within the declaration. -* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this, so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component. - -### 0.12.7 - -* Allow setting a different macro for jumptable labels with `asm_jtbl_label_macro` - * The currently recommended one is `jlabel` instead of `glabel` -* Two new options for symbols: `force_migration` and `force_not_migration` - * Useful for weird cases where the disassembler decided a rodata symbol must (or must not) be migrated when it really shouldn't (or should) -* Fix `str_encoding` defaulting to `False` instead of `None` -* Output empty rules in generated dependency files to avoid issues when the function file does not exist anymore (i.e. when it gets matched) -* Allow changing the `include_macro_inc` option in the yaml - -### 0.12.6 - -* Adds two new N64-specific segments: - * IPL3: Allows setting its correct VRAM address without messing the global segment detection - * RSP: Allows disassembling using the RSP instruction set instead of the default one -* PS2 was added as a new platform option. - * When this is selected the R5900 instruction set will be used when disassembling instead of the default one. - -### 0.12.5 - -* Update minimal spimdisasm version to 1.7.1. -* Fix spimdisasm>=1.7.0 non being able to see symbols which only are referenced by other data symbols. -* A check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user. - -### 0.12.4 - -* Fixed a bug involving the order of attributes in symbol_addrs preventing proper range searching during calls to `get_symbol` - -### 0.12.3: Initial Gamecube Support -Initial support for Gamecube disk images has been set up! Disassembly is not currently supported, and a more comprehensive explanation of Gamecube support will come once that is finished. - -* The Symbol class is now hashable -* Added the ability for segments to specify a file path (`path`) to receive that file's contents as their split input -* The `generated_s_preamble` option now will be applied to data files created by spimdisasm -* Rewrote symbol range check code to be more efficient -* Fixed bug that allowed empty top-level segments of type `code`. -* Fixed progress bars to properly update their descriptions -* Fixed bug pertaining to symbols getting assigned to segments they shouldn't if their segment is given in symbol_addrs (`segment:`) - -### 0.12.2 -* Fixed bug where `given_dir` was possibly not a `Path` - -### 0.12.1 -* The constructor for `Segment` takes far fewer arguments now, which will affect (and hopefully simplify) any custom segments that are implemented. - -* The new option `string_encoding` can be set at the global or segment level and will influence the encoding for strings in rodata during disassembly. The default encoding used is EUC-JP, as it was previously. - -## 0.12.0: Performance Boost - -In this release, we bring many performance improvements, making splat dramatically faster. We have observed speedups of 10-20x, though your results may vary. - -* Linker script `_romPos` alignment statements now take a form that is friendlier to different assemblers. - -* Fixed the default value of `use_legacy_include_asm` to be what it was before 0.11.2 - -### 0.11.2 -* The way options are parsed and accessed has been completely refactored. The following option names have changed: - -`linker_symbol_header_path` -> `ld_symbol_header_path` - -`asm_endlabels` -> `asm_end_label` - -Additionally, any custom segments or code that needs to read options will have to accommodate the new API for doing so. Options are now fields of an object named `opts` within the existing `options` namespace. Because the options are fields, `get_` is no longer necessary. To give an example: - -Before: `options.get_asm_path()` - -After: `options.opts.asm_path` - -The clean_up_path function in linker_entry.py now uses a cache, offering a small performance improvement during the linker script writing phase. - -### 0.11.1 -* The linker script now includes a `_SIZE` symbol for each segment. -* The new `create_asm_dependencies`, if enabled, will cause splat to create `.asmproc.d` files that can inform a build system which asm files a c file depends upon. If your build system is configured correctly, this can allow triggering a rebuild of a C file when its included asm files are modified. -* Splat no longer depends directly on pypng and now instead uses [n64img](https://github.com/decompals/n64img). Currently, all image behavior uses the exact same code. Eventually, n64img will be implemented in C and support rebuilding images as well. - -## 0.11.0: Spimdisasm Returns - -Spimdisasm now handles data (data, rodata, bss) disassembly in splat! This includes a few changes in behavior: - -* Rodata will be migrated to c files' asm function files when a .rodata subsegment is used that corresponds with an identically-named c file. Some symbols may not be automatically migrated to functions when it is not clear if they belong to the function itself (an example of which being const arrays). In this case, the `partial_migration` option can be enabled for the given .rodata subsegment and splat will create .s files for these unmigrated rodata symbols. These files can then be included in your c files, or you can go ahead and migrate these symbols to c and disable the `partial_migration` feature. - -* BSS can now be disassembled as well, and the size of a code segment's bss section can be specified with the `bss_size` option. This option will tell splat how large the bss section is in bytes so BSS can properly be handled during disassembly. For bss subsegments, the rom address will of course not change, but the vram address should still be specified. This currently can only be done in the dict form of segment representation, rather than the list form. - -Thanks again to [AngheloAlf](https://github.com/AngheloAlf) for adding this functionality and continuing to improve splat's disassembler. - -## 0.10.0: The Linker Script Update - -Linker scripts splat produces are now capable of being shift-friendly. Rom addresses will automatically shift, and ram addresses will still be hard-coded unless the new segment option `follows_vram` is specified. The value of this option should be the name of a segment (a) that this segment (b) should follow in memory. If a grows or shrinks, b's start address will also do so to accommodate it. - -The `enable_ld_alignment_hack` option and corresponding behavior has been removed. This proved to add too much complexity to the linker script generation code and was becoming quite a burden to keep dealing with. Apologies for any inconvenience this may cause. But trust me: in the long run, it's good you won't be depending on that madness. - -### 0.9.5 -* Changes have been made to the linker script such that it is more shiftable. Rather than setting the rom position to hard-coded addresses, it increments the position by the size of the previous segment. Some projects may experience some alignment-related issues after this change. If specified, the new segment option `align: n` will add an `ALIGN(n)` directive for that section's linker segment. - -### 0.9.4 -* A new linker script section is now automatically created when the .bss section begins, using NOLOAD as opposed to the previous hacky rom rewinding we were previously doing. Additionally, `ld_section_labels` now includes `.rodata` by default. - -### 0.9.3 -* Added `add_set_gp_64` option (true by default), which allows controlling whether to add ".set gp=64" to asm/hasm files - -### 0.9.2 -* Added "palette" argument to ci4/ci8 segments so that segments' palettes can be manually specified - -### 0.9.1 -* Fixed a bug in which local labels and jump table labels could replace raw words in data blobs during data disassembly - -## 0.9.0: The Big Update -### Introducing [spimdisasm](https://github.com/Decompollaborate/spimdisasm)! -* Thanks to [AngheloAlf](https://github.com/AngheloAlf), we now have a much better MIPS disassembler in splat! spimdisasm has much better hi/lo matching, much lower ram usage, and plenty of other goodies. - -We plan to roll this out in phases. Currently, it only handles actual code disassembly. Later on, we will probably migrate our current data assembly code to use spimdisasm as well. - -**NOTICE**: This integration has been tested on a variety of games and configurations. However, with any giant change to the platform like this, there are bound to be things we didn't catch. Please be patient with us as we handle these remaining issues. Though from what we've seen already, the slight bugs one may come across are totally worth the much improved disassembly. - -### gfx segment type -* A new `gfx` segment type is available, which creates a c file containing a disassembled display list according to the segment's start and end offsets. Thanks to [Glank](https://github.com/glankk) and [Tharo](https://github.com/thar0/) for their work on [libgfxd](https://github.com/glankk/libgfxd) and [pygfxd](https://github.com/thar0/pygfxd/), respectively, for helping make this a possibility in splat. - -### API breaking changes -* Some `Segment()` arguments have changed, which may cause extensions to break. Please see the `__init__` function for `Segment` for more details. - -### symbol_addrs.txt changes -* symbol_addrs now supports the `segment:` attribute, which allows specifying the symbol's top-level segment. This can be helpful for symbol resolution when overlays use overlapping vram ranges. See `exclusive_ram_id` below for more information. - -### Global options changes - -The new `symbol_name_format` option allows specification of how symbols will be named. This can be set as a global option and also changed per-segment. `symbol_name_format_no_rom` is used when the symbol does not have a rom address (BSS). - - The following substitutions are allowed: - -`$ROM` - the rom address of the symbol, hex-formatted and padded to 6 characters (ABCF10, 000030, 123456) (note: only for `symbol_name_format`, usage in `symbol_name_format_no_rom` will cause an error) - -`$VRAM` - the vram address of the symbol, hex-formatted and padded to 8 characters (00030010, 00020015, ABCDEF10) - -`$SEG` - the name of the top-level segment in which the symbol resides - -The default values for these options are as follows - -`symbol_name_format` : `$VRAM` - -`symbol_name_format_no_rom` : `$VRAM_$SEG` - -The appropriate prefix string will still automatically be applied depending on the type of the symbol: `D_` for data, `jtbl_` for jump tables, and `func_` for functions. This functionality may be customizable in the future. - ----- -The `auto_all_sections` option now should be a list of section names (`[".data", ".rodata", ".bss"]` by default) indicating the sections that should be linked from .o files built from source files (.c or asm/hasm .s files), when no subsegment explicitly indicates linking this type of section. - -For example, if any subsegment of a code segment is of segment type `data` or `.data`, the `.data` section from all `c`/`asm`/`hasm` subsegments will not be linked unless explicitly indicated with a relevant `.data` subsegment. - -Previously, this option was a bool, and it enabled this feature for all sections specified in `section_order`. Now, the desired sections must be specified manually. The default value for this option retains previous behavior. - ----- -The new `mips_abi_float_regs` option allows for changing the format of float registers for MIPS disassembly. The default value does not change any prior behavior, but `o32` is heavily encouraged and may become the default option in the future. For more information, see this [great writeup](https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d). - ----- -The new `gfx_ucode` option allows for specifying the target for the graphics macro format, which is used in the gfx segment type. The default is `f3dex2`. - - -### Segment options changes - -The new `exclusive_ram_id` segment option allows specifying an identifer that will prevent the segment from seeing any symbols from other segments with the same identifer. This is useful when multiple segments are mapped to the same vram address at runtime and should never be able to refer to each other's symbols. Setting all of these segments to have the same value for this option will prevent their symbols from clashing / meshing unexpectedly. - ----- - -The `overlay` setting on segments has been removed. Please see `symbol_name_format` above for info on how to influence the names of symbols, which can be applied at the segment level as well as the global level. - ----- -## 0.8.0: Arbitrary Section Order -* You can now use the option `section_order` to define the binary section order for your target binary. By default, this is `[".text", ".data", ".rodata", ".bss"]`. See options.py for more details -* Documented all options in options.py -* Support for SN64 games (thanks Wiseguy!) -* More consistent handling of paths (thanks Mkst!) -* Various other cleanup and fixes across the board - -### 0.7.10: WIP PSX support -* WIP PSX support has been added, thanks to @mkst! (https://github.com/ethteck/splat/pull/99) - * Many segments have moved to a "common" package - * Endianness of the input binary is now a configurable option -* Linker hack restored but is now optional and off by default - -### 0.7.9 -* Finally removed the dumb linker section alignment hack -* Added version number to output on execution - -### 0.7.8 - -* Fixed a bug relating to a linker section alignment hack (thanks Wiseguy!) -* Fixed a bug in linker_entry.py's clean_up_path that should make this function more versatile (thanks Wiseguy!) - -### 0.7.7 - -* Disassembly now reads the `size` property of a function in symbol_addrs.txt to disassemble `size / 4` number of instructions. Feel free to specify the size of your functions in your symbol_addrs file if splat's disassembly is chopping a function too short or making a function too long. - -### 0.7.6 - -* Fixed a bug involving detection of defined functions in c files for GLOBAL_ASM-using projects -* Added options to disable the creation of undefined_funcs/syms_auto.txt files -* Added a Vtx segment type for creating c files containg model vertex data in the n64 libultra Vtx format -* Added a `cpp` segment type which is identical to `c` but looks for a file with the extension ".cpp" instead of ".c". - -### 0.7.5: all_ types and auto_all_sections - -If you have a group segment with multiple c files and want splat to automatically create linker entries at a given position for each code object (c, asm, hasm) in the segment, you can use an `all_` type for that section. For example, you can add `[auto, all_bss]` as the last subsegment in a segment. This will direct splat to create a linker entry for each code object in the segment. This saves a lot of time when it comes to manually adding .bss subsegments for bss support, for example. The same thing can be done for data and rodata sections, but note this should probably be done later into a project when all data / rodata is migrated to c files, as the `all_` types lose the rom positioning information that's necessary for splat to do proper disassembly. - -The `auto_all_sections` option, when set to true, will automatically add `all_` types into every group. This is only done for a section in a group if no other manual declarations for that section exist. For example, if you have 30 c files in a group and a .data later on for one of them, `auto_all_sections` will not interfere with your `.data` subsegment. If you remove this, however, splat will use `auto_all_sections` to implicitly `.data` subsegments for all of your code objects behind the scenes. This feature is again particualrly helpful for bss support, as it will create bss linker entries for every file in your project (assuming you don't have any manual .bss subsegments), which eliminates the need to create dummy .bss subsegments just for the sake of configuring the linker script. - -### 0.7.2 - -* Data disassembly changes: - * String detection has been improved. Please send me false positives / negatives as you see them and I can try to improve it further! - * Symbols in a data segment pointed to by other symbols will now properly be split out as their own symbols - -### 0.7.1 - -* Image segment changes: - * Added `flip_x` and `flip_y` boolean parameters to replace `flip`. - * `flip` is deprecated and will produce a warning when used. - * Fixed flipping of `ci4` and `ci8` images. - * Fixed `extract: false` (and `start: auto`) behaviour. - -## 0.7.0: The Path Update - -* Significantly better performance, especially when using the cache feature (`--use-cache` CLI arg). -* BREAKING: Some cli args for splat have been renamed. Please consult the usage output (-h or no args) for more information. - * `--new` has been renamed to `--use-cache` - * `--modes` arg changes: - * Image modes have been combined into the `img` mode - * Code and ASM modes have been combined into the `code` mode -* BREAKING: The `name` attribute of a segment now should no longer be a subdirectory but rather a meaningful name for the segment which will be used as the name of the linker section. If your `name` was previously a directory, please change it into a `dir`. -* BREAKING: `subsections` has been renamed to `subsegments` -* New `dir` segment attribute specifies a subdirectory into which files will be saved. You can combine `dir` ("foo") with a subsegment name containing a subdirectory ("bar/out"), and the paths will be joined (foo/bar/out.c) - * If the `dir` attribute is specified but the `name` isn't, the `name` becomes `dir` with directory separation slashes replaced with underscores (foo/bar/baz -> foo_bar_baz) -* BREAKING: Many configuration options have been renamed. `_dir` options have been changed to the suffix `_path`. -* BREAKING: Assets (non-code, like `bin` and images) are now placed in the directory `asset_path` (defaults to `assets`). -* Linker symbol header generation. Set the `linker_symbol_header_path` option to use. - * `typedef u8[] Addr;` is recommended in your `common.h` header. -* You can now provide `auto` as the `start` attribute for a segment, e.g. `[auto, c, my_file]`. This causes the segment to not be extracted, but linked. This feature is intended for modding. -* Providing just a ROM address but no type or name for a segment is now valid anywhere in `segments` or `subsegments` rather than just at the end of the ROM. It specifies the end of the previous segment for types that need it (`palette`, `bin`, `Yay0`) and causes the linker to simply write padding until that address. -* The linker script file is left untouched if the contents have not changed since the previous split. -* You can now group together segments with `type: group` (similar to `code`). Note that any ASM or C segments must live under a `type: code` segment, not a basic `group`. - -### 0.6.5: Bugfixes, rodata migration, and made options static - -If you wrote a custom extension, options should be imported and statically referenced -`from util import options` - -see options.py for more info on how to now get and set options - -BREAKING: vram can only be specified on a segment if the segment is defined as a dict in the config - -### 0.6.3: More refactoring -**Breaking Change**: The command line args to split.py have changed. Currently, only the config path is now a required argument to splat. The old `rom` and `outdir` parameters are now optional (`--rom`, `--outdir`). Now, you can add rom and out directory paths in the yaml. - -The `out_dir` option specifies a directory relative to the config file. If your config file is in a subdirectory of the main repo, you can set `out_dir: ../`, for example. - -The `target_path` option spcifies a path to the binary file to split, relative to the `out_dir`. If your `baserom.z64` is in the top-level of the repo, you can set `target_path: baserom.z64`, for example. - -### 0.6.2: subsegments -I've begun a refactor of the code "files" code, which makes everything cleaner and easier to extend. -There's also a new option, `create_new_c_files`, which disables the creation of nonexistent c files. This behavior is on by default, but if you want to disable it for any reason, you now have the option to do so. - -I am also working on adding bss support as well. It should almost be all set, aside from the changes needed in the linker script. - -**Breaking change**: The `files` field in `code` segments should now be renamed to `subsegments`. - -### 0.6.1: `assets_dir` option - -This release adds a new `assets_dir` option in `splat.yaml`s that allows you to override the default `img`, `bin`, and other directories that segments output to. - -Want to interdisperse split assets with your sourcecode? `assets_dir: src`! -Want to have all assets live in a single directory? `assets_dir: assets`! - -## 0.6: The Symbol Update -Internally, there's a new Symbol class which stores information about a symbol and is stored in a couple places during disassembly. Many things should be improved, such as reconciling symbols within overlays, things being named functions vs data symbols, and more. - -**Breaking change**: The format to symbol_addrs.txt has been updated. After specifying the name and address of a symbol (`symbol = addr;`), optional properties of symbols can be set via inline comment, space delimited, in any order. The properties are of the format `name:value` - * `type:` supports `func` mostly right now but will support `label` and `data` later on. Internally, `jtbl` is used as well, for jump tables. Splat uses type information during disassembly to disambiguate symbols with the same addresses. - * `rom:` is for the hex rom address of the symbol, beginning with `0x`. If available, this information is extremely valuable for use in disambiguating symbols. - * `size:` specifies the size of the symbol, which splat will use to generate offsets during disassembly. Uses the same format as `rom:` - -**function example**: `FuncNameHere = 0x80023423; // type:func rom:0x10023` - -**data example**: `gSomeDataVar = 0x80024233; // type:data size:0x100` - -## 0.5 The Rename Update -* n64splat name changed to splat - * Some refactoring was done to support other platforms besides n64 in the future - * New `platform` option, which defaults to `n64` - * This will cause breaking changes in custom segments, so please refer to one of the changes in one of the n64 base segments for details -* Support for custom artifact paths - * New `undefined_syms_auto_path` option - * New `undefined_funcs_auto_path` option - * New `cache_path` option - * (All path-like options' names now end with `_path`) diff --git a/tools/splat/Dockerfile b/tools/splat/Dockerfile deleted file mode 100644 index 0c84c24..0000000 --- a/tools/splat/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:22.04 -RUN apt-get update -RUN apt install -y binutils-mips-linux-gnu -ADD https://github.com/decompals/mips-binutils-2.6/releases/download/main/binutils-2.6-linux.tar.gz . -ADD https://github.com/decompals/mips-gcc-2.7.2/releases/download/main/gcc-2.7.2-linux.tar.gz . -RUN mkdir -p ./gcc-2.7.2 && \ - tar -xvf gcc-2.7.2-linux.tar.gz -C ./gcc-2.7.2 && \ - tar -xvf binutils-2.6-linux.tar.gz -C ./gcc-2.7.2 diff --git a/tools/splat/LICENSE b/tools/splat/LICENSE deleted file mode 100644 index 42621fb..0000000 --- a/tools/splat/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Ethan Roseman - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/tools/splat/Makefile b/tools/splat/Makefile deleted file mode 100644 index b72aafe..0000000 --- a/tools/splat/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -UTIL_DIR := util - -default: all - -all: Yay0decompress - -Yay0decompress: - gcc $(UTIL_DIR)/n64/Yay0decompress.c -fPIC -shared -O3 -Wall -Wextra -o $(UTIL_DIR)/n64/Yay0decompress - -clean: - rm -f $(UTIL_DIR)/n64/Yay0decompress diff --git a/tools/splat/README.md b/tools/splat/README.md deleted file mode 100644 index de7abc7..0000000 --- a/tools/splat/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# splat -A binary splitting tool to assist with decompilation and modding projects - -Currently, only N64 and PSX binaries are supported. - -Please check out the [wiki](https://github.com/ethteck/splat/wiki) for more information including [examples](https://github.com/ethteck/splat/wiki/Examples) of projects that use splat. - -### Requirements -splat requires Python 3.8+. Package requirements can be installed via `pip3 install -U -r requirements.txt` diff --git a/tools/splat/create_config.py b/tools/splat/create_config.py deleted file mode 100755 index cf34481..0000000 --- a/tools/splat/create_config.py +++ /dev/null @@ -1,192 +0,0 @@ -#! /usr/bin/env python3 - -import argparse -import sys -from pathlib import Path - -from segtypes.gc.rarc import GcSegRarc -from util.gc import gcinfo - -from util.n64 import find_code_length, rominfo - -parser = argparse.ArgumentParser( - description="Create a splat config from an N64 ROM or a GameCube disc image." -) -parser.add_argument("file", help="Path to a .z64/.n64 ROM or .iso/.gcm GameCube image") - - -def main(file_path: Path): - if not file_path.exists(): - sys.exit(f"File {file_path} does not exist ({file_path.absolute()})") - if file_path.is_dir(): - sys.exit(f"Path {file_path} is a directory ({file_path.absolute()})") - - # Check for N64 ROM - if file_path.suffix.lower() == ".n64" or file_path.suffix.lower() == ".z64": - create_n64_config(file_path) - return - - file_bytes = file_path.read_bytes() - - # Check for GC disc image - if int.from_bytes(file_bytes[0x1C:0x20], byteorder="big") == 0xC2339F3D: - create_gc_config(file_path, file_bytes) - - -def create_n64_config(rom_path: Path): - rom_bytes = rominfo.read_rom(rom_path) - - rom = rominfo.get_info(rom_path, rom_bytes) - basename = rom.name.replace(" ", "").lower() - - header = f"""\ -name: {rom.name.title()} ({rom.get_country_name()}) -sha1: {rom.sha1} -options: - basename: {basename} - target_path: {rom_path.with_suffix(".z64")} - base_path: . - compiler: {rom.compiler} - find_file_boundaries: True - header_encoding: {rom.header_encoding} - platform: n64 - # undefined_funcs_auto: True - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto: True - # undefined_syms_auto_path: undefined_syms_auto.txt - # symbol_addrs_path: symbol_addrs.txt - # asm_path: asm - # src_path: src - # build_path: build - # extensions_path: tools/splat_ext - # mips_abi_float_regs: o32 - # section_order: [".text", ".data", ".rodata", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] - # libultra_symbols: True - # hardware_regs: True -""" - - first_section_end = find_code_length.run(rom_bytes, 0x1000, rom.entry_point) - - segments = f"""\ -segments: - - name: header - type: header - start: 0x0 - - - name: boot - type: bin - start: 0x40 - - - name: entry - type: code - start: 0x1000 - vram: 0x{rom.entry_point:X} - subsegments: - - [0x1000, hasm] - - - name: main - type: code - start: 0x{0x1000 + rom.entrypoint_info.entry_size:X} - vram: 0x{rom.entry_point + rom.entrypoint_info.entry_size:X} - follows_vram: entry -""" - - if rom.entrypoint_info.bss_size is not None: - segments += f"""\ - bss_size: 0x{rom.entrypoint_info.bss_size:X} -""" - - segments += f"""\ - subsegments: - - [0x{0x1000 + rom.entrypoint_info.entry_size:X}, asm] -""" - - if ( - rom.entrypoint_info.bss_size is not None - and rom.entrypoint_info.bss_start_address is not None - ): - bss_start = rom.entrypoint_info.bss_start_address - rom.entry_point + 0x1000 - # first_section_end points to the start of data - segments += f"""\ - - [0x{first_section_end:X}, data] - - {{ start: 0x{bss_start:X}, type: bss, vram: 0x{rom.entrypoint_info.bss_start_address:08X} }} -""" - # Point next segment to the detected end of the main one - first_section_end = bss_start - - segments += f"""\ - - - type: bin - start: 0x{first_section_end:X} - follows_vram: main - - [0x{rom.size:X}] -""" - - out_file = f"{basename}.yaml" - with open(out_file, "w", newline="\n") as f: - print(f"Writing config to {out_file}") - f.write(header) - f.write(segments) - - -def create_gc_config(iso_path: Path, iso_bytes: bytes): - gc = gcinfo.get_info(iso_path, iso_bytes) - basename = gc.system_code + gc.game_code + gc.region_code + gc.publisher_code - - header = f"""\ -name: \"{gc.name.title()} ({gc.get_region_name()})\" -system_code: {gc.system_code} -game_code: {gc.game_code} -region_code: {gc.region_code} -publisher_code: {gc.publisher_code} -sha1: {gc.sha1} -options: - filesystem_path: filesystem - basename: {basename} - target_path: {iso_path.with_suffix(".iso")} - base_path: . - compiler: {gc.compiler} - platform: gc - # undefined_funcs_auto: True - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto: True - # undefined_syms_auto_path: undefined_syms_auto.txt - # symbol_addrs_path: symbol_addrs.txt - # asm_path: asm - # src_path: src - # build_path: build - # extensions_path: tools/splat_ext - # section_order: [".text", ".data", ".rodata", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] -""" - - segments = f"""\ -segments: - - name: filesystem - type: fst - path: filesystem/sys/fst.bin - - name: bootinfo - type: bootinfo - path: filesystem/sys/boot.bin - - name: bi2 - type: bi2 - path: filesystem/sys/bi2.bin - - name: apploader - type: apploader - path: filesystem/sys/apploader.img - - name: main - type: dol - path: filesystem/sys/main.dol -""" - - out_file = f"{basename}.yaml" - with open(out_file, "w", newline="\n") as f: - print(f"Writing config to {out_file}") - f.write(header) - f.write(segments) - - -if __name__ == "__main__": - args = parser.parse_args() - main(Path(args.file)) diff --git a/tools/splat/disassembler/__init__.py b/tools/splat/disassembler/__init__.py deleted file mode 100644 index 09c2ca5..0000000 --- a/tools/splat/disassembler/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .disassembler import Disassembler -from .spimdisasm_disassembler import SpimdisasmDisassembler diff --git a/tools/splat/disassembler/disassembler.py b/tools/splat/disassembler/disassembler.py deleted file mode 100644 index e64aa26..0000000 --- a/tools/splat/disassembler/disassembler.py +++ /dev/null @@ -1,17 +0,0 @@ -from abc import ABC, abstractmethod -from util.options import SplatOpts -from typing import Set - - -class Disassembler(ABC): - @abstractmethod - def configure(self, options: SplatOpts): - raise NotImplementedError("configure") - - @abstractmethod - def check_version(self, skip_version_check: bool, splat_version: str): - raise NotImplementedError("check_version") - - @abstractmethod - def known_types(self) -> Set[str]: - raise NotImplementedError("known_types") diff --git a/tools/splat/disassembler/disassembler_instance.py b/tools/splat/disassembler/disassembler_instance.py deleted file mode 100644 index 988fe3c..0000000 --- a/tools/splat/disassembler/disassembler_instance.py +++ /dev/null @@ -1,26 +0,0 @@ -from .disassembler import Disassembler -from .spimdisasm_disassembler import SpimdisasmDisassembler -from .null_disassembler import NullDisassembler - -__instance: Disassembler = NullDisassembler() -__initialized = False - - -def create_disassembler_instance(platform: str): - global __instance - global __initialized - if platform in ["n64", "psx", "ps2"]: - __instance = SpimdisasmDisassembler() - __initialized = True - return - - raise NotImplementedError("No disassembler for requested platform") - - -def get_instance() -> Disassembler: - global __instance - global __initialized - if not __initialized: - raise Exception("Disassembler instance not initialized") - return None - return __instance diff --git a/tools/splat/disassembler/null_disassembler.py b/tools/splat/disassembler/null_disassembler.py deleted file mode 100644 index 046725e..0000000 --- a/tools/splat/disassembler/null_disassembler.py +++ /dev/null @@ -1,14 +0,0 @@ -from disassembler import disassembler -from util.options import SplatOpts -from typing import Set - - -class NullDisassembler(disassembler.Disassembler): - def configure(self, opts: SplatOpts): - pass - - def check_version(self, skip_version_check: bool, splat_version: str): - pass - - def known_types(self) -> Set[str]: - return set() diff --git a/tools/splat/disassembler/spimdisasm_disassembler.py b/tools/splat/disassembler/spimdisasm_disassembler.py deleted file mode 100644 index c3e49a7..0000000 --- a/tools/splat/disassembler/spimdisasm_disassembler.py +++ /dev/null @@ -1,122 +0,0 @@ -from disassembler import disassembler -import spimdisasm -import rabbitizer -from util import log, compiler -from util.options import SplatOpts -from typing import Set - - -class SpimdisasmDisassembler(disassembler.Disassembler): - # This value should be kept in sync with the version listed on requirements.txt - SPIMDISASM_MIN = (1, 16, 0) - - def configure(self, opts: SplatOpts): - # Configure spimdisasm - spimdisasm.common.GlobalConfig.PRODUCE_SYMBOLS_PLUS_OFFSET = True - spimdisasm.common.GlobalConfig.TRUST_USER_FUNCTIONS = True - spimdisasm.common.GlobalConfig.TRUST_JAL_FUNCTIONS = True - spimdisasm.common.GlobalConfig.GLABEL_ASM_COUNT = False - - if opts.rom_address_padding: - spimdisasm.common.GlobalConfig.ASM_COMMENT_OFFSET_WIDTH = 6 - else: - spimdisasm.common.GlobalConfig.ASM_COMMENT_OFFSET_WIDTH = 0 - - # spimdisasm is not performing any analyzis on non-text sections so enabling this options is pointless - spimdisasm.common.GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_SECTION_TYPE = False - spimdisasm.common.GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_DATA_TYPE = False - - spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False - - if opts.rodata_string_guesser_level is not None: - spimdisasm.common.GlobalConfig.RODATA_STRING_GUESSER_LEVEL = ( - opts.rodata_string_guesser_level - ) - - if opts.data_string_guesser_level is not None: - spimdisasm.common.GlobalConfig.DATA_STRING_GUESSER_LEVEL = ( - opts.data_string_guesser_level - ) - - rabbitizer.config.regNames_userFpcCsr = False - rabbitizer.config.regNames_vr4300Cop0NamedRegisters = False - - rabbitizer.config.misc_opcodeLJust = opts.mnemonic_ljust - 1 - - rabbitizer.config.regNames_gprAbiNames = rabbitizer.Abi.fromStr( - opts.mips_abi_gpr - ) - rabbitizer.config.regNames_fprAbiNames = rabbitizer.Abi.fromStr( - opts.mips_abi_float_regs - ) - - if opts.endianness == "big": - spimdisasm.common.GlobalConfig.ENDIAN = spimdisasm.common.InputEndian.BIG - else: - spimdisasm.common.GlobalConfig.ENDIAN = spimdisasm.common.InputEndian.LITTLE - - rabbitizer.config.pseudos_pseudoMove = False - - selected_compiler = opts.compiler - if selected_compiler == compiler.SN64: - rabbitizer.config.regNames_namedRegisters = False - rabbitizer.config.toolchainTweaks_sn64DivFix = True - rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = True - spimdisasm.common.GlobalConfig.ASM_COMMENT = False - spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = ( - False - ) - spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.SN64 - elif selected_compiler == compiler.GCC: - rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = True - spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.GCC - elif selected_compiler == compiler.IDO: - spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.IDO - - spimdisasm.common.GlobalConfig.DETECT_REDUNDANT_FUNCTION_END = ( - opts.detect_redundant_function_end - ) - - spimdisasm.common.GlobalConfig.GP_VALUE = opts.gp - - spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL = opts.asm_function_macro - spimdisasm.common.GlobalConfig.ASM_JTBL_LABEL = opts.asm_jtbl_label_macro - spimdisasm.common.GlobalConfig.ASM_DATA_LABEL = opts.asm_data_macro - spimdisasm.common.GlobalConfig.ASM_TEXT_END_LABEL = opts.asm_end_label - - if opts.asm_emit_size_directive is not None: - spimdisasm.common.GlobalConfig.ASM_EMIT_SIZE_DIRECTIVE = ( - opts.asm_emit_size_directive - ) - - if spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL == ".globl": - spimdisasm.common.GlobalConfig.ASM_TEXT_ENT_LABEL = ".ent" - spimdisasm.common.GlobalConfig.ASM_TEXT_FUNC_AS_LABEL = True - - if spimdisasm.common.GlobalConfig.ASM_DATA_LABEL == ".globl": - spimdisasm.common.GlobalConfig.ASM_DATA_SYM_AS_LABEL = True - - spimdisasm.common.GlobalConfig.LINE_ENDS = opts.c_newline - - spimdisasm.common.GlobalConfig.ALLOW_ALL_ADDENDS_ON_DATA = ( - opts.allow_data_addends - ) - - spimdisasm.common.GlobalConfig.ASM_GENERATED_BY = opts.asm_generated_by - - spimdisasm.common.GlobalConfig.DISASSEMBLE_UNKNOWN_INSTRUCTIONS = ( - opts.disasm_unknown - ) - - def check_version(self, skip_version_check: bool, splat_version: str): - if not skip_version_check and spimdisasm.__version_info__ < self.SPIMDISASM_MIN: - log.error( - f"splat {splat_version} requires as minimum spimdisasm {self.SPIMDISASM_MIN}, but the installed version is {spimdisasm.__version_info__}" - ) - - log.write( - f"splat {splat_version} (powered by spimdisasm {spimdisasm.__version__})" - ) - - def known_types(self) -> Set[str]: - return spimdisasm.common.gKnownTypes diff --git a/tools/splat/disassembler_section.py b/tools/splat/disassembler_section.py deleted file mode 100644 index 2eb745a..0000000 --- a/tools/splat/disassembler_section.py +++ /dev/null @@ -1,283 +0,0 @@ -import spimdisasm -from util import symbols -from typing import Optional, Set, Tuple -from segtypes.segment import Segment -from util import log, options, symbols - - -from abc import ABC, abstractmethod -from typing import Callable - - -class DisassemblerSection(ABC): - @abstractmethod - def disassemble(self): - raise NotImplementedError("disassemble") - - @abstractmethod - def analyze(self): - raise NotImplementedError("analyze") - - @abstractmethod - def set_comment_offset(self, rom_start: int): - raise NotImplementedError("set_comment_offset") - - @abstractmethod - def make_bss_section( - self, - rom_start, - rom_end, - vram_start, - bss_end, - name, - segment_rom_start, - exclusive_ram_id, - ): - raise NotImplementedError("make_bss_section") - - @abstractmethod - def make_data_section( - self, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ): - raise NotImplementedError("make_data_section") - - @abstractmethod - def get_section(self): - raise NotImplementedError("get_section") - - @abstractmethod - def make_rodata_section( - self, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ): - raise NotImplementedError("make_rodata_section") - - @abstractmethod - def make_text_section( - self, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ): - raise NotImplementedError("make_text_section") - - -class SpimdisasmDisassemberSection(DisassemblerSection): - def __init__(self): - self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None - - def disassemble(self) -> str: - assert self.spim_section is not None - return self.spim_section.disassemble() - - def analyze(self): - assert self.spim_section is not None - self.spim_section.analyze() - - def set_comment_offset(self, rom_start: int): - assert self.spim_section is not None - self.spim_section.setCommentOffset(rom_start) - - def make_bss_section( - self, - rom_start: int, - rom_end: int, - vram_start: int, - bss_end: int, - name: str, - segment_rom_start: int, - exclusive_ram_id, - ): - self.spim_section = spimdisasm.mips.sections.SectionBss( - symbols.spim_context, - rom_start, - rom_end, - vram_start, - bss_end, - name, - segment_rom_start, - exclusive_ram_id, - ) - - def make_data_section( - self, - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, - ): - self.spim_section = spimdisasm.mips.sections.SectionData( - symbols.spim_context, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - - def get_section(self) -> Optional[spimdisasm.mips.sections.SectionBase]: - return self.spim_section - - def make_rodata_section( - self, - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, - ): - self.spim_section = spimdisasm.mips.sections.SectionRodata( - symbols.spim_context, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - - def make_text_section( - self, - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, - ): - self.spim_section = spimdisasm.mips.sections.SectionText( - symbols.spim_context, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - - -def make_disassembler_section() -> Optional[SpimdisasmDisassemberSection]: - if options.opts.platform in ["n64", "psx", "ps2"]: - return SpimdisasmDisassemberSection() - - raise NotImplementedError("No disassembler section for requested platform") - return None - - -def make_text_section( - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, -) -> DisassemblerSection: - section = make_disassembler_section() - assert section is not None - section.make_text_section( - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - return section - - -def make_data_section( - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, -) -> DisassemblerSection: - section = make_disassembler_section() - assert section is not None - section.make_data_section( - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - return section - - -def make_rodata_section( - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, -) -> DisassemblerSection: - section = make_disassembler_section() - assert section is not None - section.make_rodata_section( - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - return section - - -def make_bss_section( - rom_start: int, - rom_end: int, - vram_start: int, - bss_end: int, - name: str, - segment_rom_start: int, - exclusive_ram_id, -) -> DisassemblerSection: - section = make_disassembler_section() - assert section is not None - section.make_bss_section( - rom_start, - rom_end, - vram_start, - bss_end, - name, - segment_rom_start, - exclusive_ram_id, - ) - return section diff --git a/tools/splat/mypy.ini b/tools/splat/mypy.ini deleted file mode 100644 index 6e7c2ef..0000000 --- a/tools/splat/mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -ignore_missing_imports = True -check_untyped_defs = True -mypy_path = stubs diff --git a/tools/splat/platforms/gc.py b/tools/splat/platforms/gc.py deleted file mode 100644 index af08a28..0000000 --- a/tools/splat/platforms/gc.py +++ /dev/null @@ -1,5 +0,0 @@ -from util.gc import gcfst - - -def init(target_bytes: bytes): - gcfst.split_iso(target_bytes) diff --git a/tools/splat/platforms/n64.py b/tools/splat/platforms/n64.py deleted file mode 100644 index ed3ee1c..0000000 --- a/tools/splat/platforms/n64.py +++ /dev/null @@ -1,10 +0,0 @@ -from util import compiler, log, options, palettes, symbols - - -def init(target_bytes: bytes): - symbols.spim_context.fillDefaultBannedSymbols() - - if options.opts.libultra_symbols: - symbols.spim_context.globalSegment.fillLibultraSymbols() - if options.opts.hardware_regs: - symbols.spim_context.globalSegment.fillHardwareRegs(True) diff --git a/tools/splat/platforms/psx.py b/tools/splat/platforms/psx.py deleted file mode 100644 index 09da192..0000000 --- a/tools/splat/platforms/psx.py +++ /dev/null @@ -1,2 +0,0 @@ -def init(target_bytes: bytes): - pass diff --git a/tools/splat/requirements.txt b/tools/splat/requirements.txt deleted file mode 100644 index 429fdea..0000000 --- a/tools/splat/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -PyYAML -pylibyaml -tqdm -intervaltree -colorama -# This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py -spimdisasm>=1.16.0 -rabbitizer>=1.7.0 -pygfxd -n64img>=0.1.4 diff --git a/tools/splat/run_tests.sh b/tools/splat/run_tests.sh deleted file mode 100644 index ca362d7..0000000 --- a/tools/splat/run_tests.sh +++ /dev/null @@ -1,9 +0,0 @@ -# docker build container -docker build --tag splat-build:latest . && \ -# get compilers and tools -# clean -cd test/basic_app && sh clean.sh && cd ../../ && \ -# build -docker run --rm -v $(pwd):/splat -w /splat/test/basic_app splat-build sh build.sh && \ -# test -python3 test.py diff --git a/tools/splat/segtypes/__init__.py b/tools/splat/segtypes/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/tools/splat/segtypes/__init__.py +++ /dev/null diff --git a/tools/splat/segtypes/address_range.py b/tools/splat/segtypes/address_range.py deleted file mode 100644 index 0c57cc0..0000000 --- a/tools/splat/segtypes/address_range.py +++ /dev/null @@ -1,10 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class AddressRange: - start: int - end: int - - def contains(self, addr: int) -> bool: - return self.start <= addr < self.end diff --git a/tools/splat/segtypes/common/__init__.py b/tools/splat/segtypes/common/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/tools/splat/segtypes/common/__init__.py +++ /dev/null diff --git a/tools/splat/segtypes/common/asm.py b/tools/splat/segtypes/common/asm.py deleted file mode 100644 index db7a184..0000000 --- a/tools/splat/segtypes/common/asm.py +++ /dev/null @@ -1,39 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import options - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment - - -class CommonSegAsm(CommonSegCodeSubsegment): - @staticmethod - def is_text() -> bool: - return True - - def out_path(self) -> Optional[Path]: - return options.opts.asm_path / self.dir / f"{self.name}.s" - - def scan(self, rom_bytes: bytes): - if ( - self.rom_start is not None - and self.rom_end is not None - and self.rom_start != self.rom_end - ): - self.scan_code(rom_bytes) - - def get_file_header(self): - return [] - - def split(self, rom_bytes: bytes): - if not self.rom_start == self.rom_end and self.spim_section is not None: - out_path = self.out_path() - if out_path: - out_path.parent.mkdir(parents=True, exist_ok=True) - - self.print_file_boundaries() - - with open(out_path, "w", newline="\n") as f: - for line in self.get_file_header(): - f.write(line + "\n") - f.write(self.spim_section.disassemble()) diff --git a/tools/splat/segtypes/common/bin.py b/tools/splat/segtypes/common/bin.py deleted file mode 100644 index b72a201..0000000 --- a/tools/splat/segtypes/common/bin.py +++ /dev/null @@ -1,28 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import log, options - -from segtypes.common.segment import CommonSegment - - -class CommonSegBin(CommonSegment): - def out_path(self) -> Optional[Path]: - return options.opts.asset_path / self.dir / f"{self.name}.bin" - - def split(self, rom_bytes): - path = self.out_path() - assert path is not None - path.parent.mkdir(parents=True, exist_ok=True) - - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - with open(path, "wb") as f: - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - - f.write(rom_bytes[self.rom_start : self.rom_end]) - self.log(f"Wrote {self.name} to {path}") diff --git a/tools/splat/segtypes/common/bss.py b/tools/splat/segtypes/common/bss.py deleted file mode 100644 index 4bc74d1..0000000 --- a/tools/splat/segtypes/common/bss.py +++ /dev/null @@ -1,62 +0,0 @@ -from util import options, symbols, log - -from segtypes.common.data import CommonSegData - -from disassembler_section import make_bss_section - - -class CommonSegBss(CommonSegData): - def get_linker_section(self) -> str: - return ".bss" - - @staticmethod - def is_noload() -> bool: - return True - - def disassemble_data(self, rom_bytes: bytes): - if not isinstance(self.rom_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" - ) - - # Supposedly logic error, not user error - assert isinstance(self.rom_end, int), f"{self.name} {self.rom_end}" - - # Supposedly logic error, not user error - segment_rom_start = self.get_most_parent().rom_start - assert isinstance(segment_rom_start, int), f"{self.name} {segment_rom_start}" - - if not isinstance(self.vram_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" - ) - - next_subsegment = self.parent.get_next_subsegment_for_ram(self.vram_start) - if next_subsegment is None: - bss_end = self.get_most_parent().vram_end - else: - bss_end = next_subsegment.vram_start - assert isinstance(bss_end, int), f"{self.name} {bss_end}" - - self.spim_section = make_bss_section( - self.rom_start, - self.rom_end, - self.vram_start, - bss_end, - self.name, - segment_rom_start, - self.get_exclusive_ram_id(), - ) - - assert self.spim_section is not None - - self.spim_section.analyze() - self.spim_section.set_comment_offset(self.rom_start) - - for spim_sym in self.spim_section.get_section().symbolList: - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), spim_sym.contextSym - ) - - def should_scan(self) -> bool: - return options.opts.is_mode_active("code") and self.vram_start is not None diff --git a/tools/splat/segtypes/common/c.py b/tools/splat/segtypes/common/c.py deleted file mode 100644 index f23e9cd..0000000 --- a/tools/splat/segtypes/common/c.py +++ /dev/null @@ -1,431 +0,0 @@ -import os -import re -from pathlib import Path -from typing import Optional, Set, List, Tuple - -import spimdisasm - -from util import log, options, symbols -from util.compiler import GCC, SN64, IDO -from util.symbols import Symbol - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment -from segtypes.common.group import CommonSegGroup -from segtypes.common.rodata import CommonSegRodata - - -class CommonSegC(CommonSegCodeSubsegment): - defined_funcs: Set[str] = set() - global_asm_funcs: Set[str] = set() - global_asm_rodata_syms: Set[str] = set() - - STRIP_C_COMMENTS_RE = re.compile( - r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', - re.DOTALL | re.MULTILINE, - ) - - C_FUNC_RE = re.compile( - r"^(?:static\s+)?[^\s]+\s+([^\s(]+)\(([^;)]*)\)[^;]+?{", re.MULTILINE - ) - - C_GLOBAL_ASM_IDO_RE = re.compile( - r"GLOBAL_ASM\(\"(\w+\/)*(\w+)\.s\"\)", re.MULTILINE - ) - - @staticmethod - def strip_c_comments(text): - def replacer(match): - s = match.group(0) - if s.startswith("/"): - return " " - else: - return s - - return re.sub(CommonSegC.STRIP_C_COMMENTS_RE, replacer, text) - - @staticmethod - def get_funcs_defined_in_c(c_file: Path) -> Set[str]: - with open(c_file, "r") as f: - text = CommonSegC.strip_c_comments(f.read()) - - return set(m.group(1) for m in CommonSegC.C_FUNC_RE.finditer(text)) - - @staticmethod - def find_all_instances(string: str, sub: str): - start = 0 - while True: - start = string.find(sub, start) - if start == -1: - return - yield start - start += len(sub) - - @staticmethod - def get_close_parenthesis(string: str, pos: int): - paren_count = 0 - while True: - cur_char = string[pos] - if cur_char == "(": - paren_count += 1 - elif cur_char == ")": - if paren_count == 0: - return pos + 1 - else: - paren_count -= 1 - pos += 1 - - @staticmethod - def find_include_macro(text: str, macro_name: str): - for pos in CommonSegC.find_all_instances(text, f"{macro_name}("): - close_paren_pos = CommonSegC.get_close_parenthesis( - text, pos + len(f"{macro_name}(") - ) - macro_contents = text[pos:close_paren_pos] - macro_args = macro_contents.split(",") - if options.opts.use_legacy_include_asm: - if len(macro_args) >= 3: - yield macro_args[2].strip(" )") - else: - if len(macro_args) >= 2: - yield macro_args[1].strip(" )") - - @staticmethod - def find_include_asm(text: str): - return CommonSegC.find_include_macro(text, "INCLUDE_ASM") - - @staticmethod - def find_include_rodata(text: str): - return CommonSegC.find_include_macro(text, "INCLUDE_RODATA") - - @staticmethod - def get_global_asm_funcs(c_file: Path) -> Set[str]: - with c_file.open() as f: - text = CommonSegC.strip_c_comments(f.read()) - if options.opts.compiler in [GCC, SN64]: - return set(CommonSegC.find_include_asm(text)) - else: - return set( - m.group(2) for m in CommonSegC.C_GLOBAL_ASM_IDO_RE.finditer(text) - ) - - @staticmethod - def get_global_asm_rodata_syms(c_file: Path) -> Set[str]: - with c_file.open() as f: - text = CommonSegC.strip_c_comments(f.read()) - if options.opts.compiler in [GCC, SN64]: - return set(CommonSegC.find_include_rodata(text)) - else: - return set( - m.group(2) for m in CommonSegC.C_GLOBAL_ASM_IDO_RE.finditer(text) - ) - - @staticmethod - def is_text() -> bool: - return True - - def out_path(self) -> Optional[Path]: - return options.opts.src_path / self.dir / f"{self.name}.c" - - def scan(self, rom_bytes: bytes): - if ( - self.rom_start is not None - and self.rom_end is not None - and self.rom_start != self.rom_end - ): - path = self.out_path() - if path: - if options.opts.do_c_func_detection and os.path.exists(path): - # TODO run cpp? - self.defined_funcs = self.get_funcs_defined_in_c(path) - self.global_asm_funcs = self.get_global_asm_funcs(path) - self.global_asm_rodata_syms = self.get_global_asm_rodata_syms(path) - symbols.to_mark_as_defined.update(self.defined_funcs) - symbols.to_mark_as_defined.update(self.global_asm_funcs) - symbols.to_mark_as_defined.update(self.global_asm_rodata_syms) - - self.scan_code(rom_bytes) - - def split(self, rom_bytes: bytes): - if self.rom_start != self.rom_end: - asm_out_dir = options.opts.nonmatchings_path / self.dir - asm_out_dir.mkdir(parents=True, exist_ok=True) - - self.print_file_boundaries() - - assert self.spim_section is not None and isinstance( - self.spim_section.get_section(), spimdisasm.mips.sections.SectionText - ), f"{self.name}, rom_start:{self.rom_start}, rom_end:{self.rom_end}" - - rodata_spim_segment: Optional[spimdisasm.mips.sections.SectionRodata] = None - if ( - options.opts.migrate_rodata_to_functions - and self.rodata_sibling is not None - ): - assert isinstance( - self.rodata_sibling, CommonSegRodata - ), self.rodata_sibling.type - if self.rodata_sibling.spim_section is not None: - assert isinstance( - self.rodata_sibling.spim_section.get_section(), - spimdisasm.mips.sections.SectionRodata, - ) - rodata_spim_segment = self.rodata_sibling.spim_section.get_section() - - # Precompute function-rodata pairings - symbols_entries = ( - spimdisasm.mips.FunctionRodataEntry.getAllEntriesFromSections( - self.spim_section.get_section(), rodata_spim_segment - ) - ) - - is_new_c_file = False - - # Check and create the C file - c_path = self.out_path() - if c_path: - if not c_path.exists() and options.opts.create_c_files: - self.create_c_file(asm_out_dir, c_path, symbols_entries) - is_new_c_file = True - - self.create_asm_dependencies_file( - c_path, asm_out_dir, is_new_c_file, symbols_entries - ) - - # Produce the asm files for functions - for entry in symbols_entries: - if entry.function is not None: - if ( - entry.function.getName() in self.global_asm_funcs - or is_new_c_file - or options.opts.disassemble_all - ): - func_sym = self.get_symbol( - entry.function.vram, - in_segment=True, - type="func", - local_only=True, - ) - assert func_sym is not None - - self.create_c_asm_file(entry, asm_out_dir, func_sym) - else: - for spim_rodata_sym in entry.rodataSyms: - if ( - spim_rodata_sym.getName() in self.global_asm_rodata_syms - or is_new_c_file - or options.opts.disassemble_all - ): - rodata_sym = self.get_symbol( - spim_rodata_sym.vram, in_segment=True, local_only=True - ) - assert rodata_sym is not None - - self.create_unmigrated_rodata_file( - spim_rodata_sym, asm_out_dir, rodata_sym - ) - - def get_c_preamble(self): - ret = [] - - preamble = options.opts.generated_c_preamble - ret.append(preamble) - ret.append("") - - return ret - - def check_gaps_in_migrated_rodata( - self, - func: spimdisasm.mips.symbols.SymbolFunction, - rodata_list: List[spimdisasm.mips.symbols.SymbolBase], - ): - for index in range(len(rodata_list) - 1): - rodata_sym = rodata_list[index] - next_rodata_sym = rodata_list[index + 1] - - if rodata_sym.vramEnd != next_rodata_sym.vram: - log.write( - f"\nA gap was detected in migrated rodata symbols!", status="warn" - ) - log.write( - f"\t In function '{func.getName()}' (0x{func.vram:08X}), gap detected between '{rodata_sym.getName()}' (0x{rodata_sym.vram:08X}) and '{next_rodata_sym.getName()}' (0x{next_rodata_sym.vram:08X})" - ) - log.write( - f"\t The address of the missing rodata symbol is 0x{rodata_sym.vramEnd:08X}" - ) - log.write( - f"\t Try to force the migration of that symbol with `force_migration:True` in the symbol_addrs.txt file; or avoid the migration of symbols around this address with `force_not_migration:True`" - ) - - def create_c_asm_file( - self, - func_rodata_entry: spimdisasm.mips.FunctionRodataEntry, - out_dir: Path, - func_sym: Symbol, - ): - outpath = out_dir / self.name / (func_sym.name + ".s") - - # Skip extraction if the file exists and the symbol is marked as extract=false - if outpath.exists() and not func_sym.extract: - return - - outpath.parent.mkdir(parents=True, exist_ok=True) - - with outpath.open("w", newline="\n") as f: - if options.opts.asm_inc_header: - f.write( - options.opts.c_newline.join(options.opts.asm_inc_header.split("\n")) - ) - - func_rodata_entry.writeToFile(f) - - if func_rodata_entry.function is not None: - self.check_gaps_in_migrated_rodata( - func_rodata_entry.function, func_rodata_entry.rodataSyms - ) - self.check_gaps_in_migrated_rodata( - func_rodata_entry.function, func_rodata_entry.lateRodataSyms - ) - - self.log(f"Disassembled {func_sym.name} to {outpath}") - - def create_unmigrated_rodata_file( - self, - spim_rodata_sym: spimdisasm.mips.symbols.SymbolBase, - out_dir: Path, - rodata_sym: Symbol, - ): - outpath = out_dir / self.name / (rodata_sym.name + ".s") - - # Skip extraction if the file exists and the symbol is marked as extract=false - if outpath.exists() and not rodata_sym.extract: - return - - outpath.parent.mkdir(parents=True, exist_ok=True) - - with outpath.open("w", newline="\n") as f: - if options.opts.include_macro_inc: - f.write('.include "macro.inc"\n\n') - preamble = options.opts.generated_s_preamble - if preamble: - f.write(preamble + "\n") - assert rodata_sym.linker_section is not None, rodata_sym.name - f.write(f".section {rodata_sym.linker_section}\n\n") - f.write(spim_rodata_sym.disassemble()) - - self.log(f"Disassembled {rodata_sym.name} to {outpath}") - - def get_c_line_include_macro( - self, - spim_sym: spimdisasm.mips.symbols.SymbolBase, - asm_out_dir: Path, - macro_name: str, - ) -> str: - if options.opts.compiler == IDO: - # IDO uses the asm processor to embeed assembly, and it doesn't require a special directive to include symbols - asm_outpath = Path( - os.path.join(asm_out_dir, self.name, spim_sym.getName() + ".s") - ) - rel_asm_outpath = os.path.relpath(asm_outpath, options.opts.base_path) - return f'#pragma GLOBAL_ASM("{rel_asm_outpath}")' - - if options.opts.use_legacy_include_asm: - rel_asm_out_dir = asm_out_dir.relative_to(options.opts.nonmatchings_path) - return f'{macro_name}(const s32, "{rel_asm_out_dir / self.name}", {spim_sym.getName()});' - - return f'{macro_name}("{asm_out_dir / self.name}", {spim_sym.getName()});' - - def get_c_lines_for_function( - self, func: spimdisasm.mips.symbols.SymbolFunction, asm_out_dir: Path - ) -> List[str]: - c_lines = [] - - # Terrible hack to "auto-decompile" empty functions - if ( - options.opts.auto_decompile_empty_functions - and len(func.instructions) == 2 - and func.instructions[0].isReturn() - and func.instructions[1].isNop() - ): - c_lines.append("void " + func.getName() + "(void) {") - c_lines.append("}") - else: - c_lines.append( - self.get_c_line_include_macro(func, asm_out_dir, "INCLUDE_ASM") - ) - c_lines.append("") - return c_lines - - def get_c_lines_for_rodata_sym( - self, rodata_sym: spimdisasm.mips.symbols.SymbolBase, asm_out_dir: Path - ): - c_lines = [ - self.get_c_line_include_macro(rodata_sym, asm_out_dir, "INCLUDE_RODATA") - ] - c_lines.append("") - return c_lines - - def create_c_file( - self, - asm_out_dir: Path, - c_path: Path, - symbols_entries: List[spimdisasm.mips.FunctionRodataEntry], - ): - c_lines = self.get_c_preamble() - - for entry in symbols_entries: - if entry.function is not None: - c_lines += self.get_c_lines_for_function(entry.function, asm_out_dir) - else: - for rodata_sym in entry.rodataSyms: - c_lines += self.get_c_lines_for_rodata_sym(rodata_sym, asm_out_dir) - - c_path.parent.mkdir(parents=True, exist_ok=True) - with c_path.open("w") as f: - f.write("\n".join(c_lines)) - log.write(f"Wrote {self.name} to {c_path}") - - def create_asm_dependencies_file( - self, - c_path: Path, - asm_out_dir: Path, - is_new_c_file: bool, - symbols_entries: List[spimdisasm.mips.FunctionRodataEntry], - ): - if not options.opts.create_asm_dependencies: - return - if ( - len(self.global_asm_funcs) + len(self.global_asm_rodata_syms) - ) == 0 and not is_new_c_file: - return - - assert self.spim_section is not None - - build_path = options.opts.build_path - - dep_path = build_path / c_path.with_suffix(".asmproc.d") - dep_path.parent.mkdir(parents=True, exist_ok=True) - with dep_path.open("w") as f: - o_path = build_path / c_path.with_suffix(".o") - f.write(f"{o_path}:") - depend_list = [] - for entry in symbols_entries: - if entry.function is not None: - func_name = entry.function.getName() - - if func_name in self.global_asm_funcs or is_new_c_file: - outpath = asm_out_dir / self.name / (func_name + ".s") - depend_list.append(outpath) - f.write(f" \\\n {outpath}") - else: - for rodata_sym in entry.rodataSyms: - rodata_name = rodata_sym.getName() - - if rodata_name in self.global_asm_rodata_syms or is_new_c_file: - outpath = asm_out_dir / self.name / (rodata_name + ".s") - depend_list.append(outpath) - f.write(f" \\\n {outpath}") - - f.write("\n") - - for depend_file in depend_list: - f.write(f"{depend_file}:\n") diff --git a/tools/splat/segtypes/common/code.py b/tools/splat/segtypes/common/code.py deleted file mode 100644 index 295776f..0000000 --- a/tools/splat/segtypes/common/code.py +++ /dev/null @@ -1,397 +0,0 @@ -import typing -from collections import OrderedDict -from typing import Dict, List, Optional, Tuple, Set - -from util import log, options -from util.range import Range -from util.symbols import Symbol - -from segtypes.common.group import CommonSegGroup -from segtypes.segment import Segment, parse_segment_align - -CODE_TYPES = ["c", "asm", "hasm"] - - -def dotless_type(type: str) -> str: - return type[1:] if type[0] == "." else type - - -# code group -class CommonSegCode(CommonSegGroup): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - self.bss_size: int = yaml.get("bss_size", 0) if isinstance(yaml, dict) else 0 - - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - - self.reported_file_split = False - self.jtbl_glabels_to_add: Set[int] = set() - self.jumptables: Dict[int, Tuple[int, int]] = {} - self.rodata_syms: Dict[int, List[Symbol]] = {} - - self.align = parse_segment_align(yaml) - if self.align is None: - self.align = 0x10 - - @property - def needs_symbols(self) -> bool: - return True - - @property - def vram_end(self) -> Optional[int]: - if self.vram_start is not None and self.size is not None: - return self.vram_start + self.size + self.bss_size - else: - return None - - def check_rodata_sym_impl(self, func_addr: int, sym: Symbol, rodata_section: Range): - if rodata_section.is_complete(): - assert rodata_section.start is not None - assert rodata_section.end is not None - - rodata_start: int = rodata_section.start - rodata_end: int = rodata_section.end - if rodata_start <= sym.vram_start < rodata_end: - if func_addr not in self.rodata_syms: - self.rodata_syms[func_addr] = [] - self.rodata_syms[func_addr].append(sym) - - # Prepare symbol for migration to the function - def check_rodata_sym(self, func_addr: int, sym: Symbol): - rodata_section = self.section_boundaries.get(".rodata") - if rodata_section is not None: - self.check_rodata_sym_impl(func_addr, sym, rodata_section) - rodata_section = self.section_boundaries.get(".rdata") - if rodata_section is not None: - self.check_rodata_sym_impl(func_addr, sym, rodata_section) - - def handle_alls(self, segs: List[Segment], base_segs) -> bool: - for i, elem in enumerate(segs): - if elem.type.startswith("all_"): - alls = [] - - rep_type = f"{elem.type[4:]}" - replace_class = Segment.get_class_for_type(rep_type) - - for base in base_segs.items(): - if isinstance(elem.rom_start, int) and isinstance( - self.rom_start, int - ): - # Shoddy rom to ram - assert self.vram_start is not None, self.vram_start - vram_start = elem.rom_start - self.rom_start + self.vram_start - else: - vram_start = None - rep: Segment = replace_class( - rom_start=elem.rom_start, - rom_end=elem.rom_end, - type=rep_type, - name=base[0], - vram_start=vram_start, - args=[], - yaml={}, - ) - rep.extract = False - rep.given_subalign = self.given_subalign - rep.exclusive_ram_id = self.get_exclusive_ram_id() - rep.given_dir = self.given_dir - rep.given_symbol_name_format = self.symbol_name_format - rep.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom - rep.sibling = base[1] - rep.parent = self - if rep.special_vram_segment: - self.special_vram_segment = True - alls.append(rep) - - # Insert alls into segs at i - del segs[i] - segs[i:i] = alls - return True - return False - - # Find places we should automatically add "all_data" / "all_rodata" / "all_bss" - def find_inserts( - self, found_sections: typing.OrderedDict[str, Range] - ) -> "OrderedDict[str, int]": - inserts: OrderedDict[str, int] = OrderedDict() - - section_order = self.section_order.copy() - section_order.remove(".text") - - for i, section in enumerate(section_order): - if section not in options.opts.auto_all_sections: - continue - - if not found_sections[section].has_start(): - search_done = False - for j in range(i - 1, -1, -1): - end = found_sections[section_order[j]].end - if end is not None: - inserts[section] = end - search_done = True - break - if not search_done: - inserts[section] = -1 - pass - - return inserts - - def parse_subsegments(self, segment_yaml) -> List[Segment]: - if "subsegments" not in segment_yaml: - if not self.parent: - raise Exception( - f"No subsegments provided in top-level code segment {self.name}" - ) - return [] - - base_segments: OrderedDict[str, Segment] = OrderedDict() - ret = [] - prev_start: Optional[int] = -1 - inserts: OrderedDict[ - str, int - ] = ( - OrderedDict() - ) # Used to manually add "all_" types for sections not otherwise defined in the yaml - - self.section_boundaries = OrderedDict( - (s_name, Range()) for s_name in options.opts.section_order - ) - - found_sections = OrderedDict( - (s_name, Range()) for s_name in self.section_boundaries - ) # Stores yaml index where a section was first found - found_sections.pop(".text") - - # Mark any manually added dot types - cur_section = None - - for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]): - # endpos marker - if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1: - continue - - typ = Segment.parse_segment_type(subsegment_yaml) - if typ.startswith("all_"): - typ = typ[4:] - if not typ.startswith("."): - typ = f".{typ}" - - if typ in found_sections: - if cur_section is None: - # Starting point - found_sections[typ].start = i - cur_section = typ - else: - if cur_section != typ: - # We're changing sections - if found_sections[cur_section].has_end(): - log.error( - f"Section {cur_section} end encountered but was already ended earlier!" - ) - if found_sections[typ].has_start(): - log.error( - f"Section {typ} start encounted but has already started earlier!" - ) - - # End the current section - found_sections[cur_section].end = i - - # Start the next section - found_sections[typ].start = i - cur_section = typ - - if cur_section is not None: - found_sections[cur_section].end = -1 - - inserts = self.find_inserts(found_sections) - - last_rom_end = None - - for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]): - # endpos marker - if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1: - continue - - typ = Segment.parse_segment_type(subsegment_yaml) - start = Segment.parse_segment_start(subsegment_yaml) - - # Add dummy segments to be expanded later - if typ.startswith("all_"): - dummy_seg = Segment( - rom_start=start, - rom_end=None, - type=typ, - name="", - vram_start=None, - args=[], - yaml={}, - ) - dummy_seg.given_subalign = self.given_subalign - dummy_seg.exclusive_ram_id = self.exclusive_ram_id - dummy_seg.given_dir = self.given_dir - dummy_seg.given_symbol_name_format = self.symbol_name_format - dummy_seg.given_symbol_name_format_no_rom = ( - self.symbol_name_format_no_rom - ) - ret.append(dummy_seg) - continue - - segment_class = Segment.get_class_for_type(typ) - - end = self.get_next_seg_start(i, segment_yaml["subsegments"]) - - if start is None: - # Attempt to infer the start address - if i == 0: - # The start address of this segment is the start address of the group - start = self.rom_start - else: - # The start address is the end address of the previous segment - start = last_rom_end - - if start is not None and end is None: - est_size = segment_class.estimate_size(subsegment_yaml) - if est_size is not None: - end = start + est_size - - if start is not None and prev_start is not None and start < prev_start: - log.error( - f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})" - ) - - vram = None - if start is not None: - assert isinstance(start, int) - vram = self.get_most_parent().rom_to_ram(start) - - if segment_class.is_noload(): - # Pretend bss's rom address is after the last actual rom segment - start = last_rom_end - # and it has a rom size of zero - end = last_rom_end - - segment: Segment = Segment.from_yaml( - segment_class, subsegment_yaml, start, end, vram - ) - - segment.sibling = base_segments.get(segment.name, None) - - if segment.sibling is not None: - if self.section_order.index(".text") < self.section_order.index( - ".rodata" - ): - if segment.is_rodata(): - segment.sibling.rodata_sibling = segment - else: - if segment.is_text() and segment.sibling.is_rodata(): - segment.rodata_sibling = segment.sibling - segment.sibling.sibling = segment - - segment.parent = self - if segment.special_vram_segment: - self.special_vram_segment = True - - for i, section in enumerate(self.section_order): - if not self.section_boundaries[section].has_start() and dotless_type( - section - ) == dotless_type(segment.type): - if i > 0: - prev_section = self.section_order[i - 1] - self.section_boundaries[prev_section].end = segment.vram_start - self.section_boundaries[section].start = segment.vram_start - - segment.bss_contains_common = self.bss_contains_common - ret.append(segment) - - if segment.is_text(): - base_segments[segment.name] = segment - - if self.section_order.index(".rodata") < self.section_order.index(".text"): - if segment.is_rodata() and segment.sibling is None: - base_segments[segment.name] = segment - - prev_start = start - if end is not None: - last_rom_end = end - - # Add the automatic all_ sections - orig_len = len(ret) - for section in reversed(inserts): - idx = inserts[section] - - if idx == -1: - idx = orig_len - - # bss hack TODO maybe rethink - if ( - section == "bss" - and self.vram_start is not None - and self.rom_end is not None - and self.rom_start is not None - ): - rom_start = self.rom_end - vram_start = self.vram_start + self.rom_end - self.rom_start - else: - rom_start = None - vram_start = None - - new_seg = Segment( - rom_start=rom_start, - rom_end=None, - type="all_" + section, - name="", - vram_start=vram_start, - args=[], - yaml={}, - ) - new_seg.given_subalign = self.given_subalign - new_seg.exclusive_ram_id = self.exclusive_ram_id - new_seg.given_dir = self.given_dir - new_seg.given_symbol_name_format = self.symbol_name_format - new_seg.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom - ret.insert(idx, new_seg) - - check = True - while check: - check = self.handle_alls(ret, base_segments) - - # TODO why is this necessary? - rodata_section = self.section_boundaries.get( - ".rodata" - ) or self.section_boundaries.get(".rdata") - if ( - rodata_section is not None - and rodata_section.has_start() - and not rodata_section.has_end() - ): - assert self.vram_end is not None - rodata_section.end = self.vram_end - - return ret - - def scan(self, rom_bytes): - # Always scan code first - for sub in self.subsegments: - if sub.type in CODE_TYPES and sub.should_scan(): - sub.scan(rom_bytes) - - # Scan everyone else - for sub in self.subsegments: - if sub.type not in CODE_TYPES and sub.should_scan(): - sub.scan(rom_bytes) diff --git a/tools/splat/segtypes/common/codesubsegment.py b/tools/splat/segtypes/common/codesubsegment.py deleted file mode 100644 index 9731c11..0000000 --- a/tools/splat/segtypes/common/codesubsegment.py +++ /dev/null @@ -1,214 +0,0 @@ -from typing import Optional - -import spimdisasm -import rabbitizer - -from util import options, symbols, log - -from segtypes import segment -from segtypes.common.code import CommonSegCode - -from segtypes.segment import Segment - -from disassembler_section import DisassemblerSection, make_text_section - - -# abstract class for c, asm, data, etc -class CommonSegCodeSubsegment(Segment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - vram = segment.parse_segment_vram(self.yaml) - if vram is not None: - self.vram_start = vram - - self.str_encoding: Optional[str] = ( - self.yaml.get("str_encoding", None) if isinstance(self.yaml, dict) else None - ) - - self.spim_section: Optional[DisassemblerSection] = None - self.instr_category = rabbitizer.InstrCategory.CPU - if options.opts.platform == "ps2": - self.instr_category = rabbitizer.InstrCategory.R5900 - elif options.opts.platform == "psx": - self.instr_category = rabbitizer.InstrCategory.R3000GTE - - self.detect_redundant_function_end: Optional[bool] = ( - self.yaml.get("detect_redundant_function_end", None) - if isinstance(self.yaml, dict) - else None - ) - - @property - def needs_symbols(self) -> bool: - return True - - def get_linker_section(self) -> str: - return ".text" - - def scan_code(self, rom_bytes, is_hasm=False): - if not isinstance(self.rom_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" - ) - - # Supposedly logic error, not user error - assert isinstance(self.rom_end, int), self.rom_end - - # Supposedly logic error, not user error - segment_rom_start = self.get_most_parent().rom_start - assert isinstance(segment_rom_start, int), segment_rom_start - - if not isinstance(self.vram_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" - ) - - self.spim_section = make_text_section( - self.rom_start, - self.rom_end, - self.vram_start, - self.name, - rom_bytes, - segment_rom_start, - self.get_exclusive_ram_id(), - ) - - assert self.spim_section is not None - - self.spim_section.get_section().isHandwritten = is_hasm - self.spim_section.get_section().instrCat = self.instr_category - self.spim_section.get_section().detectRedundantFunctionEnd = ( - self.detect_redundant_function_end - ) - - self.spim_section.analyze() - self.spim_section.set_comment_offset(self.rom_start) - - for func in self.spim_section.get_section().symbolList: - assert isinstance(func, spimdisasm.mips.symbols.SymbolFunction) - - self.process_insns(func) - - # Process jumptable labels and pass them to spimdisasm - self.gather_jumptable_labels(rom_bytes) - for jtbl_label_vram in self.parent.jtbl_glabels_to_add: - sym = self.create_symbol( - jtbl_label_vram, True, type="jtbl_label", define=True - ) - sym.type = "jtbl_label" - symbols.add_symbol_to_spim_section(self.spim_section.get_section(), sym) - - def process_insns( - self, - func_spim: spimdisasm.mips.symbols.SymbolFunction, - ): - assert isinstance(self.parent, CommonSegCode) - assert func_spim.vram is not None - assert func_spim.vramEnd is not None - assert self.spim_section is not None - self.parent: CommonSegCode = self.parent - - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), func_spim.contextSym - ) - - # Gather symbols found by spimdisasm and create those symbols in splat's side - for referenced_vram in func_spim.instrAnalyzer.referencedVrams: - context_sym = self.spim_section.get_section().getSymbol( - referenced_vram, tryPlusOffset=False - ) - if context_sym is not None: - if context_sym.type == spimdisasm.common.SymbolSpecialType.jumptable: - self.parent.jumptables[referenced_vram] = ( - func_spim.vram, - func_spim.vramEnd, - ) - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), context_sym - ) - - # Main loop - for i, insn in enumerate(func_spim.instructions): - instr_offset = i * 4 - - # update pointer accesses from this function - if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset: - sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset] - - context_sym = self.spim_section.get_section().getSymbol( - sym_address, tryPlusOffset=False - ) - if context_sym is not None: - sym = symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), context_sym - ) - - if self.parent: - self.parent.check_rodata_sym(func_spim.vram, sym) - - def print_file_boundaries(self): - if not options.opts.find_file_boundaries or not self.spim_section: - return - - assert isinstance(self.rom_start, int) - - for in_file_offset in self.spim_section.get_section().fileBoundaries: - if (in_file_offset % 16) != 0: - continue - - if not self.parent.reported_file_split: - self.parent.reported_file_split = True - - # Look up for the last symbol in this boundary - sym_addr = 0 - for sym in self.spim_section.get_section().symbolList: - symOffset = ( - sym.inFileOffset - self.spim_section.get_section().inFileOffset - ) - if in_file_offset == symOffset: - break - sym_addr = sym.vram - - print( - f"\nSegment {self.name}, symbol at vram {sym_addr:X} ends with extra nops, indicating a likely file split." - ) - print( - "File split suggestions for this segment will follow in config yaml format:" - ) - print(f" - [0x{self.rom_start+in_file_offset:X}, {self.type}]") - - def gather_jumptable_labels(self, rom_bytes): - assert isinstance(self.rom_start, int) - assert isinstance(self.vram_start, int) - - # TODO: use the seg_symbols for this - # jumptables = [j.type == "jtbl" for j in self.seg_symbols] - for jumptable in self.parent.jumptables: - start, end = self.parent.jumptables[jumptable] - rom_offset = self.rom_start + jumptable - self.vram_start - - if rom_offset <= 0: - return - - while rom_offset: - word = rom_bytes[rom_offset : rom_offset + 4] - word_int = int.from_bytes(word, options.opts.endianness) - if word_int >= start and word_int <= end: - self.parent.jtbl_glabels_to_add.add(word_int) - else: - break - - rom_offset += 4 - - def should_scan(self) -> bool: - return ( - options.opts.is_mode_active("code") - and self.rom_start is not None - and self.rom_end is not None - ) - - def should_split(self) -> bool: - return ( - self.extract and options.opts.is_mode_active("code") and self.should_scan() - ) # only split if the segment was scanned first diff --git a/tools/splat/segtypes/common/data.py b/tools/splat/segtypes/common/data.py deleted file mode 100644 index 81c648e..0000000 --- a/tools/splat/segtypes/common/data.py +++ /dev/null @@ -1,147 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import options, symbols, log - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment -from segtypes.common.group import CommonSegGroup - -from disassembler_section import make_data_section - - -class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup): - def asm_out_path(self) -> Path: - typ = self.type - if typ.startswith("."): - typ = typ[1:] - - return options.opts.data_path / self.dir / f"{self.name}.{typ}.s" - - def out_path(self) -> Optional[Path]: - if self.type.startswith("."): - if self.sibling: - # C file - return self.sibling.out_path() - else: - # Implied C file - return options.opts.src_path / self.dir / f"{self.name}.c" - else: - # ASM - return self.asm_out_path() - - def scan(self, rom_bytes: bytes): - CommonSegGroup.scan(self, rom_bytes) - - if self.rom_start is not None and self.rom_end is not None: - self.disassemble_data(rom_bytes) - - def split(self, rom_bytes: bytes): - super().split(rom_bytes) - - if self.type.startswith(".") and not options.opts.disassemble_all: - return - - if self.spim_section is None or not self.should_self_split(): - return - - path = self.asm_out_path() - - path.parent.mkdir(parents=True, exist_ok=True) - - self.print_file_boundaries() - - with path.open("w", newline="\n") as f: - f.write('.include "macro.inc"\n\n') - preamble = options.opts.generated_s_preamble - if preamble: - f.write(preamble + "\n") - - f.write(f".section {self.get_linker_section()}") - section_flags = self.get_section_flags() - if section_flags: - f.write(f', "{section_flags}"') - f.write("\n\n") - - f.write(self.spim_section.disassemble()) - - def should_self_split(self) -> bool: - return options.opts.is_mode_active("data") - - def should_scan(self) -> bool: - return True - - def should_split(self) -> bool: - return True - - def cache(self): - return [CommonSegCodeSubsegment.cache(self), CommonSegGroup.cache(self)] - - def get_linker_section(self) -> str: - return ".data" - - def get_linker_entries(self): - return CommonSegCodeSubsegment.get_linker_entries(self) - - def disassemble_data(self, rom_bytes): - if not isinstance(self.rom_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" - ) - - # Supposedly logic error, not user error - assert isinstance(self.rom_end, int), self.rom_end - - # Supposedly logic error, not user error - segment_rom_start = self.get_most_parent().rom_start - assert isinstance(segment_rom_start, int), segment_rom_start - - if not isinstance(self.vram_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" - ) - - self.spim_section = make_data_section( - self.rom_start, - self.rom_end, - self.vram_start, - self.name, - rom_bytes, - segment_rom_start, - self.get_exclusive_ram_id(), - ) - - assert self.spim_section is not None - - # Set rodata string encoding - # First check the global configuration - if options.opts.data_string_encoding is not None: - self.spim_section.get_section().stringEncoding = ( - options.opts.data_string_encoding - ) - - # Then check the per-segment configuration in case we want to override the global one - if self.str_encoding is not None: - self.spim_section.get_section().stringEncoding = self.str_encoding - - self.spim_section.analyze() - self.spim_section.set_comment_offset(self.rom_start) - - rodata_encountered = False - - for symbol in self.spim_section.get_section().symbolList: - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), symbol.contextSym - ) - - # Hint to the user that we are now in the .rodata section and no longer in the .data section (assuming rodata follows data) - if not rodata_encountered and self.get_most_parent().rodata_follows_data: - if symbol.contextSym.isJumpTable(): - rodata_encountered = True - print( - f"Data segment {self.name}, symbol at vram {symbol.contextSym.vram:X} is a jumptable, indicating the start of the rodata section _may_ be near here." - ) - print( - "Please note the real start of the rodata section may be way before this point." - ) - if symbol.contextSym.vromAddress is not None: - print(f" - [0x{symbol.contextSym.vromAddress:X}, rodata]") diff --git a/tools/splat/segtypes/common/decompressor.py b/tools/splat/segtypes/common/decompressor.py deleted file mode 100644 index 3bdc5ea..0000000 --- a/tools/splat/segtypes/common/decompressor.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Optional, Any - -from util import log, options -from util.n64.decompressor import Decompressor - -from segtypes.n64.segment import N64Segment - - -class CommonSegDecompressor(N64Segment): - decompressor: Decompressor - compression_type = "" # "Mio0" -> filename.Mio0.o - - def split(self, rom_bytes): - if self.decompressor is None: - log.error("Decompressor is not a standalone segment type") - - out_dir = options.opts.asset_path / self.dir - out_dir.mkdir(parents=True, exist_ok=True) - - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - out_path = out_dir / f"{self.name}.bin" - with open(out_path, "wb") as f: - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - - self.log(f"Decompressing {self.name}") - compressed_bytes = rom_bytes[self.rom_start : self.rom_end] - decompressed_bytes = self.decompressor.decompress(compressed_bytes) - f.write(decompressed_bytes) - self.log(f"Wrote {self.name} to {out_path}") - - def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry - - return [ - LinkerEntry( - self, - [options.opts.asset_path / self.dir / f"{self.name}.bin"], - options.opts.asset_path - / self.dir - / f"{self.name}.{self.compression_type}", - self.get_linker_section(), - ) - ] diff --git a/tools/splat/segtypes/common/group.py b/tools/splat/segtypes/common/group.py deleted file mode 100644 index 1242b22..0000000 --- a/tools/splat/segtypes/common/group.py +++ /dev/null @@ -1,160 +0,0 @@ -from typing import List, Optional - -from util import log - -from segtypes.common.segment import CommonSegment -from segtypes.segment import Segment - - -class CommonSegGroup(CommonSegment): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - - self.subsegments: List[Segment] = self.parse_subsegments(yaml) - - def get_next_seg_start(self, i, subsegment_yamls): - return ( - self.rom_end - if i == len(subsegment_yamls) - 1 - else Segment.parse_segment_start(subsegment_yamls[i + 1]) - ) - - def parse_subsegments(self, yaml) -> List[Segment]: - ret: List[Segment] = [] - - if not yaml or "subsegments" not in yaml: - return ret - - prev_start: Optional[int] = -1 - last_rom_end = 0 - - for i, subsegment_yaml in enumerate(yaml["subsegments"]): - # endpos marker - if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1: - continue - - typ = Segment.parse_segment_type(subsegment_yaml) - start = Segment.parse_segment_start(subsegment_yaml) - - segment_class = Segment.get_class_for_type(typ) - - end = self.get_next_seg_start(i, yaml["subsegments"]) - - if start is None: - # Attempt to infer the start address - if i == 0: - # The start address of this segment is the start address of the group - start = self.rom_start - else: - # The start address is the end address of the previous segment - start = last_rom_end - - if start is not None and end is None: - est_size = segment_class.estimate_size(subsegment_yaml) - if est_size is not None: - end = start + est_size - - if start is not None and prev_start is not None and start < prev_start: - log.error( - f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})" - ) - - vram = None - if start is not None: - most_parent = self.get_most_parent() - if ( - most_parent.vram_start is not None - and most_parent.rom_start is not None - ): - vram = most_parent.vram_start + start - most_parent.rom_start - - if segment_class.is_noload(): - # Pretend bss's rom address is after the last actual rom segment - start = last_rom_end - # and it has a rom size of zero - end = last_rom_end - - segment: Segment = Segment.from_yaml( - segment_class, subsegment_yaml, start, end, vram - ) - segment.parent = self - if segment.special_vram_segment: - self.special_vram_segment = True - - ret.append(segment) - prev_start = start - if end is not None: - last_rom_end = end - - return ret - - @property - def needs_symbols(self) -> bool: - for seg in self.subsegments: - if seg.needs_symbols: - return True - return False - - def get_linker_entries(self): - return [entry for sub in self.subsegments for entry in sub.get_linker_entries()] - - def scan(self, rom_bytes): - for sub in self.subsegments: - if sub.should_scan(): - sub.scan(rom_bytes) - - def split(self, rom_bytes): - for sub in self.subsegments: - if sub.should_split(): - sub.split(rom_bytes) - - def should_split(self) -> bool: - return self.extract - - def should_scan(self) -> bool: - return self.extract - - def cache(self): - c = [] - - for sub in self.subsegments: - c.append(sub.cache()) - - return c - - def get_subsegment_for_ram(self, addr: int) -> Optional[Segment]: - for sub in self.subsegments: - if sub.contains_vram(addr): - return sub - return None - - def get_next_subsegment_for_ram(self, addr: int) -> Optional[Segment]: - """ - Returns the first subsegment which comes after the specified address, - or None in case this address belongs to the last subsegment of this group - """ - - for sub in self.subsegments: - if sub.vram_start is None: - continue - assert isinstance(sub.vram_start, int) - if sub.vram_start > addr: - return sub - return None diff --git a/tools/splat/segtypes/common/hasm.py b/tools/splat/segtypes/common/hasm.py deleted file mode 100644 index de7c54c..0000000 --- a/tools/splat/segtypes/common/hasm.py +++ /dev/null @@ -1,24 +0,0 @@ -from segtypes.common.asm import CommonSegAsm - - -class CommonSegHasm(CommonSegAsm): - def scan(self, rom_bytes: bytes): - if ( - self.rom_start is not None - and self.rom_end is not None - and self.rom_start != self.rom_end - ): - self.scan_code(rom_bytes, is_hasm=True) - - def split(self, rom_bytes: bytes): - if not self.rom_start == self.rom_end and self.spim_section is not None: - out_path = self.out_path() - if out_path and not out_path.exists(): - out_path.parent.mkdir(parents=True, exist_ok=True) - - self.print_file_boundaries() - - with open(out_path, "w", newline="\n") as f: - for line in self.get_file_header(): - f.write(line + "\n") - f.write(self.spim_section.disassemble()) diff --git a/tools/splat/segtypes/common/header.py b/tools/splat/segtypes/common/header.py deleted file mode 100644 index b03b0ae..0000000 --- a/tools/splat/segtypes/common/header.py +++ /dev/null @@ -1,42 +0,0 @@ -from pathlib import Path - -from util import options - -from segtypes.common.segment import CommonSegment - - -class CommonSegHeader(CommonSegment): - def should_split(self): - return self.extract and options.opts.is_mode_active("code") - - @staticmethod - def get_line(typ, data, comment): - if typ == "ascii": - text = data.decode("ASCII").strip() - text = text.replace("\x00", "\\0") # escape NUL chars - dstr = '"' + text + '"' - else: # .word, .byte - dstr = "0x" + data.hex().upper() - - dstr = dstr.ljust(20 - len(typ)) - - return f".{typ} {dstr} /* {comment} */" - - def out_path(self) -> Path: - return options.opts.asm_path / self.dir / f"{self.name}.s" - - def parse_header(self, rom_bytes): - return [] - - def split(self, rom_bytes): - header_lines = self.parse_header(rom_bytes) - - src_path = self.out_path() - src_path.parent.mkdir(parents=True, exist_ok=True) - with open(src_path, "w", newline="\n") as f: - f.write("\n".join(header_lines)) - self.log(f"Wrote {self.name} to {src_path}") - - @staticmethod - def get_default_name(addr): - return "header" diff --git a/tools/splat/segtypes/common/lib.py b/tools/splat/segtypes/common/lib.py deleted file mode 100644 index 51371c1..0000000 --- a/tools/splat/segtypes/common/lib.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import log, options - -from segtypes.linker_entry import LinkerEntry -from segtypes.n64.segment import N64Segment - - -class CommonSegLib(N64Segment): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - - if isinstance(yaml, dict): - log.error("Error: 'dict' not currently supported for 'lib' segment") - return - if len(args) < 1: - log.error(f"Error: {self.name} is missing object file") - return - - self.extract = False - - if len(args) > 1: - self.object, self.section = args[0], args[1] - else: - self.object, self.section = args[0], ".text" - - def get_linker_section(self) -> str: - return self.section - - def get_linker_entries(self): - path = options.opts.lib_path / self.name - - object_path = Path(f"{path}.a:{self.object}.o") - - return [LinkerEntry(self, [path], object_path, self.get_linker_section())] diff --git a/tools/splat/segtypes/common/rdata.py b/tools/splat/segtypes/common/rdata.py deleted file mode 100644 index 37adbef..0000000 --- a/tools/splat/segtypes/common/rdata.py +++ /dev/null @@ -1,6 +0,0 @@ -from segtypes.common.rodata import CommonSegRodata - - -class CommonSegRdata(CommonSegRodata): - def get_linker_section(self) -> str: - return ".rdata" diff --git a/tools/splat/segtypes/common/rodata.py b/tools/splat/segtypes/common/rodata.py deleted file mode 100644 index bac0826..0000000 --- a/tools/splat/segtypes/common/rodata.py +++ /dev/null @@ -1,102 +0,0 @@ -from typing import Optional, Set, Tuple -import spimdisasm -from segtypes.segment import Segment -from util import log, options, symbols - -from segtypes.common.data import CommonSegData - -from disassembler_section import make_rodata_section - - -class CommonSegRodata(CommonSegData): - def get_linker_section(self) -> str: - return ".rodata" - - @staticmethod - def is_rodata() -> bool: - return True - - def get_possible_text_subsegment_for_symbol( - self, rodata_sym: spimdisasm.mips.symbols.SymbolBase - ) -> Optional[Tuple[Segment, spimdisasm.common.ContextSymbol]]: - # Check if this rodata segment does not have a corresponding code file, try to look for one - - if self.sibling is not None or not options.opts.pair_rodata_to_text: - return None - - if not rodata_sym.shouldMigrate(): - return None - - if len(rodata_sym.contextSym.referenceFunctions) != 1: - return None - - func = list(rodata_sym.contextSym.referenceFunctions)[0] - text_segment = self.parent.get_subsegment_for_ram(func.vram) - - if text_segment is None or not text_segment.is_text(): - return None - return text_segment, func - - def disassemble_data(self, rom_bytes): - if not isinstance(self.rom_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" - ) - - # Supposedly logic error, not user error - assert isinstance(self.rom_end, int), self.rom_end - - # Supposedly logic error, not user error - segment_rom_start = self.get_most_parent().rom_start - assert isinstance(segment_rom_start, int), segment_rom_start - - if not isinstance(self.vram_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" - ) - - self.spim_section = make_rodata_section( - self.rom_start, - self.rom_end, - self.vram_start, - self.name, - rom_bytes, - segment_rom_start, - self.get_exclusive_ram_id(), - ) - - assert self.spim_section is not None - - # Set rodata string encoding - # First check the global configuration - if options.opts.string_encoding is not None: - self.spim_section.get_section().stringEncoding = ( - options.opts.string_encoding - ) - - # Then check the per-segment configuration in case we want to override the global one - if self.str_encoding is not None: - self.spim_section.get_section().stringEncoding = self.str_encoding - - self.spim_section.analyze() - self.spim_section.set_comment_offset(self.rom_start) - - possible_text_segments: Set[Segment] = set() - - for symbol in self.spim_section.get_section().symbolList: - generated_symbol = symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), symbol.contextSym - ) - generated_symbol.linker_section = self.get_linker_section() - - possible_text = self.get_possible_text_subsegment_for_symbol(symbol) - if possible_text is not None: - text_segment, refenceeFunction = possible_text - if text_segment not in possible_text_segments: - print( - f"\nRodata segment '{self.name}' may belong to the text segment '{text_segment.name}'" - ) - print( - f" Based on the usage from the function {refenceeFunction.getName()} to the symbol {symbol.getName()}" - ) - possible_text_segments.add(text_segment) diff --git a/tools/splat/segtypes/common/sbss.py b/tools/splat/segtypes/common/sbss.py deleted file mode 100644 index cbd0046..0000000 --- a/tools/splat/segtypes/common/sbss.py +++ /dev/null @@ -1,6 +0,0 @@ -from segtypes.common.data import CommonSegData - - -class CommonSegSbss(CommonSegData): - def get_linker_section(self) -> str: - return ".sbss" diff --git a/tools/splat/segtypes/common/sdata.py b/tools/splat/segtypes/common/sdata.py deleted file mode 100644 index a3a0e52..0000000 --- a/tools/splat/segtypes/common/sdata.py +++ /dev/null @@ -1,6 +0,0 @@ -from segtypes.common.data import CommonSegData - - -class CommonSegSdata(CommonSegData): - def get_linker_section(self) -> str: - return ".sdata" diff --git a/tools/splat/segtypes/common/segment.py b/tools/splat/segtypes/common/segment.py deleted file mode 100644 index 10e2b85..0000000 --- a/tools/splat/segtypes/common/segment.py +++ /dev/null @@ -1,5 +0,0 @@ -from segtypes.segment import Segment - - -class CommonSegment(Segment): - pass diff --git a/tools/splat/segtypes/gc/apploader.py b/tools/splat/segtypes/gc/apploader.py deleted file mode 100644 index 810e822..0000000 --- a/tools/splat/segtypes/gc/apploader.py +++ /dev/null @@ -1,8 +0,0 @@ -import struct -from pathlib import Path - -from segtypes.gc.segment import GCSegment - - -class GcSegApploader(GCSegment): - pass diff --git a/tools/splat/segtypes/gc/bi2.py b/tools/splat/segtypes/gc/bi2.py deleted file mode 100644 index 2bf6a97..0000000 --- a/tools/splat/segtypes/gc/bi2.py +++ /dev/null @@ -1,67 +0,0 @@ -import struct -from pathlib import Path - -from util import options - -from segtypes.gc.segment import GCSegment - - -class GcSegBi2(GCSegment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def split(self, bi2_bytes): - lines = [] - - # Gathering variables - debug_monitor_size = struct.unpack_from(">I", bi2_bytes, 0x00)[0] - simulated_memory_size = struct.unpack_from(">I", bi2_bytes, 0x04)[0] - - argument_offset = struct.unpack_from(">I", bi2_bytes, 0x08)[0] - - debug_flag = struct.unpack_from(">I", bi2_bytes, 0x0C)[0] - - track_offset = struct.unpack_from(">I", bi2_bytes, 0x10)[0] - track_size = struct.unpack_from(">I", bi2_bytes, 0x14)[0] - - country_code_bi2 = struct.unpack_from(">I", bi2_bytes, 0x18)[0] - - unk_int = struct.unpack_from(">I", bi2_bytes, 0x1C)[0] - unk_int_2 = struct.unpack_from(">I", bi2_bytes, 0x20)[0] - - # Outputting .s file - lines.append(f"# GameCube disc image bi2 data, located at 0x440 in the disc.\n") - lines.append(f"# Generated by splat.\n\n") - - lines.append(f".section .data\n\n") - - lines.append(f"debug_monitor_size: .long 0x{debug_monitor_size:08X}\n") - lines.append(f"simulated_memory_size: .long 0x{simulated_memory_size:08X}\n\n") - - lines.append(f"argument_offset: .long 0x{argument_offset:08X}\n\n") - - lines.append(f"debug_flag: .long 0x{debug_flag:08X}\n\n") - - lines.append(f"track_offset: .long 0x{track_offset:08X}\n") - lines.append(f"track_size: .long 0x{track_size:08X}\n\n") - - lines.append(f"country_code_bi2: .long 0x{country_code_bi2:08X}\n\n") - - lines.append(f"ukn_int_bi2: .long 0x{unk_int:08X}\n") - lines.append(f"ukn_int_bi2_2: .long 0x{unk_int_2:08X}\n\n") - - lines.append(f".fill 0x00001FDC\n\n") - - out_path = self.out_path() - out_path.parent.mkdir(parents=True, exist_ok=True) - - with open(out_path, "w", encoding="utf-8") as f: - f.writelines(lines) - - return - - def should_split(self) -> bool: - return True - - def out_path(self) -> Path: - return options.opts.asm_path / "sys" / "bi2.s" diff --git a/tools/splat/segtypes/gc/bootinfo.py b/tools/splat/segtypes/gc/bootinfo.py deleted file mode 100644 index 28a93fd..0000000 --- a/tools/splat/segtypes/gc/bootinfo.py +++ /dev/null @@ -1,117 +0,0 @@ -import struct -from pathlib import Path - -from util import options - -from segtypes.gc.segment import GCSegment - - -class GcSegBootinfo(GCSegment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def split(self, iso_bytes): - lines = [] - - gc_dvd_magic = struct.unpack_from(">I", iso_bytes, 0x1C)[0] - assert gc_dvd_magic == 0xC2339F3D - - # Gathering variables - system_code = chr(iso_bytes[0x00]) - game_code = iso_bytes[0x01:0x03].decode("utf-8") - region_code = chr(iso_bytes[0x03]) - publisher_code = iso_bytes[0x04:0x06].decode("utf-8") - - disc_id = iso_bytes[0x06] - game_version = iso_bytes[0x07] - audio_streaming = iso_bytes[0x08] - stream_buffer_size = iso_bytes[0x09] - - name = iso_bytes[0x20:0x400].decode("utf-8").strip("\x00") - name_padding_len = 0x3E0 - len(name) - - # The following is from YAGCD, don't know what they were for: - # https://web.archive.org/web/20220528011846/http://hitmen.c02.at/files/yagcd/yagcd/chap13.html#sec13.1 - apploader_size = struct.unpack_from(">I", iso_bytes, 0x400)[0] - debug_monitor_address = struct.unpack_from(">I", iso_bytes, 0x404)[0] - - # These on the other hand are easy to understand - dol_offset = struct.unpack_from(">I", iso_bytes, 0x420)[0] - fst_offset = struct.unpack_from(">I", iso_bytes, 0x424)[0] - fst_size = struct.unpack_from(">I", iso_bytes, 0x428)[0] - fst_max_size = struct.unpack_from(">I", iso_bytes, 0x42C)[0] - - user_position = struct.unpack_from(">I", iso_bytes, 0x430)[0] - user_length = struct.unpack_from(">I", iso_bytes, 0x434)[0] - unk_int = struct.unpack_from(">I", iso_bytes, 0x438)[0] - - # Outputting .s file - lines.append(f"# GameCube disc image boot data, located at 0x00 in the disc.\n") - lines.append(f"# Generated by splat.\n\n") - - lines.append(f".section .data\n\n") - - # Game ID stuff - lines.append(f'system_code: .ascii "{system_code}"\n') - lines.append(f'game_code: .ascii "{game_code}"\n') - lines.append(f'region_code: .ascii "{region_code}"\n') - lines.append(f'publisher_code: .ascii "{publisher_code}"\n\n') - - lines.append(f"disc_id: .byte {disc_id:X}\n") - lines.append(f"game_version: .byte {game_version:X}\n") - lines.append(f"audio_streaming: .byte {audio_streaming:X}\n") - lines.append(f"stream_buffer_size: .byte {stream_buffer_size:X}\n\n") - - # padding - lines.append(f".fill 0x12\n\n") - - # GC magic number - lines.append(f"gc_magic: .long 0xC2339F3D\n\n") - - # Long game name - lines.append(f'game_name: .ascii "{name}"\n') - lines.append(f".org 0x400\n\n") - - lines.append(f"apploader_size: .long 0x{apploader_size:08X}\n\n") - - # Unknown stuff gleaned from YAGCD - lines.append(f"debug_monitor_address: .long 0x{debug_monitor_address:08X}\n\n") - - # More padding - lines.append(f".fill 0x18\n\n") - - # DOL and FST data - lines.append(f"dol_offset: .long 0x{dol_offset:08X}\n") - lines.append(f"fst_offset: .long 0x{fst_offset:08X}\n\n") - - lines.append( - f"# The FST is only allocated once per game boot, even in games with multiple disks. fst_max_size is used to ensure that\n" - ) - lines.append( - f"# there is enough space allocated for the FST in the event that a game spans multiple disks, and one disk has a larger FST than another.\n" - ) - lines.append(f"fst_size: .long 0x{fst_size:08X}\n") - lines.append(f"fst_max_size: .long 0x{fst_max_size:08X}\n\n") - - # Honestly not sure what this data is for - lines.append(f"# Not even YAGCD knows what these are for.\n") - lines.append(f"user_position: .long 0x{user_position:08X}\n") - lines.append(f"user_length: .long 0x{user_length:08X}\n") - lines.append(f"unk_int: .long 0x{unk_int:08X}\n\n") - - # Final padding - lines.append(f".word 0\n") - out_path = self.out_path() - - out_path.parent.mkdir(parents=True, exist_ok=True) - - with open(out_path, "w", encoding="utf-8") as f: - f.writelines(lines) - - return - - def should_split(self) -> bool: - return True - - def out_path(self) -> Path: - return options.opts.asm_path / "sys" / "boot.s" diff --git a/tools/splat/segtypes/gc/dol.py b/tools/splat/segtypes/gc/dol.py deleted file mode 100644 index 3a92bda..0000000 --- a/tools/splat/segtypes/gc/dol.py +++ /dev/null @@ -1,8 +0,0 @@ -import struct -from pathlib import Path - -from segtypes.gc.segment import GCSegment - - -class GcSegDol(GCSegment): - pass diff --git a/tools/splat/segtypes/gc/dolheader.py b/tools/splat/segtypes/gc/dolheader.py deleted file mode 100644 index f8a060a..0000000 --- a/tools/splat/segtypes/gc/dolheader.py +++ /dev/null @@ -1,68 +0,0 @@ -from util import options - -from segtypes.common.header import CommonSegHeader - - -class DolSegHeader(CommonSegHeader): - def parse_header(self, dol_bytes): - header_lines = [] - header_lines.append(".section .data\n") - - # Text file offsets - for i in range(0x00, 0x1C, 4): - header_lines.append( - self.get_line("word", dol_bytes[i : i + 4], f"Text {i / 4} Offset") - ) - # Data file offsets - for i in range(0x1C, 0x48, 4): - header_lines.append( - self.get_line("word", dol_bytes[i : i + 4], f"Data {i / 4} Offset") - ) - - # Text RAM addresses - for i in range(0x48, 0x64, 4): - header_lines.append( - self.get_line( - "word", - dol_bytes[i : i + 4], - f"Text {(i - 0x48) / 4} Address", - ) - ) - # Data RAM addresses - for i in range(0x64, 0x90, 4): - header_lines.append( - self.get_line( - "word", - dol_bytes[i : i + 4], - f"Data {(i - 0x64) / 4} Address", - ) - ) - - # Text file sizes - for i in range(0x90, 0xAC, 4): - header_lines.append( - self.get_line( - "word", - dol_bytes[i : i + 4], - f"Text {(i - 0x90) / 4} Size", - ) - ) - # Data file sizes - for i in range(0xAC, 0xD8, 4): - header_lines.append( - self.get_line( - "word", - dol_bytes[i : i + 4], - f"Data {(i - 0xAC) / 4} Size", - ) - ) - - # BSS RAM address - header_lines.append(self.get_line("word", dol_bytes[0xD8:0xDC], "BSS Address")) - # BSS size - header_lines.append(self.get_line("word", dol_bytes[0xDC:0xE0], "BSS Size")) - - # Entry point - header_lines.append(self.get_line("word", dol_bytes[0xE0:0xE4], "Entry Point")) - - return header_lines diff --git a/tools/splat/segtypes/gc/fst.py b/tools/splat/segtypes/gc/fst.py deleted file mode 100644 index be091fe..0000000 --- a/tools/splat/segtypes/gc/fst.py +++ /dev/null @@ -1,8 +0,0 @@ -import struct -from pathlib import Path - -from segtypes.gc.segment import GCSegment - - -class GcSegFst(GCSegment): - pass diff --git a/tools/splat/segtypes/gc/rarc.py b/tools/splat/segtypes/gc/rarc.py deleted file mode 100644 index bea41ad..0000000 --- a/tools/splat/segtypes/gc/rarc.py +++ /dev/null @@ -1,321 +0,0 @@ -import struct -from enum import IntEnum -from pathlib import Path - -from typing import List, Optional - -from util import options -from util.gc.gcutil import read_string_from_bytes -from util.n64.Yay0decompress import Yay0Decompressor - -from segtypes.gc.segment import GCSegment - - -# Represents the RARC archive format used by first-party Nintendo games. -class GcSegRarc(GCSegment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def split(self, file_bytes): - assert self.file_path is not None - - archive = GCRARCArchive(self.file_path, file_bytes) - archive.build_hierarchy(file_bytes) - - archive.emit(file_bytes) - - def should_split(self) -> bool: - return True - - -class GCRARCArchive: - def __init__(self, file_path: Path, file_bytes): - self.file_path = file_path - self.compression = "none" - file_bytes = self.try_decompress_archive(file_bytes) - - self.magic = struct.unpack_from(">I", file_bytes, 0x0000)[0] - self.file_size = struct.unpack_from(">I", file_bytes, 0x0004)[0] - self.data_header_offset = struct.unpack_from(">I", file_bytes, 0x0008)[0] - self.file_data_offset = struct.unpack_from(">I", file_bytes, 0x000C)[0] + 0x0020 - self.total_file_data_size = struct.unpack_from(">I", file_bytes, 0x0010)[0] - - self.mram_preload_size = struct.unpack_from(">I", file_bytes, 0x0014)[0] - self.aram_preload_size = struct.unpack_from(">I", file_bytes, 0x0018)[0] - - self.data_header = GCRARCDataHeader(self.data_header_offset, file_bytes) - self.nodes: List[GCRARCNode] = [] - - def try_decompress_archive(self, file_bytes): - compression_scheme = struct.unpack_from(">I", file_bytes, 0x0000)[0] - - # Yaz0 - if compression_scheme == 0x59617A30: - self.compression = "yaz0" - return file_bytes - # Yay0 - elif compression_scheme == 0x59617930: - self.compression = "yay0" - return Yay0Decompressor.decompress(file_bytes) - # Not compressed! - else: - return file_bytes - - def build_hierarchy(self, file_bytes): - string_table_offset = self.data_header.string_table_offset - string_table_size = self.data_header.string_table_size - - string_table_bytes = file_bytes[ - string_table_offset : string_table_offset + string_table_size - ] - - # Load the file entries into their corresponding nodes. - for i in range(self.data_header.node_count): - offset = self.data_header.node_offset + i * 0x10 - - new_node = GCRARCNode(offset, file_bytes, string_table_bytes) - new_node.get_entries( - self.data_header.file_entry_offset, file_bytes, string_table_bytes - ) - - self.nodes.append(new_node) - - # Now, organize the nodes into a hierarchy. - for n in self.nodes: - for e in n.entries: - # We're only looking for directory nodes, so ignore files. - if e.flags & int(GCRARCFlags.IS_FILE) != 0x00: - continue - - if e.name == "." or e.name == "..": - continue - - # This is the node that the current entry corresponds to. - dir_node = self.nodes[e.data_offset] - - # Set up hierarchy relationship. - dir_node.parent = n - n.children.append(dir_node) - - def emit(self, file_bytes): - assert options.opts.filesystem_path is not None - - rel_path = self.file_path.relative_to(options.opts.filesystem_path / "files") - arc_root_path = options.opts.asset_path / rel_path.with_suffix("") - - self.nodes[0].emit_to_filesystem_recursive( - arc_root_path, self.file_data_offset, file_bytes - ) - self.emit_config(arc_root_path) - - def emit_config(self, config_path: Path): - lines = [] - - lines.append(f'name: "{self.file_path.name}"\n') - - if self.compression != "none": - lines.append(f"compression: {self.compression}\n") - - lines.append(f"next_file_id: 0x{self.data_header.next_free_file_id:04X}\n") - lines.append( - f"sync_file_ids_to_indices: {self.data_header.sync_file_ids_to_indices}\n" - ) - - root_node = self.nodes[0] - - lines.append("root_dir:\n") - lines.append(f' res_type: "{root_node.resource_type}"\n') - lines.append(f' name: "{root_node.name}"\n') - - if len(root_node.entries) != 0: - lines.append(" entries:\n") - for e in root_node.entries: - entry_config = e.emit_config(2) - if entry_config != None: - lines.extend(entry_config) - - if len(root_node.children) != 0: - lines.append(" subdirs:\n") - for n in root_node.children: - node_config = n.emit_config(2) - if node_config != None: - lines.extend(node_config) - - with open(config_path / "arcinfo.yaml", "w", newline="\n") as f: - f.writelines(lines) - - -class GCRARCDataHeader: - def __init__(self, offset, file_bytes): - self.node_count = struct.unpack_from(">I", file_bytes, offset + 0x0000)[0] - self.node_offset = ( - struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] + 0x0020 - ) - - self.file_entry_count = struct.unpack_from(">I", file_bytes, offset + 0x0008)[0] - self.file_entry_offset = ( - struct.unpack_from(">I", file_bytes, offset + 0x000C)[0] + 0x0020 - ) - - self.string_table_size = struct.unpack_from(">I", file_bytes, offset + 0x0010)[ - 0 - ] - self.string_table_offset = ( - struct.unpack_from(">I", file_bytes, offset + 0x0014)[0] + 0x0020 - ) - - self.next_free_file_id = struct.unpack_from(">H", file_bytes, offset + 0x0018)[ - 0 - ] - self.sync_file_ids_to_indices = bool(file_bytes[offset + 0x001A]) - - -class GCRARCNode: - def __init__(self, offset, file_bytes, string_table_bytes): - self.resource_type = file_bytes[offset + 0x0000 : offset + 0x0004].decode( - "utf-8" - ) - self.name_offset = struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] - self.name_hash = struct.unpack_from(">H", file_bytes, offset + 0x0008)[0] - self.file_entry_count = struct.unpack_from(">H", file_bytes, offset + 0x000A)[0] - self.first_file_entry_index = struct.unpack_from( - ">I", file_bytes, offset + 0x000C - )[0] - - self.name = read_string_from_bytes(self.name_offset, string_table_bytes) - self.entries = [] - - self.parent: Optional[GCRARCNode] = None - self.children = [] - - def get_entries(self, file_entry_offset, file_bytes, string_table_bytes): - for i in range(self.file_entry_count): - entry_offset = file_entry_offset + (self.first_file_entry_index + i) * 0x14 - - new_entry = GCRARCFileEntry(entry_offset, file_bytes, string_table_bytes) - new_entry.parent_node = self - - self.entries.append(new_entry) - - def emit_to_filesystem_recursive( - self, root_path: Path, file_data_offset, file_bytes - ): - dir_path = root_path / self.get_full_directory_path() - dir_path.mkdir(parents=True, exist_ok=True) - - for n in self.children: - n.emit_to_filesystem_recursive(root_path, file_data_offset, file_bytes) - - for e in self.entries: - e.emit_to_filesystem(root_path, file_data_offset, file_bytes) - - def emit_config(self, level): - lines = [] - - lines.append(" " * level + f'- res_type: "{self.resource_type}"\n') - lines.append(" " * level + f' name: "{self.name}"\n') - - if len(self.entries) != 0: - lines.append(" " * level + " entries:\n") - for e in self.entries: - entry_config = e.emit_config(level + 1) - if entry_config != None: - lines.extend(entry_config) - - if len(self.children) != 0: - lines.append(" " * level + " subdirs:\n") - for n in self.children: - node_config = n.emit_config(level + 1) - if node_config != None: - lines.extend(node_config) - - return lines - - def print_recursive(self, level): - print((" " * level) + self.name) - - for n in self.children: - n.print_recursive(level + 1) - - def get_full_directory_path(self): - path_components: List[str] = [] - - node: Optional[GCRARCNode] = self - while node is not None: - path_components.insert(0, node.name) - node = node.parent - - return Path(*path_components) - - -class GCRARCFileEntry: - def __init__(self, offset, file_bytes, string_table_bytes): - self.file_id = struct.unpack_from(">H", file_bytes, offset + 0x0000)[0] - self.name_hash = struct.unpack_from(">H", file_bytes, offset + 0x0002)[0] - self.flags = file_bytes[offset + 0x0004] - self.name_offset = ( - struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] & 0x00FFFFFF - ) - self.data_offset = struct.unpack_from(">I", file_bytes, offset + 0x0008)[0] - self.data_size = struct.unpack_from(">I", file_bytes, offset + 0x000C)[0] - - self.name = read_string_from_bytes(self.name_offset, string_table_bytes) - self.parent_node: Optional[GCRARCNode] = None - - def emit_to_filesystem(self, dir_path: Path, file_data_offset, file_bytes): - if self.flags & int(GCRARCFlags.IS_DIR) != 0x00: - return - - file_path = dir_path / self.get_full_file_path() - - file_data = file_bytes[ - file_data_offset - + self.data_offset : file_data_offset - + self.data_offset - + self.data_size - ] - with open(file_path, "wb") as f: - f.write(file_data) - - def emit_config(self, level): - if self.flags & int(GCRARCFlags.IS_DIR) != 0x00: - return - - lines = [] - - lines.append(" " * level + f' - name: "{self.name}"\n') - lines.append(" " * level + f" file_id: 0x{self.file_id:04X}\n") - - if self.flags & int(GCRARCFlags.IS_COMPRESSED) != 0x00: - if self.flags & int(GCRARCFlags.IS_YAZ0_COMPRESSED) != 0x00: - lines.append(" " * level + f" compression: yaz0\n") - else: - lines.append(" " * level + f" compression: yay0\n") - - if self.flags & int(GCRARCFlags.PRELOAD_TO_MRAM) == 0x00: - if self.flags & int(GCRARCFlags.PRELOAD_TO_ARAM) != 0x00: - lines.append(" " * level + f" preload_type: aram\n") - else: - lines.append(" " * level + f" preload_type: dvd\n") - - return lines - - def get_full_file_path(self): - path_components = [self.name] - - node = self.parent_node - while node is not None: - path_components.insert(0, node.name) - node = node.parent - - return Path("/".join(path_components)) - - -class GCRARCFlags(IntEnum): - IS_FILE = 0x01 - IS_DIR = 0x02 - IS_COMPRESSED = 0x04 - PRELOAD_TO_MRAM = 0x10 - PRELOAD_TO_ARAM = 0x20 - LOAD_FROM_DVD = 0x40 - IS_YAZ0_COMPRESSED = 0x80 diff --git a/tools/splat/segtypes/gc/relheader.py b/tools/splat/segtypes/gc/relheader.py deleted file mode 100644 index e92fc63..0000000 --- a/tools/splat/segtypes/gc/relheader.py +++ /dev/null @@ -1,114 +0,0 @@ -from util import options - -from segtypes.common.header import CommonSegHeader - - -class RelSegHeader(CommonSegHeader): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - if isinstance(self.yaml, dict): - self.version: int = self.yaml.get("version", 0) - - def parse_header(self, rel_bytes): - header_lines = [] - header_lines.append(".section .data\n") - - # Module ID - header_lines.append(self.get_line("word", rel_bytes[0x00:0x04], "Module ID")) - - # Next module (filled at runtime) - header_lines.append(self.get_line("word", rel_bytes[0x04:0x08], "Next Module")) - # Last module (filled at runtime) - header_lines.append(self.get_line("word", rel_bytes[0x08:0x0C], "Last Module")) - - # Section count - header_lines.append( - self.get_line("word", rel_bytes[0x0C:0x10], "Section Count") - ) - # Section table offset - header_lines.append( - self.get_line("word", rel_bytes[0x10:0x14], "Section Table Offset") - ) - - # Module name offset (might be null) - header_lines.append( - self.get_line("word", rel_bytes[0x14:0x18], "Module Name Offset") - ) - # Module name length - header_lines.append( - self.get_line("word", rel_bytes[0x18:0x1C], "Module Name Length") - ) - - # REL format version - header_lines.append( - self.get_line("word", rel_bytes[0x1C:0x20], "REL Format Version") - ) - - # BSS size - header_lines.append(self.get_line("word", rel_bytes[0x20:0x24], "BSS Size")) - - # Relocation table offset - header_lines.append( - self.get_line("word", rel_bytes[0x24:0x28], "Relocation Table Offset") - ) - # Import table offset - header_lines.append( - self.get_line("word", rel_bytes[0x28:0x2C], "Import Table Offset") - ) - # Import table size - header_lines.append( - self.get_line("word", rel_bytes[0x2C:0x30], "Import Table Size") - ) - - # Prolog section index - header_lines.append( - self.get_line("byte", rel_bytes[0x30:0x31], "Prolog Section Index") - ) - # Epilog section index - header_lines.append( - self.get_line("byte", rel_bytes[0x31:0x32], "Epilog Section Index") - ) - # Unresolved section index - header_lines.append( - self.get_line("byte", rel_bytes[0x32:0x33], "Unresolved Section Index") - ) - # BSS section index (filled at runtime) - header_lines.append( - self.get_line("byte", rel_bytes[0x33:0x34], "BSS Section Index") - ) - - # Prolog function offset - header_lines.append( - self.get_line("word", rel_bytes[0x34:0x38], "Prolog Function Offset") - ) - # Epilog function offset - header_lines.append( - self.get_line("word", rel_bytes[0x38:0x3C], "Epilog Function Offset") - ) - # Unresolved function offset - header_lines.append( - self.get_line("word", rel_bytes[0x3C:0x40], "Unresolved Function Offset") - ) - - # Version 1 is only 0x40 bytes long - if self.version <= 1: - return header_lines - - # Alignment constraint - header_lines.append( - self.get_line("word", rel_bytes[0x40:0x44], "Alignment Constraint") - ) - # BSS alignment constraint - header_lines.append( - self.get_line("word", rel_bytes[0x44:0x48], "BSS Alignment Constraint") - ) - - # Version 2 is only 0x48 bytes long - if self.version <= 2: - return header_lines - - # Fix size - header_lines.append(self.get_line("word", rel_bytes[0x48:0x4C], "Fix Size")) - - return header_lines diff --git a/tools/splat/segtypes/gc/segment.py b/tools/splat/segtypes/gc/segment.py deleted file mode 100644 index 8e4d9bc..0000000 --- a/tools/splat/segtypes/gc/segment.py +++ /dev/null @@ -1,5 +0,0 @@ -from segtypes.segment import Segment - - -class GCSegment(Segment): - pass diff --git a/tools/splat/segtypes/linker_entry.py b/tools/splat/segtypes/linker_entry.py deleted file mode 100644 index fa49e57..0000000 --- a/tools/splat/segtypes/linker_entry.py +++ /dev/null @@ -1,467 +0,0 @@ -import os -import re -from dataclasses import dataclass -from functools import lru_cache -from pathlib import Path -from typing import Dict, List, OrderedDict, Set, Tuple, Union -from segtypes.n64.palette import N64SegPalette - -from util import options - -from segtypes.segment import Segment -from util.symbols import to_cname - - -# clean 'foo/../bar' to 'bar' -@lru_cache(maxsize=None) -def clean_up_path(path: Path) -> Path: - path_resolved = path.resolve() - base_resolved = options.opts.base_path.resolve() - try: - return path_resolved.relative_to(base_resolved) - except ValueError: - pass - - # If the path wasn't relative to the splat file, use the working directory instead - cwd = Path(os.getcwd()) - try: - return path_resolved.relative_to(cwd) - except ValueError: - pass - - # If it wasn't relative to that too, then just return the path as-is - return path - - -def path_to_object_path(path: Path) -> Path: - path = clean_up_path(path) - if options.opts.use_o_as_suffix: - full_suffix = ".o" - else: - full_suffix = path.suffix + ".o" - - if not str(path).startswith(str(options.opts.build_path)): - path = options.opts.build_path / path - return clean_up_path(path.with_suffix(full_suffix)) - - -def write_file_if_different(path: Path, new_content: str): - if path.exists(): - old_content = path.read_text() - else: - old_content = "" - - if old_content != new_content: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w") as f: - f.write(new_content) - - -def segment_cname(segment: Segment) -> str: - name = segment.name - if segment.parent: - name = segment.parent.name + "_" + name - - if isinstance(segment, N64SegPalette): - name += "_pal" - - return to_cname(name) - - -def get_segment_rom_start(cname: str) -> str: - if options.opts.segment_symbols_style == "makerom": - return f"_{cname}SegmentRomStart" - return f"{cname}_ROM_START" - - -def get_segment_rom_end(cname: str) -> str: - if options.opts.segment_symbols_style == "makerom": - return f"_{cname}SegmentRomEnd" - return f"{cname}_ROM_END" - - -def get_segment_vram_start(cname: str) -> str: - if options.opts.segment_symbols_style == "makerom": - return f"_{cname}SegmentStart" - return f"{cname}_VRAM" - - -def get_segment_vram_end(cname: str) -> str: - if options.opts.segment_symbols_style == "makerom": - return f"_{cname}SegmentEnd" - return f"{cname}_VRAM_END" - - -def convert_section_name_to_linker_format(section_type: str) -> str: - assert section_type.startswith(".") - if options.opts.segment_symbols_style == "makerom": - if section_type == ".rodata": - return "RoData" - return section_type[1:].capitalize() - - return to_cname(section_type.upper()) - - -def get_segment_section_start(segment_name: str, section_type: str) -> str: - sec = convert_section_name_to_linker_format(section_type) - if options.opts.segment_symbols_style == "makerom": - return f"_{segment_name}Segment{sec}Start" - return f"{segment_name}{sec}_START" - - -def get_segment_section_end(segment_name: str, section_type: str) -> str: - sec = convert_section_name_to_linker_format(section_type) - if options.opts.segment_symbols_style == "makerom": - return f"_{segment_name}Segment{sec}End" - return f"{segment_name}{sec}_END" - - -def get_segment_section_size(segment_name: str, section_type: str) -> str: - sec = convert_section_name_to_linker_format(section_type) - if options.opts.segment_symbols_style == "makerom": - return f"_{segment_name}Segment{sec}Size" - return f"{segment_name}{sec}_SIZE" - - -def get_segment_vram_end_symbol_name(segment: Segment) -> str: - return get_segment_vram_end(segment_cname(segment)) - - -@dataclass -class LinkerSection: - name: str - started: bool = False - ended: bool = False - - @property - def section_type(self) -> str: - if self.name == ".rdata": - return ".rodata" - return self.name - - -class LinkerEntry: - def __init__( - self, segment: Segment, src_paths: List[Path], object_path: Path, section: str - ): - self.segment = segment - self.src_paths = [clean_up_path(p) for p in src_paths] - self.section = section - if self.section == "linker" or self.section == "linker_offset": - self.object_path = None - elif self.segment.type == "lib": - self.object_path = object_path - else: - self.object_path = path_to_object_path(object_path) - - @property - def section_type(self) -> str: - if self.section == ".rdata": - return ".rodata" - else: - return self.section - - -class LinkerWriter: - def __init__(self): - self.linker_discard_section: bool = options.opts.ld_discard_section - # Used to store all the linker entries - build tools may want this information - self.entries: List[LinkerEntry] = [] - - self.buffer: List[str] = [] - self.header_symbols: Set[str] = set() - - self._indent_level = 0 - - self._writeln("SECTIONS") - self._begin_block() - self._writeln(f"__romPos = {options.opts.ld_rom_start};") - - if options.opts.gp is not None: - self._writeln("_gp = " + f"0x{options.opts.gp:X};") - - # Write a series of statements which compute a symbol that represents the highest address among a list of segments' end addresses - def write_max_vram_end_sym(self, symbol: str, overlays: List[Segment]): - for segment in overlays: - if segment == overlays[0]: - self._writeln( - f"{symbol} = {get_segment_vram_end_symbol_name(segment)};" - ) - else: - self._writeln( - f"{symbol} = MAX({symbol}, {get_segment_vram_end_symbol_name(segment)});" - ) - - # Adds all the entries of a segment to the linker script buffer - def add(self, segment: Segment, max_vram_syms: List[Tuple[str, List[Segment]]]): - entries = segment.get_linker_entries() - self.entries.extend(entries) - - seg_name = segment_cname(segment) - - for sym, segs in max_vram_syms: - self.write_max_vram_end_sym(sym, segs) - - section_labels: OrderedDict[str, LinkerSection] = OrderedDict( - { - l: LinkerSection(l) - for l in options.opts.section_order - if l in options.opts.ld_section_labels - } - ) - - # Start the first linker section - - seg_rom_start = get_segment_rom_start(seg_name) - self._write_symbol(seg_rom_start, "__romPos") - - if entries[0].section_type == ".bss": - self._begin_bss_segment(segment, is_first=True) - seg_bss_start = get_segment_section_start(seg_name, ".bss") - self._write_symbol(seg_bss_start, ".") - if ".bss" in section_labels: - section_labels[".bss"].started = True - else: - self._begin_segment(segment) - - last_seen_sections: Dict[LinkerEntry, str] = {} - - # Find where sections are last seen - for entry in reversed(entries): - if ( - entry.section_type in section_labels.keys() - and entry.section_type not in last_seen_sections.values() - ): - last_seen_sections[entry] = entry.section_type - - cur_section = None - prev_section = None - for entry in entries: - entering_bss = False - leaving_bss = False - cur_section = entry.section_type - - if cur_section == "linker_offset": - self._write_symbol(f"{segment_cname(entry.segment)}_OFFSET", ".") - continue - - for i, section in enumerate(section_labels.values()): - # If we haven't seen this section yet - if not section.started and section.section_type == entry.section_type: - if prev_section == ".bss": - leaving_bss = True - elif cur_section == ".bss": - entering_bss = True - - if not (entering_bss or leaving_bss): - # Don't write a START symbol if we are about to end the section - section_start = get_segment_section_start( - seg_name, entry.section_type - ) - self._write_symbol(section_start, ".") - section_labels[entry.section_type].started = True - - if ( - entry.object_path - and cur_section == ".data" - and entry.segment.type != "lib" - ): - path_cname = re.sub( - r"[^0-9a-zA-Z_]", - "_", - str(entry.segment.dir / entry.segment.name) - + ".".join(entry.object_path.suffixes[:-1]), - ) - self._write_symbol(path_cname, ".") - - wildcard = "*" if options.opts.ld_wildcard_sections else "" - - # Create new linker section for BSS - if entering_bss or leaving_bss: - # If this is the last entry of its type, add the END marker for the section we're ending - if ( - entry in last_seen_sections - and section_labels[entry.section_type].started - ): - self._end_section( - seg_name, last_seen_sections[entry], section_labels - ) - - self._end_block() - - if entering_bss: - self._begin_bss_segment(segment) - else: - self._begin_segment(segment) - - section_start = get_segment_section_start(seg_name, entry.section_type) - self._write_symbol(section_start, ".") - section_labels[cur_section].started = True - - # Write THIS linker entry - self._writeln(f"{entry.object_path}({entry.section}{wildcard});") - else: - # Write THIS linker entry - if entry.section == ".bss" and entry.segment.bss_contains_common: - self._writeln(f"{entry.object_path}(.bss COMMON .scommon);") - else: - self._writeln(f"{entry.object_path}({entry.section}{wildcard});") - - # If this is the last entry of its type, add the END marker for the section we're ending - if entry in last_seen_sections: - self._end_section(seg_name, cur_section, section_labels) - - prev_section = cur_section - - # End all un-ended sections - for section in section_labels.values(): - if section.started and not section.ended: - self._end_section(seg_name, section.name, section_labels) - - all_bss = all(e.section == ".bss" for e in entries) - self._end_segment(segment, all_bss) - - def save_linker_script(self): - if self.linker_discard_section: - self._writeln("/DISCARD/ :") - self._begin_block() - self._writeln("*(*);") - self._end_block() - - self._end_block() # SECTIONS - - assert self._indent_level == 0 - - write_file_if_different( - options.opts.ld_script_path, "\n".join(self.buffer) + "\n" - ) - - def save_symbol_header(self): - path = options.opts.ld_symbol_header_path - - if path: - write_file_if_different( - path, - "#ifndef _HEADER_SYMBOLS_H_\n" - "#define _HEADER_SYMBOLS_H_\n" - "\n" - '#include "common.h"\n' - "\n" - + "".join(f"extern Addr {symbol};\n" for symbol in self.header_symbols) - + "\n" - "#endif\n", - ) - - def _writeln(self, line: str): - if len(line) == 0: - self.buffer.append(line) - else: - self.buffer.append(" " * self._indent_level + line) - - def _begin_block(self): - self._writeln("{") - self._indent_level += 1 - - def _end_block(self): - self._indent_level -= 1 - self._writeln("}") - - def _write_symbol(self, symbol: str, value: Union[str, int]): - symbol = to_cname(symbol) - - if isinstance(value, int): - value = f"0x{value:X}" - - self._writeln(f"{symbol} = {value};") - - self.header_symbols.add(symbol) - - def _begin_segment(self, segment: Segment): - if options.opts.ld_use_follows and segment.vram_of_symbol: - vram_str = segment.vram_of_symbol + " " - else: - vram_str = ( - f"0x{segment.vram_start:X} " - if isinstance(segment.vram_start, int) - else "" - ) - - name = segment_cname(segment) - - seg_vram_start = get_segment_vram_start(name) - self._write_symbol(seg_vram_start, f"ADDR(.{name})") - - seg_rom_start = get_segment_rom_start(name) - line = f".{name} {vram_str}: AT({seg_rom_start})" - if segment.subalign != None: - line += f" SUBALIGN({segment.subalign})" - - self._writeln(line) - self._begin_block() - - def _begin_bss_segment(self, segment: Segment, is_first: bool = False): - if options.opts.ld_use_follows and segment.vram_of_symbol: - vram_str = segment.vram_of_symbol + " " - else: - vram_str = ( - f"0x{segment.vram_start:X} " - if isinstance(segment.vram_start, int) - else "" - ) - - name = segment_cname(segment) + "_bss" - - seg_vram_start = get_segment_vram_start(name) - self._write_symbol(seg_vram_start, f"ADDR(.{name})") - - if is_first: - addr_str = vram_str + "(NOLOAD)" - else: - addr_str = "(NOLOAD)" - - line = f".{name} {addr_str} :" - if segment.subalign != None: - line += f" SUBALIGN({segment.subalign})" - - self._writeln(line) - self._begin_block() - - def _end_segment(self, segment: Segment, all_bss=False): - self._end_block() - - name = segment_cname(segment) - - if not all_bss: - self._writeln(f"__romPos += SIZEOF(.{name});") - - # Align directive - if not options.opts.segment_end_before_align: - if segment.align: - self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});") - - seg_rom_end = get_segment_rom_end(name) - self._write_symbol(seg_rom_end, "__romPos") - self._write_symbol(get_segment_vram_end_symbol_name(segment), ".") - - # Align directive - if options.opts.segment_end_before_align: - if segment.align: - self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});") - - self._writeln("") - - def _end_section( - self, - seg_name: str, - cur_section: str, - section_labels: OrderedDict[str, LinkerSection], - ) -> None: - section_start = get_segment_section_start(seg_name, cur_section) - section_end = get_segment_section_end(seg_name, cur_section) - section_size = get_segment_section_size(seg_name, cur_section) - self._write_symbol(section_end, ".") - self._write_symbol( - section_size, - f"ABSOLUTE({section_end} - {section_start})", - ) - section_labels[cur_section].ended = True diff --git a/tools/splat/segtypes/n64/__init__.py b/tools/splat/segtypes/n64/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/tools/splat/segtypes/n64/__init__.py +++ /dev/null diff --git a/tools/splat/segtypes/n64/asm.py b/tools/splat/segtypes/n64/asm.py deleted file mode 100644 index 4b93b2c..0000000 --- a/tools/splat/segtypes/n64/asm.py +++ /dev/null @@ -1,28 +0,0 @@ -from util import options - -from segtypes.common.asm import CommonSegAsm - - -class N64SegAsm(CommonSegAsm): - @staticmethod - def get_file_header(): - ret = [] - - ret.append('.include "macro.inc"') - ret.append("") - ret.append("/* assembler directives */") - ret.append(".set noat /* allow manual use of $at */") - ret.append(".set noreorder /* don't insert nops after branches */") - if options.opts.add_set_gp_64: - ret.append( - ".set gp=64 /* allow use of 64-bit general purpose registers */" - ) - ret.append("") - preamble = options.opts.generated_s_preamble - if preamble: - ret.append(preamble) - ret.append("") - ret.append('.section .text, "ax"') - ret.append("") - - return ret diff --git a/tools/splat/segtypes/n64/ci.py b/tools/splat/segtypes/n64/ci.py deleted file mode 100644 index 16756bf..0000000 --- a/tools/splat/segtypes/n64/ci.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import Optional, TYPE_CHECKING - -from util import log - -from segtypes.n64.img import N64SegImg - -if TYPE_CHECKING: - from segtypes.n64.palette import N64SegPalette - - -# Base class for CI4/CI8 -class N64SegCi(N64SegImg): - def parse_palette_name(self, yaml, args) -> str: - ret = self.name - if isinstance(yaml, dict): - if "palette" in yaml: - ret = yaml["palette"] - elif len(args) > 2: - ret = args[2] - - return ret - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.palette: "Optional[N64SegPalette]" = None - self.palette_name = self.parse_palette_name(self.yaml, self.args) - - def scan(self, rom_bytes: bytes) -> None: - self.n64img.data = rom_bytes[self.rom_start : self.rom_end] - - def split(self, rom_bytes): - if self.palette is None: - # TODO: output with blank palette - log.error( - f"no palette sibling segment exists\n(hint: add a segment with type 'palette' and name '{self.name}')" - ) - assert self.palette is not None - self.palette.extract = False - self.n64img.palette = self.palette.parse_palette(rom_bytes) - - super().split(rom_bytes) diff --git a/tools/splat/segtypes/n64/ci4.py b/tools/splat/segtypes/n64/ci4.py deleted file mode 100644 index 8325881..0000000 --- a/tools/splat/segtypes/n64/ci4.py +++ /dev/null @@ -1,8 +0,0 @@ -import n64img.image - -from segtypes.n64.ci import N64SegCi - - -class N64SegCi4(N64SegCi): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs, img_cls=n64img.image.CI4) diff --git a/tools/splat/segtypes/n64/ci8.py b/tools/splat/segtypes/n64/ci8.py deleted file mode 100644 index dbf66cd..0000000 --- a/tools/splat/segtypes/n64/ci8.py +++ /dev/null @@ -1,8 +0,0 @@ -import n64img.image - -from segtypes.n64.ci import N64SegCi - - -class N64SegCi8(N64SegCi): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs, img_cls=n64img.image.CI8) diff --git a/tools/splat/segtypes/n64/gfx.py b/tools/splat/segtypes/n64/gfx.py deleted file mode 100644 index 45f4a21..0000000 --- a/tools/splat/segtypes/n64/gfx.py +++ /dev/null @@ -1,277 +0,0 @@ -""" -N64 f3dex display list splitter -Dumps out Gfx[] as a .inc.c file. -""" - -import re -from typing import Optional - -from pathlib import Path - -from pygfxd import ( - gfxd_buffer_to_string, - gfxd_cimg_callback, - gfxd_dl_callback, - gfxd_endian, - gfxd_execute, - gfxd_input_buffer, - gfxd_light_callback, - gfxd_lookat_callback, - gfxd_macro_dflt, - gfxd_macro_fn, - gfxd_mtx_callback, - gfxd_output_buffer, - gfxd_printf, - gfxd_puts, - gfxd_target, - gfxd_timg_callback, - gfxd_tlut_callback, - gfxd_vp_callback, - gfxd_vtx_callback, - gfxd_zimg_callback, - GfxdEndian, - gfxd_f3d, - gfxd_f3db, - gfxd_f3dex, - gfxd_f3dexb, - gfxd_f3dex2, -) -from segtypes.segment import Segment - -from util import log, options -from util.log import error - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment - -from util import symbols - -LIGHTS_RE = re.compile(r"\*\(Lightsn \*\)0x[0-9A-F]{8}") - - -class N64SegGfx(CommonSegCodeSubsegment): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - self.file_text = None - self.data_only = isinstance(yaml, dict) and yaml.get("data_only", False) - self.in_segment = not isinstance(yaml, dict) or yaml.get("in_segment", True) - - def format_sym_name(self, sym) -> str: - return sym.name - - def get_linker_section(self) -> str: - return ".data" - - def out_path(self) -> Path: - return options.opts.asset_path / self.dir / f"{self.name}.gfx.inc.c" - - def scan(self, rom_bytes: bytes): - self.file_text = self.disassemble_data(rom_bytes) - - def get_gfxd_target(self): - opt = options.opts.gfx_ucode - - if opt == "f3d": - return gfxd_f3d - elif opt == "f3db": - return gfxd_f3db - elif opt == "f3dex": - return gfxd_f3dex - elif opt == "f3dexb": - return gfxd_f3dexb - elif opt == "f3dex2": - return gfxd_f3dex2 - else: - log.error(f"Unknown target {opt}") - - def tlut_handler(self, addr, idx, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def timg_handler(self, addr, fmt, size, width, height, pal): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def cimg_handler(self, addr, fmt, size, width): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def zimg_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def dl_handler(self, addr): - # Look for 'Gfx'-typed symbols first - sym = self.retrieve_sym_type(symbols.all_symbols_dict, addr, "Gfx") - - if not sym: - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def mtx_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(f"&{self.format_sym_name(sym)}") - return 1 - - def lookat_handler(self, addr, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def light_handler(self, addr, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def vtx_handler(self, addr, count): - # Look for 'Vtx'-typed symbols first - sym = self.retrieve_sym_type(symbols.all_symbols_dict, addr, "Vtx") - - if not sym: - sym = self.create_symbol( - addr=addr, - in_segment=self.in_segment, - type="Vtx", - reference=True, - search_ranges=True, - ) - - index = int((addr - sym.vram_start) / 0x10) - gfxd_printf(f"&{self.format_sym_name(sym)}[{index}]") - return 1 - - def vp_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(f"&{self.format_sym_name(sym)}") - return 1 - - def macro_fn(self): - gfxd_puts(" ") - gfxd_macro_dflt() - gfxd_puts(",\n") - return 0 - - def disassemble_data(self, rom_bytes): - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - assert isinstance(self.vram_start, int) - - gfx_data = rom_bytes[self.rom_start : self.rom_end] - segment_length = len(gfx_data) - if (segment_length) % 8 != 0: - error( - f"Error: gfx segment {self.name} length ({segment_length}) is not a multiple of 8!" - ) - - out_str = "" if self.data_only else options.opts.generated_c_preamble + "\n\n" - - sym = self.create_symbol( - addr=self.vram_start, in_segment=True, type="data", define=True - ) - - gfxd_input_buffer(gfx_data) - - # TODO terrible guess at the size we'll need - improve this - outb = bytes([0] * segment_length * 100) - outbuf = gfxd_output_buffer(outb, len(outb)) - - gfxd_target(self.get_gfxd_target()) - gfxd_endian( - GfxdEndian.big if options.opts.endianness == "big" else GfxdEndian.little, 4 - ) - - # Callbacks - gfxd_macro_fn(self.macro_fn) - - gfxd_tlut_callback(self.tlut_handler) - gfxd_timg_callback(self.timg_handler) - gfxd_cimg_callback(self.cimg_handler) - gfxd_zimg_callback(self.zimg_handler) - gfxd_dl_callback(self.dl_handler) - gfxd_mtx_callback(self.mtx_handler) - gfxd_lookat_callback(self.lookat_handler) - gfxd_light_callback(self.light_handler) - # gfxd_seg_callback ? - gfxd_vtx_callback(self.vtx_handler) - gfxd_vp_callback(self.vp_handler) - # gfxd_uctext_callback ? - # gfxd_ucdata_callback ? - # gfxd_dram_callback ? - - gfxd_execute() - - if self.data_only: - out_str += gfxd_buffer_to_string(outbuf) - else: - out_str += "Gfx " + self.format_sym_name(sym) + "[] = {\n" - out_str += gfxd_buffer_to_string(outbuf) - out_str += "};\n" - - # Poor man's light fix until we get my libgfxd PR merged - def light_sub_func(match): - light = match.group(0) - addr = int(light[12:], 0) - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - return self.format_sym_name(sym) - - out_str = re.sub(LIGHTS_RE, light_sub_func, out_str) - - return out_str - - def split(self, rom_bytes: bytes): - if self.file_text and self.out_path(): - self.out_path().parent.mkdir(parents=True, exist_ok=True) - - with open(self.out_path(), "w", newline="\n") as f: - f.write(self.file_text) - - def should_scan(self) -> bool: - return ( - options.opts.is_mode_active("gfx") - and self.rom_start is not None - and self.rom_end is not None - ) - - def should_split(self) -> bool: - return self.extract and options.opts.is_mode_active("gfx") diff --git a/tools/splat/segtypes/n64/hasm.py b/tools/splat/segtypes/n64/hasm.py deleted file mode 100644 index 18def92..0000000 --- a/tools/splat/segtypes/n64/hasm.py +++ /dev/null @@ -1,28 +0,0 @@ -from util import options - -from segtypes.common.hasm import CommonSegHasm - - -class N64SegHasm(CommonSegHasm): - @staticmethod - def get_file_header(): - ret = [] - - ret.append('.include "macro.inc"') - ret.append("") - ret.append("/* assembler directives */") - ret.append(".set noat /* allow manual use of $at */") - ret.append(".set noreorder /* don't insert nops after branches */") - if options.opts.add_set_gp_64: - ret.append( - ".set gp=64 /* allow use of 64-bit general purpose registers */" - ) - ret.append("") - preamble = options.opts.generated_s_preamble - if preamble: - ret.append(preamble) - ret.append("") - ret.append('.section .text, "ax"') - ret.append("") - - return ret diff --git a/tools/splat/segtypes/n64/header.py b/tools/splat/segtypes/n64/header.py deleted file mode 100644 index 96c59d5..0000000 --- a/tools/splat/segtypes/n64/header.py +++ /dev/null @@ -1,50 +0,0 @@ -from util import options - -from segtypes.common.header import CommonSegHeader - - -class N64SegHeader(CommonSegHeader): - def parse_header(self, rom_bytes): - encoding = options.opts.header_encoding - - header_lines = [] - header_lines.append(".section .data\n") - header_lines.append( - self.get_line("word", rom_bytes[0x00:0x04], "PI BSB Domain 1 register") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x04:0x08], "Clockrate setting") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x08:0x0C], "Entrypoint address") - ) - header_lines.append(self.get_line("word", rom_bytes[0x0C:0x10], "Revision")) - header_lines.append(self.get_line("word", rom_bytes[0x10:0x14], "Checksum 1")) - header_lines.append(self.get_line("word", rom_bytes[0x14:0x18], "Checksum 2")) - header_lines.append(self.get_line("word", rom_bytes[0x18:0x1C], "Unknown 1")) - header_lines.append(self.get_line("word", rom_bytes[0x1C:0x20], "Unknown 2")) - - if encoding != "word": - header_lines.append( - '.ascii "' - + rom_bytes[0x20:0x34].decode(encoding).strip().ljust(20) - + '" /* Internal name */' - ) - else: - for i in range(0x20, 0x34, 4): - header_lines.append( - self.get_line("word", rom_bytes[i : i + 4], "Internal name") - ) - - header_lines.append(self.get_line("word", rom_bytes[0x34:0x38], "Unknown 3")) - header_lines.append(self.get_line("word", rom_bytes[0x38:0x3C], "Cartridge")) - header_lines.append( - self.get_line("ascii", rom_bytes[0x3C:0x3E], "Cartridge ID") - ) - header_lines.append( - self.get_line("ascii", rom_bytes[0x3E:0x3F], "Country code") - ) - header_lines.append(self.get_line("byte", rom_bytes[0x3F:0x40], "Version")) - header_lines.append("") - - return header_lines diff --git a/tools/splat/segtypes/n64/i1.py b/tools/splat/segtypes/n64/i1.py deleted file mode 100644 index c293099..0000000 --- a/tools/splat/segtypes/n64/i1.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image
-
-from segtypes.n64.img import N64SegImg
-
-
-class N64SegI1(N64SegImg):
- def __init__(self, *args, **kwargs):
- kwargs["img_cls"] = n64img.image.I1
- super().__init__(*args, **kwargs)
diff --git a/tools/splat/segtypes/n64/i4.py b/tools/splat/segtypes/n64/i4.py deleted file mode 100644 index 16a68c5..0000000 --- a/tools/splat/segtypes/n64/i4.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegI4(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.I4 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/i8.py b/tools/splat/segtypes/n64/i8.py deleted file mode 100644 index c2364f2..0000000 --- a/tools/splat/segtypes/n64/i8.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegI8(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.I8 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia16.py b/tools/splat/segtypes/n64/ia16.py deleted file mode 100644 index f288472..0000000 --- a/tools/splat/segtypes/n64/ia16.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegIa16(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.IA16 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia4.py b/tools/splat/segtypes/n64/ia4.py deleted file mode 100644 index 0eef15b..0000000 --- a/tools/splat/segtypes/n64/ia4.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegIa4(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.IA4 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia8.py b/tools/splat/segtypes/n64/ia8.py deleted file mode 100644 index 186be5e..0000000 --- a/tools/splat/segtypes/n64/ia8.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegIa8(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.IA8 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/img.py b/tools/splat/segtypes/n64/img.py deleted file mode 100644 index dd4e32b..0000000 --- a/tools/splat/segtypes/n64/img.py +++ /dev/null @@ -1,99 +0,0 @@ -from pathlib import Path -from typing import Dict, List, Tuple, Type, Optional, Union - -from n64img.image import Image -from util import log, options - -from segtypes.n64.segment import N64Segment - - -class N64SegImg(N64Segment): - @staticmethod - def parse_dimensions(yaml: Union[Dict, List]) -> Tuple[int, int]: - if isinstance(yaml, dict): - return yaml["width"], yaml["height"] - else: - if len(yaml) < 5: - log.error(f"Error: {yaml} is missing width and height parameters") - return yaml[3], yaml[4] - - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - img_cls: Type[Image], - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - - if rom_start is None: - log.error(f"Error: {type} segment {name} rom start could not be determined") - - self.n64img: Image = img_cls(b"", 0, 0) - - if isinstance(yaml, dict): - self.n64img.flip_h = bool(yaml.get("flip_x", False)) - self.n64img.flip_v = bool(yaml.get("flip_y", False)) - - self.width, self.height = self.parse_dimensions(yaml) - - self.n64img.width = self.width - self.n64img.height = self.height - - self.check_len() - - def check_len(self) -> None: - expected_len = int(self.n64img.size()) - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - assert isinstance(self.subalign, int) - actual_len = self.rom_end - self.rom_start - if actual_len > expected_len and actual_len - expected_len > self.subalign: - log.error( - f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)" - ) - - def out_path(self) -> Path: - return options.opts.asset_path / self.dir / f"{self.name}.png" - - def should_split(self) -> bool: - return options.opts.is_mode_active("img") - - def split(self, rom_bytes): - path = self.out_path() - path.parent.mkdir(parents=True, exist_ok=True) - - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - - if self.n64img.data == b"": - self.n64img.data = rom_bytes[self.rom_start : self.rom_end] - self.n64img.write(path) - - self.log(f"Wrote {self.name} to {path}") - - @staticmethod - def estimate_size(yaml: Union[Dict, List]) -> int: - width, height = N64SegImg.parse_dimensions(yaml) - typ = N64Segment.parse_segment_type(yaml) - - if typ == "ci4" or typ == "i4" or typ == "ia4": - return width * height // 2 - elif typ in ("ia16", "rgba16"): - return width * height * 2 - elif typ == "rgba32": - return width * height * 4 - else: - return width * height diff --git a/tools/splat/segtypes/n64/ipl3.py b/tools/splat/segtypes/n64/ipl3.py deleted file mode 100644 index 6a76cb3..0000000 --- a/tools/splat/segtypes/n64/ipl3.py +++ /dev/null @@ -1,9 +0,0 @@ -from segtypes.common.code import CommonSegCode -from segtypes.common.hasm import CommonSegHasm - - -class N64SegIpl3(CommonSegHasm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.special_vram_segment = True diff --git a/tools/splat/segtypes/n64/linker_offset.py b/tools/splat/segtypes/n64/linker_offset.py deleted file mode 100644 index ac4d945..0000000 --- a/tools/splat/segtypes/n64/linker_offset.py +++ /dev/null @@ -1,10 +0,0 @@ -from pathlib import Path - -from segtypes.n64.segment import N64Segment - - -class N64SegLinker_offset(N64Segment): - def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry - - return [LinkerEntry(self, [], Path(self.name), "linker_offset")] diff --git a/tools/splat/segtypes/n64/mio0.py b/tools/splat/segtypes/n64/mio0.py deleted file mode 100644 index 5ad4635..0000000 --- a/tools/splat/segtypes/n64/mio0.py +++ /dev/null @@ -1,10 +0,0 @@ -from util.n64.Mio0decompress import Mio0Decompressor - -from segtypes.common.decompressor import CommonSegDecompressor - - -class N64SegMio0(CommonSegDecompressor): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.decompressor = Mio0Decompressor() - self.compression_type = "Mio0" diff --git a/tools/splat/segtypes/n64/palette.py b/tools/splat/segtypes/n64/palette.py deleted file mode 100644 index 553f642..0000000 --- a/tools/splat/segtypes/n64/palette.py +++ /dev/null @@ -1,106 +0,0 @@ -from itertools import zip_longest -from pathlib import Path -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING, Union - -from util import log, options -from util.color import unpack_color - -from segtypes.n64.segment import N64Segment - -if TYPE_CHECKING: - from segtypes.n64.ci import N64SegCi as Raster - - -def iter_in_groups(iterable, n, fillvalue=None): - args = [iter(iterable)] * n - return zip_longest(*args, fillvalue=fillvalue) - - -VALID_SIZES = [0x20, 0x40, 0x80, 0x100, 0x200] - - -class N64SegPalette(N64Segment): - require_unique_name = False - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.raster: "Optional[Raster]" = None - - # palette segments must be named as one of the following: - # 1) same as the relevant raster segment name (max. 1 palette) - # 2) relevant raster segment name + "." + unique palette name - # 3) unique, referencing the relevant raster segment using `raster_name` - self.raster_name = ( - self.yaml.get("raster_name", self.name.split(".")[0]) - if isinstance(self.yaml, dict) - else self.name.split(".")[0] - ) - - if self.extract: - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - if not isinstance(self.yaml, dict) or "size" not in self.yaml: - assert self.rom_end is not None - assert self.rom_start is not None - actual_len = self.rom_end - self.rom_start - - hint_msg = "(hint: add a 'bin' segment after it or specify the size in the segment)" - - if actual_len > VALID_SIZES[-1]: - log.error( - f"Error: {self.name} (0x{actual_len:X} bytes) is too long, max 0x{VALID_SIZES[-1]:X})\n{hint_msg}" - ) - - if actual_len not in VALID_SIZES: - log.error( - f"Error: {self.name} (0x{actual_len:X} bytes) is not a valid palette size ({', '.join(hex(s) for s in VALID_SIZES)})\n{hint_msg}" - ) - - def split(self, rom_bytes): - if self.raster is None: - # TODO: output with no raster - log.error(f"orphaned palette segment: {self.name} lacks ci4/ci8 sibling") - - assert self.raster is not None - self.raster.n64img.palette = self.parse_palette(rom_bytes) # type: ignore - - self.raster.n64img.write(self.out_path()) - self.raster.extract = False - - def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]: - data = rom_bytes[self.rom_start : self.rom_end] - palette = [] - - for a, b in iter_in_groups(data, 2): - palette.append(unpack_color([a, b])) - - return palette - - def out_path(self) -> Path: - return options.opts.asset_path / self.dir / f"{self.name}.png" - - def should_split(self) -> bool: - return self.extract and options.opts.is_mode_active("img") - - def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry - - return [ - LinkerEntry( - self, - [options.opts.asset_path / self.dir / f"{self.name}.png"], - options.opts.asset_path / self.dir / f"{self.name}.pal", - self.get_linker_section(), - ) - ] - - @staticmethod - def estimate_size(yaml: Union[Dict, List]) -> int: - if isinstance(yaml, dict): - if "size" in yaml: - return int(yaml["size"]) - return 0x20 diff --git a/tools/splat/segtypes/n64/rgba16.py b/tools/splat/segtypes/n64/rgba16.py deleted file mode 100644 index 4b6f4fd..0000000 --- a/tools/splat/segtypes/n64/rgba16.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegRgba16(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.RGBA16 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/rgba32.py b/tools/splat/segtypes/n64/rgba32.py deleted file mode 100644 index ae3d538..0000000 --- a/tools/splat/segtypes/n64/rgba32.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegRgba32(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.RGBA32 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/rsp.py b/tools/splat/segtypes/n64/rsp.py deleted file mode 100644 index f69123a..0000000 --- a/tools/splat/segtypes/n64/rsp.py +++ /dev/null @@ -1,10 +0,0 @@ -import rabbitizer - -from segtypes.common.hasm import CommonSegHasm - - -class N64SegRsp(CommonSegHasm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.instr_category = rabbitizer.InstrCategory.RSP diff --git a/tools/splat/segtypes/n64/segment.py b/tools/splat/segtypes/n64/segment.py deleted file mode 100644 index 8f9c3cc..0000000 --- a/tools/splat/segtypes/n64/segment.py +++ /dev/null @@ -1,5 +0,0 @@ -from segtypes.segment import Segment - - -class N64Segment(Segment): - pass diff --git a/tools/splat/segtypes/n64/vtx.py b/tools/splat/segtypes/n64/vtx.py deleted file mode 100644 index e0d1bc9..0000000 --- a/tools/splat/segtypes/n64/vtx.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -N64 Vtx struct splitter -Dumps out Vtx as a .inc.c file. - -Originally written by Mark Street (https://github.com/mkst) -""" - -import struct -from pathlib import Path -from typing import Optional - -from util import options, log - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment - - -class N64SegVtx(CommonSegCodeSubsegment): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - self.file_text: Optional[str] = None - self.data_only = isinstance(yaml, dict) and yaml.get("data_only", False) - - def format_sym_name(self, sym) -> str: - return sym.name - - def get_linker_section(self) -> str: - return ".data" - - def out_path(self) -> Path: - return options.opts.asset_path / self.dir / f"{self.name}.vtx.inc.c" - - def scan(self, rom_bytes: bytes): - self.file_text = self.disassemble_data(rom_bytes) - - def disassemble_data(self, rom_bytes) -> str: - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - assert isinstance(self.vram_start, int) - - vertex_data = rom_bytes[self.rom_start : self.rom_end] - segment_length = len(vertex_data) - if (segment_length) % 16 != 0: - log.error( - f"Error: Vtx segment {self.name} length ({segment_length}) is not a multiple of 16!" - ) - - lines = [] - if not self.data_only: - lines.append(options.opts.generated_c_preamble) - lines.append("") - - vertex_count = segment_length // 16 - sym = self.create_symbol( - addr=self.vram_start, in_segment=True, type="data", define=True - ) - - if not self.data_only: - lines.append(f"Vtx {self.format_sym_name(sym)}[{vertex_count}] = {{") - - for vtx in struct.iter_unpack(">hhhHhhBBBB", vertex_data): - x, y, z, flg, t, c, r, g, b, a = vtx - vtx_string = f" {{{{{{ {x:5}, {y:5}, {z:5} }}, {flg}, {{ {t:5}, {c:5} }}, {{ {r:3}, {g:3}, {b:3}, {a:3} }}}}}}," - if flg != 0: - self.warn(f"Non-zero flag found in vertex data {self.name}!") - lines.append(vtx_string) - - if not self.data_only: - lines.append("};") - - # enforce newline at end of file - lines.append("") - return "\n".join(lines) - - def split(self, rom_bytes: bytes): - if self.file_text and self.out_path(): - self.out_path().parent.mkdir(parents=True, exist_ok=True) - - with open(self.out_path(), "w", newline="\n") as f: - f.write(self.file_text) - - def should_scan(self) -> bool: - return options.opts.is_mode_active("vtx") - - def should_split(self) -> bool: - return self.extract and options.opts.is_mode_active("vtx") diff --git a/tools/splat/segtypes/n64/yay0.py b/tools/splat/segtypes/n64/yay0.py deleted file mode 100644 index 210f761..0000000 --- a/tools/splat/segtypes/n64/yay0.py +++ /dev/null @@ -1,10 +0,0 @@ -from util.n64.Yay0decompress import Yay0Decompressor - -from segtypes.common.decompressor import CommonSegDecompressor - - -class N64SegYay0(CommonSegDecompressor): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.decompressor = Yay0Decompressor() - self.compression_type = "Yay0" diff --git a/tools/splat/segtypes/psx/__init__.py b/tools/splat/segtypes/psx/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/tools/splat/segtypes/psx/__init__.py +++ /dev/null diff --git a/tools/splat/segtypes/psx/asm.py b/tools/splat/segtypes/psx/asm.py deleted file mode 100644 index 2e9ff5e..0000000 --- a/tools/splat/segtypes/psx/asm.py +++ /dev/null @@ -1,23 +0,0 @@ -from util import options - -from segtypes.common.asm import CommonSegAsm - - -class PsxSegAsm(CommonSegAsm): - @staticmethod - def get_file_header(): - ret = [] - - ret.append('.include "macro.inc"') - ret.append("") - ret.append(".set noat") - ret.append(".set noreorder") - ret.append("") - preamble = options.opts.generated_s_preamble - if preamble: - ret.append(preamble) - ret.append("") - ret.append('.section .text, "ax"') - ret.append("") - - return ret diff --git a/tools/splat/segtypes/psx/header.py b/tools/splat/segtypes/psx/header.py deleted file mode 100644 index aeda3ee..0000000 --- a/tools/splat/segtypes/psx/header.py +++ /dev/null @@ -1,61 +0,0 @@ -from segtypes.common.header import CommonSegHeader - - -class PsxSegHeader(CommonSegHeader): - # little endian so reverse words, TODO: use struct.unpack("<i",...) ? - # breakdown from https://psx-spx.consoledev.net/cdromdrive/#filenameexe-general-purpose-executable - def parse_header(self, rom_bytes): - header_lines = [] - header_lines.append(".section .data\n") - header_lines.append( - self.get_line("ascii", rom_bytes[0x00:0x08], "Magic number") - ) - header_lines.append(self.get_line("word", rom_bytes[0x08:0x0C], "Zerofilled")) - header_lines.append(self.get_line("word", rom_bytes[0x0C:0x10], "Zerofilled")) - header_lines.append( - self.get_line("word", rom_bytes[0x10:0x14][::-1], "Initial PC") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x14:0x18][::-1], "Initial $gp/r28") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x18:0x1C][::-1], ".text start") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x1C:0x20][::-1], ".text size") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x20:0x24][::-1], ".data start") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x24:0x28][::-1], ".data size") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x28:0x2C][::-1], ".bss start") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x2C:0x30][::-1], ".bss size") - ) - header_lines.append( - self.get_line( - "word", rom_bytes[0x30:0x34][::-1], "Initial $sp/r29 & $fp/r30 base" - ) - ) - header_lines.append( - self.get_line( - "word", rom_bytes[0x34:0x38][::-1], "Initial $sp/r29 & $fp/r30 offset" - ) - ) - header_lines.append(self.get_line("word", rom_bytes[0x38:0x3C], "Reserved")) - header_lines.append(self.get_line("word", rom_bytes[0x3C:0x40], "Reserved")) - header_lines.append(self.get_line("word", rom_bytes[0x40:0x44], "Reserved")) - header_lines.append(self.get_line("word", rom_bytes[0x44:0x48], "Reserved")) - header_lines.append(self.get_line("word", rom_bytes[0x48:0x4C], "Reserved")) - assert isinstance(self.rom_end, int) - header_lines.append( - self.get_line("ascii", rom_bytes[0x4C : self.rom_end], "Sony Inc") - ) - - header_lines.append("") - - return header_lines diff --git a/tools/splat/segtypes/psx/segment.py b/tools/splat/segtypes/psx/segment.py deleted file mode 100644 index eff2875..0000000 --- a/tools/splat/segtypes/psx/segment.py +++ /dev/null @@ -1,5 +0,0 @@ -from segtypes.segment import Segment - - -class PsxSegment(Segment): - pass diff --git a/tools/splat/segtypes/segment.py b/tools/splat/segtypes/segment.py deleted file mode 100644 index 6f03426..0000000 --- a/tools/splat/segtypes/segment.py +++ /dev/null @@ -1,649 +0,0 @@ -import importlib -import importlib.util -from pathlib import Path - -from typing import Dict, List, Optional, Set, Type, TYPE_CHECKING, Union - -from intervaltree import Interval, IntervalTree - -from util import log, options, symbols -from util.symbols import Symbol, to_cname - -# circular import -if TYPE_CHECKING: - from segtypes.linker_entry import LinkerEntry - - -def parse_segment_vram(segment: Union[dict, list]) -> Optional[int]: - if isinstance(segment, dict) and "vram" in segment: - return int(segment["vram"]) - else: - return None - - -def parse_segment_align(segment: Union[dict, list]) -> Optional[int]: - if isinstance(segment, dict) and "align" in segment: - return int(segment["align"]) - return None - - -def parse_segment_subalign(segment: Union[dict, list]) -> int: - default = options.opts.subalign - if isinstance(segment, dict): - subalign = segment.get("subalign", default) - if subalign != None: - subalign = int(subalign) - return subalign - return default - - -def parse_segment_section_order(segment: Union[dict, list]) -> List[str]: - default = options.opts.section_order - if isinstance(segment, dict): - return segment.get("section_order", default) - return default - - -def parse_segment_follows_vram(segment: Union[dict, list]) -> Optional[str]: - if isinstance(segment, dict): - return segment.get("follows_vram", None) - return None - - -def parse_segment_vram_of_symbol(segment: Union[dict, list]) -> Optional[str]: - if isinstance(segment, dict): - return segment.get("vram_of_symbol", segment.get("follows_vram_symbol", None)) - return None - - -class Segment: - require_unique_name = True - - @staticmethod - def get_class_for_type(seg_type) -> Type["Segment"]: - # so .data loads SegData, for example - if seg_type.startswith("."): - seg_type = seg_type[1:] - - segment_class = Segment.get_base_segment_class(seg_type) - if segment_class == None: - # Look in extensions - segment_class = Segment.get_extension_segment_class(seg_type) - return segment_class - - @staticmethod - def get_base_segment_class(seg_type): - platform = options.opts.platform - is_platform_seg = False - - # heirarchy is platform -> common -> fail - try: - segmodule = importlib.import_module(f"segtypes.{platform}.{seg_type}") - is_platform_seg = True - except ModuleNotFoundError: - try: - segmodule = importlib.import_module(f"segtypes.common.{seg_type}") - except ModuleNotFoundError: - return None - - seg_prefix = platform.capitalize() if is_platform_seg else "Common" - return getattr(segmodule, f"{seg_prefix}Seg{seg_type.capitalize()}") - - @staticmethod - def get_extension_segment_class(seg_type): - platform = options.opts.platform - - ext_path = options.opts.extensions_path - if not ext_path: - log.error( - f"could not load presumed extended segment type '{seg_type}' because no extensions path is configured" - ) - assert ext_path is not None - - try: - ext_spec = importlib.util.spec_from_file_location( - f"{platform}.segtypes.{seg_type}", ext_path / f"{seg_type}.py" - ) - assert ext_spec is not None - ext_mod = importlib.util.module_from_spec(ext_spec) - assert ext_spec.loader is not None - ext_spec.loader.exec_module(ext_mod) - except Exception as err: - log.write(err, status="error") - log.error( - f"could not load segment type '{seg_type}'\n(hint: confirm your extension directory is configured correctly)" - ) - - return getattr( - ext_mod, f"{platform.upper()}Seg{seg_type[0].upper()}{seg_type[1:]}" - ) - - @staticmethod - def parse_segment_start(segment: Union[dict, list]) -> Optional[int]: - if isinstance(segment, dict): - s = segment.get("start", "auto") - else: - s = segment[0] - - if s == "auto": - return None - elif s == "...": - return None - else: - return int(s) - - @staticmethod - def parse_segment_type(segment: Union[dict, list]) -> str: - if isinstance(segment, dict): - return str(segment["type"]) - else: - return str(segment[1]) - - @staticmethod - def parse_segment_name(cls, rom_start, segment: Union[dict, list]) -> str: - if isinstance(segment, dict) and "name" in segment: - return str(segment["name"]) - elif isinstance(segment, dict) and "dir" in segment: - return str(segment["dir"]) - elif isinstance(segment, list) and len(segment) >= 3: - return str(segment[2]) - else: - return str(cls.get_default_name(rom_start)) - - @staticmethod - def parse_segment_symbol_name_format(segment: Union[dict, list]) -> str: - if isinstance(segment, dict) and "symbol_name_format" in segment: - return str(segment["symbol_name_format"]) - else: - return options.opts.symbol_name_format - - @staticmethod - def parse_segment_symbol_name_format_no_rom(segment: Union[dict, list]) -> str: - if isinstance(segment, dict) and "symbol_name_format_no_rom" in segment: - return str(segment["symbol_name_format_no_rom"]) - else: - return options.opts.symbol_name_format_no_rom - - @staticmethod - def parse_segment_file_path(segment: Union[dict, list]) -> Optional[Path]: - if isinstance(segment, dict) and "path" in segment: - return Path(segment["path"]) - return None - - @staticmethod - def parse_segment_bss_contains_common(segment: Union[dict, list]) -> bool: - if isinstance(segment, dict) and "bss_contains_common" in segment: - return bool(segment["bss_contains_common"]) - else: - return False - - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - self.rom_start = rom_start - self.rom_end = rom_end - self.type = type - self.name = name - self.vram_start: Optional[int] = vram_start - - self.align: Optional[int] = None - self.given_subalign: int = options.opts.subalign - self.exclusive_ram_id: Optional[str] = None - self.given_dir: Path = Path() - - # Symbols known to be in this segment - self.given_seg_symbols: Dict[int, List[Symbol]] = {} - - # Ranges for faster symbol lookup - self.symbol_ranges_ram: IntervalTree = IntervalTree() - self.symbol_ranges_rom: IntervalTree = IntervalTree() - - self.given_section_order: List[str] = options.opts.section_order - self.given_follows_vram: Optional[str] = None - self.vram_of_symbol: Optional[str] = None - - self.given_symbol_name_format: str = options.opts.symbol_name_format - self.given_symbol_name_format_no_rom: str = ( - options.opts.symbol_name_format_no_rom - ) - - self.parent: Optional[Segment] = None - self.sibling: Optional[Segment] = None - self.rodata_sibling: Optional[Segment] = None - self.file_path: Optional[Path] = None - - self.args: List[str] = args - self.yaml = yaml - - self.extract: bool = True - self.has_linker_entry: bool = True - if self.rom_start is None: - self.extract = False - elif self.type.startswith("."): - self.extract = False - - self.warnings: List[str] = [] - self.did_run = False - self.bss_contains_common = Segment.parse_segment_bss_contains_common(yaml) - - # For segments which are not in the usual VRAM segment space, like N64's IPL3 which lives in 0xA4... - self.special_vram_segment: bool = False - - if self.rom_start is not None and self.rom_end is not None: - if self.rom_start > self.rom_end: - log.error( - f"Error: segments out of order - ({self.name} starts at 0x{self.rom_start:X}, but next segment starts at 0x{self.rom_end:X})" - ) - - @staticmethod - def from_yaml( - cls: Type["Segment"], - yaml: Union[dict, list], - rom_start: Optional[int], - rom_end: Optional[int], - vram=None, - ): - type = Segment.parse_segment_type(yaml) - name = Segment.parse_segment_name(cls, rom_start, yaml) - vram_start = vram if vram is not None else parse_segment_vram(yaml) - - args: List[str] = [] if isinstance(yaml, dict) else yaml[3:] - - ret = cls( - rom_start=rom_start, - rom_end=rom_end, - type=type, - name=name, - vram_start=vram_start, - args=args, - yaml=yaml, - ) - ret.given_section_order = parse_segment_section_order(yaml) - ret.given_subalign = parse_segment_subalign(yaml) - if isinstance(yaml, dict): - ret.extract = bool(yaml.get("extract", ret.extract)) - ret.exclusive_ram_id = yaml.get("exclusive_ram_id") - ret.given_dir = Path(yaml.get("dir", "")) - ret.has_linker_entry = bool(yaml.get("linker_entry", True)) - ret.given_symbol_name_format = Segment.parse_segment_symbol_name_format(yaml) - ret.given_symbol_name_format_no_rom = ( - Segment.parse_segment_symbol_name_format_no_rom(yaml) - ) - ret.file_path = Segment.parse_segment_file_path(yaml) - - ret.bss_contains_common = Segment.parse_segment_bss_contains_common(yaml) - if not ret.given_follows_vram: - ret.given_follows_vram = parse_segment_follows_vram(yaml) - - if not ret.vram_of_symbol: - ret.vram_of_symbol = parse_segment_vram_of_symbol(yaml) - - if not ret.align: - ret.align = parse_segment_align(yaml) - return ret - - # For executable segments (.text); like c, asm or hasm - @staticmethod - def is_text() -> bool: - return False - - # For readonly segments (.rodata); like rodata or rdata - @staticmethod - def is_rodata() -> bool: - return False - - # For segments which does not take space in ROM; like bss - @staticmethod - def is_noload() -> bool: - return False - - @staticmethod - def estimate_size(yaml: Union[Dict, List]) -> Optional[int]: - return None - - @property - def needs_symbols(self) -> bool: - return False - - @property - def dir(self) -> Path: - if self.parent: - return self.parent.dir / self.given_dir - else: - return self.given_dir - - @property - def symbol_name_format(self) -> str: - return self.given_symbol_name_format - - @property - def symbol_name_format_no_rom(self) -> str: - return self.given_symbol_name_format_no_rom - - @property - def subalign(self) -> int: - if self.parent: - return self.parent.subalign - else: - return self.given_subalign - - def get_exclusive_ram_id(self) -> Optional[str]: - if self.parent: - return self.parent.get_exclusive_ram_id() - return self.exclusive_ram_id - - def add_symbol(self, symbol: Symbol): - if symbol.vram_start not in self.given_seg_symbols: - self.given_seg_symbols[symbol.vram_start] = [] - self.given_seg_symbols[symbol.vram_start].append(symbol) - - # For larger symbols, add their ranges to interval trees for faster lookup - if symbol.size > 4: - self.symbol_ranges_ram.addi(symbol.vram_start, symbol.vram_end, symbol) - if symbol.rom is not None: - self.symbol_ranges_rom.addi(symbol.rom, symbol.rom_end, symbol) - - @property - def seg_symbols(self) -> Dict[int, List[Symbol]]: - if self.parent: - return self.parent.seg_symbols - else: - return self.given_seg_symbols - - @property - def size(self) -> Optional[int]: - if self.rom_start is not None and self.rom_end is not None: - return self.rom_end - self.rom_start - else: - return None - - @property - def vram_end(self) -> Optional[int]: - if self.vram_start is not None and self.size is not None: - return self.vram_start + self.size - else: - return None - - @property - def section_order(self) -> List[str]: - return self.given_section_order - - @property - def rodata_follows_data(self) -> bool: - if ".rodata" not in self.section_order or ".data" not in self.section_order: - return False - return ( - self.section_order.index(".rodata") - self.section_order.index(".data") == 1 - ) - - def contains_vram(self, vram: int) -> bool: - if self.vram_start is not None and self.vram_end is not None: - return vram >= self.vram_start and vram < self.vram_end - else: - return False - - def contains_rom(self, rom: int) -> bool: - if self.rom_start is not None and self.rom_end is not None: - return rom >= self.rom_start and rom < self.rom_end - else: - return False - - def rom_to_ram(self, rom_addr: int) -> Optional[int]: - if self.vram_start is not None and self.rom_start is not None: - return self.vram_start + rom_addr - self.rom_start - else: - return None - - def ram_to_rom(self, ram_addr: int) -> Optional[int]: - if not self.contains_vram(ram_addr) and ram_addr != self.vram_end: - return None - - if self.vram_start is not None and self.rom_start is not None: - return self.rom_start + ram_addr - self.vram_start - else: - return None - - def should_scan(self) -> bool: - return self.should_split() - - def should_split(self) -> bool: - return self.extract and options.opts.is_mode_active(self.type) - - def scan(self, rom_bytes: bytes): - pass - - def split(self, rom_bytes: bytes): - pass - - def cache(self): - return (self.yaml, self.rom_end) - - def get_linker_section(self) -> str: - return ".data" - - def get_section_flags(self) -> Optional[str]: - """ - Allows specifying flags for a section. - - This can be useful when creating a custom section, since sections not recognized by the linker will not be linked properly. - - GNU as docs about the section directive and flags: https://sourceware.org/binutils/docs/as/Section.html#ELF-Version - - Example: - - ``` - def get_section_flags(self) -> Optional[str]: - # Tells the linker to allocate this section - return "a" - ``` - """ - return None - - def out_path(self) -> Optional[Path]: - return None - - def get_most_parent(self) -> "Segment": - seg = self - - while seg.parent: - seg = seg.parent - - return seg - - def get_linker_entries(self) -> "List[LinkerEntry]": - from segtypes.linker_entry import LinkerEntry - - if not self.has_linker_entry: - return [] - - path = self.out_path() - - if path: - return [LinkerEntry(self, [path], path, self.get_linker_section())] - else: - return [] - - def log(self, msg): - if options.opts.verbose: - log.write(f"{self.type} {self.name}: {msg}") - - def warn(self, msg: str): - self.warnings.append(msg) - - @staticmethod - def get_default_name(addr) -> str: - return f"{addr:X}" - - def is_name_default(self): - return self.name == self.get_default_name(self.rom_start) - - def unique_id(self): - if self.parent: - s = self.parent.unique_id() + "_" - else: - s = "" - - return s + self.type + "_" + self.name - - @staticmethod - def visible_ram(seg1: "Segment", seg2: "Segment") -> bool: - if seg1.get_most_parent() == seg2.get_most_parent(): - return True - if seg1.get_exclusive_ram_id() is None or seg2.get_exclusive_ram_id() is None: - return True - return seg1.get_exclusive_ram_id() != seg2.get_exclusive_ram_id() - - def retrieve_symbol( - self, syms: Dict[int, List[Symbol]], addr: int - ) -> Optional[Symbol]: - if addr not in syms: - return None - - items = syms[addr] - - # Filter out symbols that are in different top-level segments with the same unique_ram_id - items = [ - i - for i in items - if i.segment is None or Segment.visible_ram(self, i.segment) - ] - - if len(items) > 1: - # print(f"Trying to retrieve {addr:X} from symbol dict but there are {len(items)} entries to pick from - picking the first") - pass - if len(items) == 0: - return None - return items[0] - - def retrieve_sym_type( - self, syms: Dict[int, List[Symbol]], addr: int, type: str - ) -> Optional[symbols.Symbol]: - if addr not in syms: - return None - - items = syms[addr] - - items = [ - i - for i in items - if i.segment is None - or Segment.visible_ram(self, i.segment) - and (type == i.type) - ] - - if len(items) == 0: - return None - - return items[0] - - def get_symbol( - self, - addr: int, - in_segment: bool = False, - type: Optional[str] = None, - create: bool = False, - define: bool = False, - reference: bool = False, - search_ranges: bool = False, - local_only: bool = False, - dead: bool = True, - ) -> Optional[Symbol]: - ret: Optional[Symbol] = None - rom: Optional[int] = None - - most_parent = self.get_most_parent() - - if in_segment: - # If the vram address is within this segment, we can calculate the symbol's rom address - rom = most_parent.ram_to_rom(addr) - ret = most_parent.retrieve_symbol(most_parent.seg_symbols, addr) - - if not ret and search_ranges: - # Search ranges first, starting with rom - if rom is not None: - cands: Set[Interval] = most_parent.symbol_ranges_rom[rom] - if cands: - ret = cands.pop().data - # and then vram if we can't find a rom match - if not ret: - cands = most_parent.symbol_ranges_ram[addr] - if cands: - ret = cands.pop().data - elif not local_only: - ret = most_parent.retrieve_symbol(symbols.all_symbols_dict, addr) - - if not ret and search_ranges: - cands = symbols.all_symbols_ranges[addr] - if cands: - ret = cands.pop().data - - # Reject dead symbols unless we allow them - if not dead and ret and ret.dead: - ret = None - - # Create the symbol if it doesn't exist - if not ret and create: - ret = Symbol(addr, rom=rom, type=type) - symbols.add_symbol(ret) - - if in_segment: - ret.segment = most_parent - if addr not in most_parent.seg_symbols: - most_parent.seg_symbols[addr] = [] - most_parent.seg_symbols[addr].append(ret) - - if ret: - if define: - ret.defined = True - if reference: - ret.referenced = True - if ret.type is None: - ret.type = type - if ret.rom is None: - ret.rom = rom - if in_segment: - if ret.segment is None: - ret.segment = most_parent - - return ret - - def create_symbol( - self, - addr: int, - in_segment: bool, - type: Optional[str] = None, - define: bool = False, - reference: bool = False, - search_ranges: bool = False, - local_only: bool = False, - dead: bool = True, - ) -> Symbol: - ret = self.get_symbol( - addr, - in_segment=in_segment, - type=type, - create=True, - define=define, - reference=reference, - search_ranges=search_ranges, - local_only=local_only, - dead=dead, - ) - assert ret is not None - - return ret - - def get_func_for_addr(self, addr) -> Optional[Symbol]: - for syms in self.seg_symbols.values(): - for sym in syms: - if sym.type == "func" and sym.contains_vram(addr): - return sym - - return None diff --git a/tools/splat/split.py b/tools/splat/split.py deleted file mode 100755 index 8568450..0000000 --- a/tools/splat/split.py +++ /dev/null @@ -1,506 +0,0 @@ -#! /usr/bin/env python3 - -import argparse -import hashlib -import importlib -import pickle -from typing import Any, Dict, List, Optional, Set, Tuple, Union -from disassembler import disassembler_instance -from util import progress_bar - -# This unused import makes the yaml library faster. don't remove -import pylibyaml # pyright: ignore -import yaml - -from colorama import Fore, Style -from intervaltree import Interval, IntervalTree -import sys - -from segtypes.linker_entry import ( - LinkerWriter, - get_segment_vram_end_symbol_name, - to_cname, -) -from segtypes.segment import Segment -from util import log, options, palettes, symbols, relocs - -VERSION = "0.16.9" - -parser = argparse.ArgumentParser( - description="Split a rom given a rom, a config, and output directory" -) -parser.add_argument("config", help="path to a compatible config .yaml file", nargs="+") -parser.add_argument("--modes", nargs="+", default="all") -parser.add_argument("--verbose", action="store_true", help="Enable debug logging") -parser.add_argument( - "--use-cache", action="store_true", help="Only split changed segments in config" -) -parser.add_argument( - "--skip-version-check", - action="store_true", - help="Skips the disassembler's version check", -) -parser.add_argument( - "--stdout-only", help="Print all output to stdout", action="store_true" -) -parser.add_argument( - "--disassemble-all", - help="Disasemble matched functions and migrated data", - action="store_true", -) - -linker_writer: LinkerWriter -config: Dict[str, Any] - -segment_roms: IntervalTree = IntervalTree() -segment_rams: IntervalTree = IntervalTree() - - -def fmt_size(size): - if size > 1000000: - return str(size // 1000000) + " MB" - elif size > 1000: - return str(size // 1000) + " KB" - else: - return str(size) + " B" - - -def initialize_segments(config_segments: Union[dict, list]) -> List[Segment]: - global segment_roms - global segment_rams - - segment_roms = IntervalTree() - segment_rams = IntervalTree() - - segments_by_name: Dict[str, Segment] = {} - ret = [] - - last_rom_end = 0 - - for i, seg_yaml in enumerate(config_segments): - # end marker - if isinstance(seg_yaml, list) and len(seg_yaml) == 1: - continue - - seg_type = Segment.parse_segment_type(seg_yaml) - - segment_class = Segment.get_class_for_type(seg_type) - - this_start = Segment.parse_segment_start(seg_yaml) - - if i == len(config_segments) - 1 and Segment.parse_segment_file_path(seg_yaml): - next_start: Optional[int] = 0 - else: - next_start = Segment.parse_segment_start(config_segments[i + 1]) - - if segment_class.is_noload(): - # Pretend bss's rom address is after the last actual rom segment - this_start = last_rom_end - # and it has a rom size of zero - next_start = last_rom_end - - segment: Segment = Segment.from_yaml( - segment_class, seg_yaml, this_start, next_start - ) - - if segment.require_unique_name: - if segment.name in segments_by_name: - log.error(f"segment name '{segment.name}' is not unique") - - segments_by_name[segment.name] = segment - - ret.append(segment) - if ( - isinstance(segment.rom_start, int) - and isinstance(segment.rom_end, int) - and segment.rom_start != segment.rom_end - ): - segment_roms.addi(segment.rom_start, segment.rom_end, segment) - if ( - isinstance(segment.vram_start, int) - and isinstance(segment.vram_end, int) - and segment.vram_start != segment.vram_end - ): - segment_rams.addi(segment.vram_start, segment.vram_end, segment) - - if next_start is not None: - last_rom_end = next_start - - for segment in ret: - if segment.given_follows_vram: - if segment.given_follows_vram not in segments_by_name: - log.error( - f"segment '{segment.given_follows_vram}', the 'follows_vram' value for segment '{segment.name}', does not exist" - ) - segment.vram_of_symbol = get_segment_vram_end_symbol_name( - segments_by_name[segment.given_follows_vram] - ) - - return ret - - -def assign_symbols_to_segments(): - for symbol in symbols.all_symbols: - if symbol.segment: - continue - - if symbol.rom: - cands: Set[Interval] = segment_roms[symbol.rom] - if len(cands) > 1: - log.error("multiple segments rom overlap symbol", symbol) - elif len(cands) == 0: - log.error("no segment rom overlaps symbol", symbol) - else: - cand: Interval = cands.pop() - seg: Segment = cand.data - seg.add_symbol(symbol) - else: - cands = segment_rams[symbol.vram_start] - segs: List[Segment] = [cand.data for cand in cands] - for seg in segs: - if not seg.get_exclusive_ram_id(): - seg.add_symbol(symbol) - - -def do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached): - unk_size = seg_sizes.get("unk", 0) - rest_size = 0 - total_size = len(rom_bytes) - - for typ in seg_sizes: - if typ != "unk": - rest_size += seg_sizes[typ] - - known_ratio = rest_size / total_size - unk_ratio = unk_size / total_size - - log.write(f"Split {fmt_size(rest_size)} ({known_ratio:.2%}) in defined segments") - for typ in seg_sizes: - if typ != "unk": - tmp_size = seg_sizes[typ] - tmp_ratio = tmp_size / total_size - log.write( - f"{typ:>20}: {fmt_size(tmp_size):>8} ({tmp_ratio:.2%}) {Fore.GREEN}{seg_split[typ]} split{Style.RESET_ALL}, {Style.DIM}{seg_cached[typ]} cached" - ) - log.write( - f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files" - ) - - -def merge_configs(main_config, additional_config): - # Merge rules are simple - # For each key in the dictionary - # - If list then append to list - # - If a dictionary then repeat merge on sub dictionary entries - # - Else assume string or number and replace entry - - for curkey in additional_config: - if curkey not in main_config: - main_config[curkey] = additional_config[curkey] - elif type(main_config[curkey]) != type(additional_config[curkey]): - log.error(f"Type for key {curkey} in configs does not match") - else: - # keys exist and match, see if a list to append - if type(main_config[curkey]) == list: - main_config[curkey] += additional_config[curkey] - elif type(main_config[curkey]) == dict: - # need to merge sub areas - main_config[curkey] = merge_configs( - main_config[curkey], additional_config[curkey] - ) - else: - # not a list or dictionary, must be a number or string, overwrite - main_config[curkey] = additional_config[curkey] - - return main_config - - -def brief_seg_name(seg: Segment, limit: int, ellipsis="…") -> str: - s = seg.name.strip() - if len(s) > limit: - return s[:limit].strip() + ellipsis - return s - - -def main( - config_path, - modes, - verbose, - use_cache=True, - skip_version_check=False, - stdout_only=False, - disassemble_all=False, -): - global config - - if stdout_only: - progress_bar.out_file = sys.stdout - - # Load config - config = {} - for entry in config_path: - with open(entry) as f: - additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader) - config = merge_configs(config, additional_config) - - options.initialize(config, config_path, modes, verbose, disassemble_all) - - disassembler_instance.create_disassembler_instance(options.opts.platform) - disassembler_instance.get_instance().check_version(skip_version_check, VERSION) - - with options.opts.target_path.open("rb") as f2: - rom_bytes = f2.read() - - if "sha1" in config: - sha1 = hashlib.sha1(rom_bytes).hexdigest() - e_sha1 = config["sha1"].lower() - if e_sha1 != sha1: - log.error(f"sha1 mismatch: expected {e_sha1}, was {sha1}") - - # Create main output dir - options.opts.base_path.mkdir(parents=True, exist_ok=True) - - processed_segments: List[Segment] = [] - - seg_sizes: Dict[str, int] = {} - seg_split: Dict[str, int] = {} - seg_cached: Dict[str, int] = {} - - # Load cache - if use_cache: - try: - with options.opts.cache_path.open("rb") as f3: - cache = pickle.load(f3) - - if verbose: - log.write(f"Loaded cache ({len(cache.keys())} items)") - except Exception: - cache = {} - else: - cache = {} - - # invalidate entire cache if options change - if use_cache and cache.get("__options__") != config.get("options"): - if verbose: - log.write("Options changed, invalidating cache") - - cache = { - "__options__": config.get("options"), - } - - disassembler_instance.get_instance().configure(options.opts) - - platform_module = importlib.import_module(f"platforms.{options.opts.platform}") - platform_init = getattr(platform_module, "init") - platform_init(rom_bytes) - - # Initialize segments - all_segments = initialize_segments(config["segments"]) - - # Load and process symbols - symbols.initialize(all_segments) - relocs.initialize() - - # Assign symbols to segments - assign_symbols_to_segments() - - if options.opts.is_mode_active("code"): - symbols.initialize_spim_context(all_segments) - relocs.initialize_spim_context() - - # Resolve raster/palette siblings - if options.opts.is_mode_active("img"): - palettes.initialize(all_segments) - - # Scan - scan_bar = progress_bar.get_progress_bar(all_segments) - for segment in scan_bar: - assert isinstance(segment, Segment) - scan_bar.set_description(f"Scanning {brief_seg_name(segment, 20)}") - typ = segment.type - if segment.type == "bin" and segment.is_name_default(): - typ = "unk" - - if typ not in seg_sizes: - seg_sizes[typ] = 0 - seg_split[typ] = 0 - seg_cached[typ] = 0 - seg_sizes[typ] += 0 if segment.size is None else segment.size - - if segment.should_scan(): - # Check cache but don't write anything - if use_cache: - if segment.cache() == cache.get(segment.unique_id()): - continue - - segment.did_run = True - segment.scan(rom_bytes) - - processed_segments.append(segment) - - seg_split[typ] += 1 - - symbols.mark_c_funcs_as_defined() - - # Split - split_bar = progress_bar.get_progress_bar(all_segments) - for segment in split_bar: - split_bar.set_description(f"Splitting {brief_seg_name(segment, 20)}") - - if use_cache: - cached = segment.cache() - - if cached == cache.get(segment.unique_id()): - # Cache hit - if segment.type not in seg_cached: - seg_cached[segment.type] = 0 - seg_cached[segment.type] += 1 - continue - else: - # Cache miss; split - cache[segment.unique_id()] = cached - - if segment.should_split(): - segment_bytes = rom_bytes - if segment.file_path: - with open(segment.file_path, "rb") as segment_input_file: - segment_bytes = segment_input_file.read() - segment.split(segment_bytes) - - if ( - options.opts.is_mode_active("ld") and options.opts.platform != "gc" - ): # TODO move this to platform initialization when it gets implemented - # Calculate list of segments for which we need to find the largest, so we can safely place the symbol after it - max_vram_end_syms: Dict[str, List[Segment]] = {} - for sym in symbols.appears_after_overlays_syms: - max_vram_end_syms[sym.name] = [ - seg - for seg in all_segments - if isinstance(seg.vram_start, int) - and seg.vram_start == sym.appears_after_overlays_addr - ] - max_vram_end_sym_names: Set[str] = set(max_vram_end_syms.keys()) - - max_vram_end_insertion_points: Dict[ - Segment, List[Tuple[str, List[Segment]]] - ] = {} - # Find the last segment whose vram_of_symbol is one of the max_vram_end_syms - for segment in reversed(all_segments): - vram_of_sym = segment.vram_of_symbol - if vram_of_sym is not None and vram_of_sym in max_vram_end_sym_names: - if segment not in max_vram_end_insertion_points: - max_vram_end_insertion_points[segment] = [] - max_vram_end_insertion_points[segment].append( - (vram_of_sym, max_vram_end_syms[vram_of_sym]) - ) - max_vram_end_sym_names.remove(vram_of_sym) - - global linker_writer - linker_writer = LinkerWriter() - linker_bar = progress_bar.get_progress_bar(all_segments) - - for segment in linker_bar: - linker_bar.set_description(f"Linker script {brief_seg_name(segment, 20)}") - linker_writer.add(segment, max_vram_end_insertion_points.get(segment, [])) - linker_writer.save_linker_script() - linker_writer.save_symbol_header() - - # write elf_sections.txt - this only lists the generated sections in the elf, not subsections - # that the elf combines into one section - if options.opts.elf_section_list_path: - section_list = "" - for segment in all_segments: - section_list += "." + to_cname(segment.name) + "\n" - with open(options.opts.elf_section_list_path, "w", newline="\n") as f: - f.write(section_list) - - # Write undefined_funcs_auto.txt - if options.opts.create_undefined_funcs_auto: - to_write = [ - s - for s in symbols.all_symbols - if s.referenced and not s.defined and not s.dead and s.type == "func" - ] - to_write.sort(key=lambda x: x.vram_start) - - with open(options.opts.undefined_funcs_auto_path, "w", newline="\n") as f: - for symbol in to_write: - f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n") - - # write undefined_syms_auto.txt - if options.opts.create_undefined_syms_auto: - to_write = [ - s - for s in symbols.all_symbols - if s.referenced - and not s.defined - and not s.dead - and s.type not in {"func", "label", "jtbl_label"} - ] - to_write.sort(key=lambda x: x.vram_start) - - with open(options.opts.undefined_syms_auto_path, "w", newline="\n") as f: - for symbol in to_write: - f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n") - - # print warnings during split - for segment in all_segments: - if len(segment.warnings) > 0: - log.write( - f"{Style.DIM}0x{segment.rom_start:06X}{Style.RESET_ALL} {segment.type} {Style.BRIGHT}{segment.name}{Style.RESET_ALL}:" - ) - - for warn in segment.warnings: - log.write("warning: " + warn, status="warn") - - log.write("") # empty line - - # Statistics - do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached) - - # Save cache - if cache != {} and use_cache: - if verbose: - log.write("Writing cache") - with open(options.opts.cache_path, "wb") as f4: - pickle.dump(cache, f4) - - if options.opts.dump_symbols and options.opts.is_mode_active("code"): - from pathlib import Path - - splat_hidden_folder = Path(".splat/") - splat_hidden_folder.mkdir(exist_ok=True) - - with open(splat_hidden_folder / "splat_symbols.csv", "w") as f: - f.write( - "vram_start,given_name,name,type,given_size,size,rom,defined,user_declared,referenced,dead,extract\n" - ) - for s in sorted(symbols.all_symbols, key=lambda x: x.vram_start): - f.write(f"{s.vram_start:X},{s.given_name},{s.name},{s.type},") - if s.given_size is not None: - f.write(f"0x{s.given_size:X},") - else: - f.write("None,") - f.write(f"{s.size},") - if s.rom is not None: - f.write(f"0x{s.rom:X},") - else: - f.write("None,") - f.write( - f"{s.defined},{s.user_declared},{s.referenced},{s.dead},{s.extract}\n" - ) - - symbols.spim_context.saveContextToFile(splat_hidden_folder / "spim_context.csv") - - -if __name__ == "__main__": - args = parser.parse_args() - main( - args.config, - args.modes, - args.verbose, - args.use_cache, - args.skip_version_check, - args.stdout_only, - args.disassemble_all, - ) diff --git a/tools/splat/stubs/colorama.pyi b/tools/splat/stubs/colorama.pyi deleted file mode 100644 index 5d13e50..0000000 --- a/tools/splat/stubs/colorama.pyi +++ /dev/null @@ -1,8 +0,0 @@ -from typing import Any - -def init(**kwargs): ... - -Fore: Any -Back: Any -Style: Any -Cursor: Any diff --git a/tools/splat/test.py b/tools/splat/test.py deleted file mode 100644 index 3685541..0000000 --- a/tools/splat/test.py +++ /dev/null @@ -1,515 +0,0 @@ -from spimdisasm.common import FileSectionType - -from split import * -import unittest -import io -import filecmp -from util import symbols, options -import spimdisasm -from segtypes.common.rodata import CommonSegRodata -from segtypes.common.code import CommonSegCode -from segtypes.common.c import CommonSegC -from segtypes.common.bss import CommonSegBss -import difflib - - -class Testing(unittest.TestCase): - def compare_files(self, test_path, ref_path): - with io.open(test_path) as test_f, io.open(ref_path) as ref_f: - self.assertListEqual(list(test_f), list(ref_f)) - - def get_same_files(self, dcmp, out): - for name in dcmp.same_files: - out.append((name, dcmp.left, dcmp.right)) - - for sub_dcmp in dcmp.subdirs.values(): - self.get_same_files(sub_dcmp, out) - - def get_diff_files(self, dcmp, out): - for name in dcmp.diff_files: - out.append((name, dcmp.left, dcmp.right)) - - for sub_dcmp in dcmp.subdirs.values(): - self.get_diff_files(sub_dcmp, out) - - def get_left_only_files(self, dcmp, out): - for name in dcmp.left_only: - out.append((name, dcmp.left, dcmp.right)) - - for sub_dcmp in dcmp.subdirs.values(): - self.get_left_only_files(sub_dcmp, out) - - def get_right_only_files(self, dcmp, out): - for name in dcmp.right_only: - out.append((name, dcmp.left, dcmp.right)) - - for sub_dcmp in dcmp.subdirs.values(): - self.get_right_only_files(sub_dcmp, out) - - def test_basic_app(self): - main(["test/basic_app/splat.yaml"], None, None) - - comparison = filecmp.dircmp("test/basic_app/split", "test/basic_app/expected") - - diff_files: List[Tuple[str, str, str]] = [] - self.get_diff_files(comparison, diff_files) - - same_files: List[Tuple[str, str, str]] = [] - self.get_same_files(comparison, same_files) - - left_only_files: List[Tuple[str, str, str]] = [] - self.get_left_only_files(comparison, left_only_files) - - right_only_files: List[Tuple[str, str, str]] = [] - self.get_right_only_files(comparison, right_only_files) - - print("same_files", same_files) - print("diff_files", diff_files) - print("left_only_files", left_only_files) - print("right_only_files", right_only_files) - - # if the files are different print out the difference - for file in diff_files: - # can't diff binary - if file[0] == ".splache": - continue - with open(f"{file[1]}/{file[0]}") as file1: - file1_lines = file1.readlines() - with open(f"{file[2]}/{file[0]}") as file2: - file2_lines = file2.readlines() - - for line in difflib.unified_diff( - file1_lines, file2_lines, fromfile="file1", tofile="file2", lineterm="" - ): - print(line) - - assert len(diff_files) == 0 - assert len(left_only_files) == 0 - assert len(right_only_files) == 0 - - -def test_init(): - options_dict = { - "options": { - "platform": "n64", - "basename": "basic_app", - "base_path": ".", - "build_path": "build", - "target_path": "build/main.bin", - "asm_path": "split/asm", - "src_path": "split/src", - "ld_script_path": "split/basic_app.ld", - "cache_path": "split/.splache", - "symbol_addrs_path": "split/generated.symbols.txt", - "undefined_funcs_auto_path": "split/undefined_funcs_auto.txt", - "undefined_syms_auto_path": "split/undefined_syms_auto.txt", - }, - "segments": [ - { - "name": "basic_app", - "type": "code", - "start": 0, - "vram": 0x400000, - "subalign": 4, - "subsegments": [[0, "data"], [0x1DC, "c", "main"], [0x1FC, "data"]], - }, - [0x1290], - ], - } - options.initialize(options_dict, ["./test/basic_app/splat.yaml"], [], False) - - -class Symbols(unittest.TestCase): - def test_check_valid_type(self): - disassembler_instance.create_disassembler_instance("n64") - - # first char is uppercase - assert symbols.check_valid_type("Symbol") - - splat_sym_types = {"func", "jtbl", "jtbl_label", "label"} - - for type in splat_sym_types: - assert symbols.check_valid_type(type) - - spim_types = [ - "char*", - "u32", - "Vec3f", - "u8", - "char", - "u16", - "f32", - "u64", - "asciz", - "s8", - "s64", - "f64", - "s16", - "s32", - ] - - for type in spim_types: - assert symbols.check_valid_type(type) - - def test_add_symbol_to_spim_segment(self): - segment = spimdisasm.common.SymbolsSegment( - context=spimdisasm.common.Context(), - vromStart=0x0, - vromEnd=0x10, - vramStart=0x40000000 + 0x0, - vramEnd=0x40000000 + 0x10, - ) - sym = symbols.Symbol(0x40000000) - sym.user_declared = False - sym.defined = True - sym.rom = 0x0 - sym.type = "func" - result = symbols.add_symbol_to_spim_segment(segment, sym) - assert result.type == spimdisasm.common.SymbolSpecialType.function - assert sym.user_declared == result.isUserDeclared - assert sym.defined == result.isDefined - - def test_add_symbol_to_spim_section(self): - section = spimdisasm.mips.sections.SectionBase( - context=spimdisasm.common.Context(), - vromStart=0x0, - vromEnd=0x10, - vram=0x40000000, - filename="test", - words=[], - sectionType=FileSectionType.Text, - segmentVromStart=0x0, - overlayCategory=None, - ) - sym = symbols.Symbol(0x100) - sym.type = "func" - sym.user_declared = False - sym.defined = True - result = symbols.add_symbol_to_spim_section(section, sym) - assert result.type == spimdisasm.common.SymbolSpecialType.function - assert sym.user_declared == result.isUserDeclared - assert sym.defined == result.isDefined - - def test_create_symbol_from_spim_symbol(self): - # need to init otherwise options.opts isn't defined. - # used in initializing a Segment - test_init() - - segment = Segment( - rom_start=0x0, - rom_end=0x100, - type="func", - name="MyFunc", - vram_start=0x40000000, - args=[], - yaml=None, - ) - context_sym = spimdisasm.common.ContextSymbol(address=0) - result = symbols.create_symbol_from_spim_symbol(segment, context_sym) - assert result.referenced - assert result.extract - assert result.name == "D_0" - - -def get_yaml(): - return { - "name": "basic_app", - "type": "code", - "start": 0, - "vram": 0x400000, - "subalign": 4, - "subsegments": [[0, "data"], [0x1DC, "c", "main"], [0x1FC, "data"]], - } - - -class Rodata(unittest.TestCase): - def test_disassemble_data(self): - test_init() - common_seg_rodata = CommonSegRodata( - rom_start=0x0, - rom_end=0x100, - type=".rodata", - name="MyRodata", - vram_start=0x400, - args=None, - yaml=None, - ) - rom_data = [] - for i in range(0x100): - rom_data.append(i) - common_seg_rodata.disassemble_data(bytes(rom_data)) - assert common_seg_rodata.spim_section is not None - assert common_seg_rodata.spim_section.get_section().words[0] == 0x0010203 - assert symbols.get_all_symbols()[0].vram_start == 0x400 - assert symbols.get_all_symbols()[0].segment == common_seg_rodata - assert symbols.get_all_symbols()[0].linker_section == ".rodata" - - def test_get_possible_text_subsegment_for_symbol(self): - context = spimdisasm.common.Context() - - result_symbol_addr = 0x2DC - - # use SymbolRodata to test migration - rodata_sym = spimdisasm.mips.symbols.SymbolRodata( - context=context, - vromStart=0x100, - vromEnd=0x200, - inFileOffset=0, - vram=0x100, - words=[0, 1, 2, 3, 4, 5, 6, 7], - segmentVromStart=0, - overlayCategory=None, - ) - rodata_sym.contextSym.forceMigration = True - - context_sym = spimdisasm.common.ContextSymbol(address=0) - context_sym.address = result_symbol_addr - - rodata_sym.contextSym.referenceFunctions = {context_sym} - # Segment __init__ requires opts to be initialized - test_init() - - common_seg_rodata = CommonSegRodata( - rom_start=0x0, - rom_end=0x100, - type=".rodata", - name="MyRodata", - vram_start=0x400, - args=None, - yaml=None, - ) - - common_seg_rodata.parent = CommonSegCode( - rom_start=0x0, - rom_end=0x200, - type="code", - name="MyCode", - vram_start=0x100, - args=[], - yaml=get_yaml(), - ) - - result = common_seg_rodata.get_possible_text_subsegment_for_symbol(rodata_sym) - assert result is not None - assert type(result[0]) == CommonSegC - assert result[1].address == result_symbol_addr - - -class Bss(unittest.TestCase): - def test_disassemble_data(self): - # Segment __init__ requires opts to be initialized - test_init() - - bss = CommonSegBss( - rom_start=0x0, - rom_end=0x100, - type=".bss", - name=None, - vram_start=0x40000000, - args=None, - yaml=None, - ) - - bss.parent = CommonSegCode( - rom_start=0x0, - rom_end=0x200, - type="code", - name="MyCode", - vram_start=0x100, - args=[], - yaml=get_yaml(), - ) - - rom_bytes = bytes([0, 1, 2, 3, 4, 5, 6, 7]) - bss.disassemble_data(rom_bytes) - - assert bss.spim_section is not None - - assert isinstance( - bss.spim_section.get_section(), spimdisasm.mips.sections.SectionBss - ) - assert bss.spim_section.get_section().bssVramStart == 0x40000000 - assert bss.spim_section.get_section().bssVramEnd == 0x300 - - -class SymbolsInitialize(unittest.TestCase): - def test_attrs(self): - import pathlib - - symbols.reset_symbols() - test_init() - - sym_addrs_lines = [ - "func_1 = 0x100 // type:func size:10 rom:100 segment:test_segment name_end:the_name_end " - "appears_after_overlays_addr:1234" - ] - - all_segments = [ - Segment( - rom_start=0x100, - rom_end=0x200, - type="func", - name="test_segment", - vram_start=0x300, - args=[], - yaml={}, - ) - ] - - symbols.handle_sym_addrs( - pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments - ) - assert symbols.all_symbols[0].given_name == "func_1" - assert symbols.all_symbols[0].type == "func" - assert symbols.all_symbols[0].given_size == 10 - assert symbols.all_symbols[0].rom == 100 - assert symbols.all_symbols[0].segment == all_segments[0] - assert symbols.all_symbols[0].given_name_end == "the_name_end" - assert symbols.appears_after_overlays_syms[0] == symbols.all_symbols[0] - - def test_boolean_attrs(self): - import pathlib - - symbols.reset_symbols() - test_init() - - sym_addrs_lines = [ - "func_1 = 0x100 // dead:True defined:True extract:True force_migration:True force_not_migration:True " - "allow_addend:True dont_allow_addend:True" - ] - - all_segments = [ - Segment( - rom_start=0x100, - rom_end=0x200, - type="func", - name="test_segment", - vram_start=0x300, - args=[], - yaml={}, - ) - ] - - symbols.handle_sym_addrs( - pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments - ) - assert symbols.all_symbols[0].dead == True - assert symbols.all_symbols[0].defined == True - assert symbols.all_symbols[0].force_migration == True - assert symbols.all_symbols[0].force_not_migration == True - assert symbols.all_symbols[0].allow_addend == True - assert symbols.all_symbols[0].dont_allow_addend == True - - # test spim ban range - def test_ignore(self): - import pathlib - - symbols.reset_symbols() - test_init() - - sym_addrs_lines = ["func_1 = 0x100 // ignore:True size:4"] - - all_segments = [ - Segment( - rom_start=0x100, - rom_end=0x200, - type="func", - name="test_segment", - vram_start=0x300, - args=[], - yaml={}, - ) - ] - - symbols.handle_sym_addrs( - pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments - ) - assert symbols.spim_context.bannedRangedSymbols[0].start == 16 - assert symbols.spim_context.bannedRangedSymbols[0].end == 20 - - -class InitializeSpimContext(unittest.TestCase): - def test_overlay(self): - symbols.reset_symbols() - test_init() - - yaml = { - "name": "boot", - "type": "code", - "start": 4096, - "vram": 2147484672, - "bss_size": 128, - "exclusive_ram_id": "overlay", - "subsegments": [ - [4096, "c", "main"], - [4336, "hasm", "handwritten"], - [4352, "data", "main"], - [4368, "rodata", "main"], - {"type": "bss", "vram": 2147484992, "name": "main"}, - ], - } - - all_segments: List["Segment"] = [ - CommonSegCode( - rom_start=0x0, - rom_end=0x200, - type="code", - name="main", - vram_start=0x100, - args=[], - yaml=yaml, - ) - ] - - # force this since it's hard to set up - all_segments[0].exclusive_ram_id = "overlay" - - symbols.initialize_spim_context(all_segments) - # spim should have added something to overlaySegments - assert ( - type(symbols.spim_context.overlaySegments["overlay"][0]) - == spimdisasm.common.SymbolsSegment - ) - - # test globalSegment settings - def test_global(self): - symbols.reset_symbols() - test_init() - - yaml = { - "name": "boot", - "type": "code", - "start": 4096, - "vram": 2147484672, - "bss_size": 128, - "exclusive_ram_id": "overlay", - "subsegments": [ - [4096, "c", "main"], - [4336, "hasm", "handwritten"], - [4352, "data", "main"], - [4368, "rodata", "main"], - {"type": "bss", "vram": 2147484992, "name": "main"}, - ], - } - - all_segments: List["Segment"] = [ - CommonSegCode( - rom_start=0x0, - rom_end=0x200, - type="code", - name="main", - vram_start=0x100, - args=[], - yaml=yaml, - ) - ] - - assert symbols.spim_context.globalSegment.vramStart == 2147483648 - assert symbols.spim_context.globalSegment.vramEnd == 2147487744 - symbols.initialize_spim_context(all_segments) - assert symbols.spim_context.globalSegment.vramStart == 256 - assert symbols.spim_context.globalSegment.vramEnd == 896 - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/splat/test/basic_app/basic_app.ld b/tools/splat/test/basic_app/basic_app.ld deleted file mode 100644 index 7a94e16..0000000 --- a/tools/splat/test/basic_app/basic_app.ld +++ /dev/null @@ -1,78 +0,0 @@ -SECTIONS -{ - __romPos = 0; - _gp = 0x0; - header_ROM_START = __romPos; - header_VRAM = ADDR(.header); - .header : AT(header_ROM_START) SUBALIGN(16) - { - header_DATA_START = .; - build/header.o(.data); - header_DATA_END = .; - header_DATA_SIZE = ABSOLUTE(header_DATA_END - header_DATA_START); - } - __romPos += SIZEOF(.header); - header_ROM_END = __romPos; - header_VRAM_END = .; - - dummy_ipl3_ROM_START = __romPos; - dummy_ipl3_VRAM = ADDR(.dummy_ipl3); - .dummy_ipl3 0xA4000040 : AT(dummy_ipl3_ROM_START) SUBALIGN(16) - { - dummy_ipl3_DATA_START = .; - build/dummy_ipl3.o(.data); - dummy_ipl3_DATA_END = .; - dummy_ipl3_DATA_SIZE = ABSOLUTE(dummy_ipl3_DATA_END - dummy_ipl3_DATA_START); - } - __romPos += SIZEOF(.dummy_ipl3); - __romPos = ALIGN(__romPos, 16); - dummy_ipl3_ROM_END = __romPos; - dummy_ipl3_VRAM_END = .; - - boot_ROM_START = __romPos; - boot_VRAM = ADDR(.boot); - .boot 0x80000400 : AT(boot_ROM_START) SUBALIGN(16) - { - boot_TEXT_START = .; - build/main.o(.text); - build/handwritten.o(.text); - boot_TEXT_END = .; - boot_TEXT_SIZE = ABSOLUTE(boot_TEXT_END - boot_TEXT_START); - - boot_DATA_START = .; - build/main.o(.data); - build/handwritten.o(.data); - boot_DATA_END = .; - boot_DATA_SIZE = ABSOLUTE(boot_DATA_END - boot_DATA_START); - - boot_RODATA_START = .; - build/main.o(.rodata); - build/handwritten.o(.rodata); - boot_RODATA_END = .; - boot_RODATA_SIZE = ABSOLUTE(boot_RODATA_END - boot_RODATA_START); - } - - boot_bss_VRAM = ADDR(.boot_bss); - .boot_bss (NOLOAD) : SUBALIGN(16) - { - boot_BSS_START = .; - build/main.o(.bss); - build/main.o(.scommon); - build/main.o(COMMON); - build/handwritten.o(.bss); - build/handwritten.o(.scommon); - build/handwritten.o(COMMON); - boot_BSS_END = .; - boot_BSS_SIZE = ABSOLUTE(boot_BSS_END - boot_BSS_START); - } - - __romPos += SIZEOF(.boot); - __romPos = ALIGN(__romPos, 16); - boot_ROM_END = __romPos; - boot_VRAM_END = .; - - /DISCARD/ : - { - *(*); - } -} diff --git a/tools/splat/test/basic_app/build.sh b/tools/splat/test/basic_app/build.sh deleted file mode 100755 index 0e11e36..0000000 --- a/tools/splat/test/basic_app/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -mkdir -p build -echo "Building..." -export PATH=/gcc-2.7.2:$PATH -COMPILER_PATH=/gcc-2.7.2 /gcc-2.7.2/gcc -v -G0 -mgp32 -mfp32 -mips3 main.c -c -o build/main.o -COMPILER_PATH=/gcc-2.7.2 /gcc-2.7.2/gcc -G0 -mgp32 -mfp32 -mips3 header.s -c -o build/header.o -COMPILER_PATH=/gcc-2.7.2 /gcc-2.7.2/gcc -G0 -mgp32 -mfp32 -mips3 handwritten.s -c -o build/handwritten.o -COMPILER_PATH=/gcc-2.7.2 /gcc-2.7.2/gcc -G0 -mgp32 -mfp32 -mips3 dummy_ipl3.s -c -o build/dummy_ipl3.o -mips-linux-gnu-strip -N dummy_symbol_name build/*.o -echo "Linking..." -mips-linux-gnu-ld -Map build/basic_app.map -T basic_app.ld -o build/basic_app.elf -echo "Dumping bin..." -mips-linux-gnu-objcopy build/basic_app.elf -O binary build/basic_app.bin --pad-to=0x1140 --gap-fill=0xFF diff --git a/tools/splat/test/basic_app/clean.sh b/tools/splat/test/basic_app/clean.sh deleted file mode 100644 index 2fcd367..0000000 --- a/tools/splat/test/basic_app/clean.sh +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf ./build -rm -rf ./split diff --git a/tools/splat/test/basic_app/dummy_ipl3.s b/tools/splat/test/basic_app/dummy_ipl3.s deleted file mode 100644 index fc223a1..0000000 --- a/tools/splat/test/basic_app/dummy_ipl3.s +++ /dev/null @@ -1,3 +0,0 @@ -.section .data -dummy_ipl3: -.zero 0x1000 - 0x40 diff --git a/tools/splat/test/basic_app/expected/.splache b/tools/splat/test/basic_app/expected/.splache Binary files differdeleted file mode 100644 index c855f15..0000000 --- a/tools/splat/test/basic_app/expected/.splache +++ /dev/null diff --git a/tools/splat/test/basic_app/expected/asm/data/main.bss.s b/tools/splat/test/basic_app/expected/asm/data/main.bss.s deleted file mode 100644 index 96a20c2..0000000 --- a/tools/splat/test/basic_app/expected/asm/data/main.bss.s +++ /dev/null @@ -1,6 +0,0 @@ -.include "macro.inc" - -.section .bss - -glabel D_80000540 -/* 1110 80000540 */ .space 0x78 diff --git a/tools/splat/test/basic_app/expected/asm/data/main.data.s b/tools/splat/test/basic_app/expected/asm/data/main.data.s deleted file mode 100644 index 1ed582a..0000000 --- a/tools/splat/test/basic_app/expected/asm/data/main.data.s +++ /dev/null @@ -1,13 +0,0 @@ -.include "macro.inc" - -.section .data - -glabel D_80000500 -/* 1100 80000500 */ .word 0x00000001 -.size D_80000500, . - D_80000500 - -glabel D_80000504 -/* 1104 80000504 */ .word 0x00000000 -/* 1108 80000508 */ .word 0x00000000 -/* 110C 8000050C */ .word 0x00000000 -.size D_80000504, . - D_80000504 diff --git a/tools/splat/test/basic_app/expected/asm/handwritten.s b/tools/splat/test/basic_app/expected/asm/handwritten.s deleted file mode 100644 index 539d92a..0000000 --- a/tools/splat/test/basic_app/expected/asm/handwritten.s +++ /dev/null @@ -1,16 +0,0 @@ -.include "macro.inc" - -/* assembler directives */ -.set noat /* allow manual use of $at */ -.set noreorder /* don't insert nops after branches */ -.set gp=64 /* allow use of 64-bit general purpose registers */ - -.section .text, "ax" - -# Handwritten function -glabel func_800004F0 -/* 10F0 800004F0 00851020 */ add $v0, $a0, $a1 # handwritten instruction -/* 10F4 800004F4 03E00008 */ jr $ra -/* 10F8 800004F8 00000000 */ nop -/* 10FC 800004FC 00000000 */ nop -.size func_800004F0, . - func_800004F0 diff --git a/tools/splat/test/basic_app/expected/asm/header.s b/tools/splat/test/basic_app/expected/asm/header.s deleted file mode 100644 index be176b7..0000000 --- a/tools/splat/test/basic_app/expected/asm/header.s +++ /dev/null @@ -1,16 +0,0 @@ -.section .data - -.word 0x80371240 /* PI BSB Domain 1 register */ -.word 0x0000000F /* Clockrate setting */ -.word 0x800004A0 /* Entrypoint address */ -.word 0x00001444 /* Revision */ -.word 0x00000000 /* Checksum 1 */ -.word 0x00000000 /* Checksum 2 */ -.word 0x00000000 /* Unknown 1 */ -.word 0x00000000 /* Unknown 2 */ -.ascii "SPLAT TEST ROM " /* Internal name */ -.word 0x00000000 /* Unknown 3 */ -.word 0x0000004E /* Cartridge */ -.ascii "SP" /* Cartridge ID */ -.ascii "E" /* Country code */ -.byte 0x00 /* Version */ diff --git a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s b/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s deleted file mode 100644 index 21d0010..0000000 --- a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s +++ /dev/null @@ -1,55 +0,0 @@ -.set noat /* allow manual use of $at */ -.set noreorder /* don't insert nops after branches */ - -glabel func_80000400 -/* 1000 80000400 27BDFFF8 */ addiu $sp, $sp, -0x8 -/* 1004 80000404 AFBE0000 */ sw $fp, 0x0($sp) -/* 1008 80000408 03A0F021 */ addu $fp, $sp, $zero -/* 100C 8000040C 3C028000 */ lui $v0, %hi(D_80000504) -/* 1010 80000410 8C420504 */ lw $v0, %lo(D_80000504)($v0) -/* 1014 80000414 2C430008 */ sltiu $v1, $v0, 0x8 -/* 1018 80000418 1060001B */ beqz $v1, .L80000488 -/* 101C 8000041C 00000000 */ nop -/* 1020 80000420 3C028000 */ lui $v0, %hi(D_80000504) -/* 1024 80000424 8C420504 */ lw $v0, %lo(D_80000504)($v0) -/* 1028 80000428 00401821 */ addu $v1, $v0, $zero -/* 102C 8000042C 00031080 */ sll $v0, $v1, 2 -/* 1030 80000430 3C038000 */ lui $v1, %hi(jtbl_80000518) -/* 1034 80000434 24630518 */ addiu $v1, $v1, %lo(jtbl_80000518) -/* 1038 80000438 00431021 */ addu $v0, $v0, $v1 -/* 103C 8000043C 8C430000 */ lw $v1, 0x0($v0) -/* 1040 80000440 00600008 */ jr $v1 -/* 1044 80000444 00000000 */ nop -glabel .L80000448 -/* 1048 80000448 08000124 */ j .L80000490 -/* 104C 8000044C 24020007 */ addiu $v0, $zero, 0x7 -glabel .L80000450 -/* 1050 80000450 08000124 */ j .L80000490 -/* 1054 80000454 24020006 */ addiu $v0, $zero, 0x6 -glabel .L80000458 -/* 1058 80000458 08000124 */ j .L80000490 -/* 105C 8000045C 24020005 */ addiu $v0, $zero, 0x5 -glabel .L80000460 -/* 1060 80000460 08000124 */ j .L80000490 -/* 1064 80000464 24020004 */ addiu $v0, $zero, 0x4 -glabel .L80000468 -/* 1068 80000468 08000124 */ j .L80000490 -/* 106C 8000046C 24020003 */ addiu $v0, $zero, 0x3 -glabel .L80000470 -/* 1070 80000470 08000124 */ j .L80000490 -/* 1074 80000474 24020002 */ addiu $v0, $zero, 0x2 -glabel .L80000478 -/* 1078 80000478 08000124 */ j .L80000490 -/* 107C 8000047C 24020001 */ addiu $v0, $zero, 0x1 -glabel .L80000480 -/* 1080 80000480 08000124 */ j .L80000490 -/* 1084 80000484 00001021 */ addu $v0, $zero, $zero -.L80000488: -/* 1088 80000488 08000124 */ j .L80000490 -/* 108C 8000048C 00001021 */ addu $v0, $zero, $zero -.L80000490: -/* 1090 80000490 03C0E821 */ addu $sp, $fp, $zero -/* 1094 80000494 8FBE0000 */ lw $fp, 0x0($sp) -/* 1098 80000498 03E00008 */ jr $ra -/* 109C 8000049C 27BD0008 */ addiu $sp, $sp, 0x8 -.size func_80000400, . - func_80000400 diff --git a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s b/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s deleted file mode 100644 index 112eb68..0000000 --- a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s +++ /dev/null @@ -1,26 +0,0 @@ -.set noat /* allow manual use of $at */ -.set noreorder /* don't insert nops after branches */ - -glabel func_800004A0 -/* 10A0 800004A0 27BDFFE8 */ addiu $sp, $sp, -0x18 -/* 10A4 800004A4 AFBF0014 */ sw $ra, 0x14($sp) -/* 10A8 800004A8 AFBE0010 */ sw $fp, 0x10($sp) -/* 10AC 800004AC 0C000100 */ jal func_80000400 -/* 10B0 800004B0 03A0F021 */ addu $fp, $sp, $zero -.L800004B4: -/* 10B4 800004B4 3C028000 */ lui $v0, %hi(D_80000500) -/* 10B8 800004B8 8C420500 */ lw $v0, %lo(D_80000500)($v0) -/* 10BC 800004BC 24430001 */ addiu $v1, $v0, 0x1 -/* 10C0 800004C0 3C018000 */ lui $at, %hi(D_80000500) -/* 10C4 800004C4 AC230500 */ sw $v1, %lo(D_80000500)($at) -/* 10C8 800004C8 3C028000 */ lui $v0, %hi(D_80000500) -/* 10CC 800004CC 8C420500 */ lw $v0, %lo(D_80000500)($v0) -/* 10D0 800004D0 0800012D */ j .L800004B4 -/* 10D4 800004D4 00000000 */ nop -/* 10D8 800004D8 03C0E821 */ addu $sp, $fp, $zero -/* 10DC 800004DC 8FBF0014 */ lw $ra, 0x14($sp) -/* 10E0 800004E0 8FBE0010 */ lw $fp, 0x10($sp) -/* 10E4 800004E4 03E00008 */ jr $ra -/* 10E8 800004E8 27BD0018 */ addiu $sp, $sp, 0x18 -/* 10EC 800004EC 00000000 */ nop -.size func_800004A0, . - func_800004A0 diff --git a/tools/splat/test/basic_app/expected/assets/dummy_ipl3.bin b/tools/splat/test/basic_app/expected/assets/dummy_ipl3.bin Binary files differdeleted file mode 100644 index 26460b7..0000000 --- a/tools/splat/test/basic_app/expected/assets/dummy_ipl3.bin +++ /dev/null diff --git a/tools/splat/test/basic_app/expected/basic_app.ld b/tools/splat/test/basic_app/expected/basic_app.ld deleted file mode 100644 index 0779b7e..0000000 --- a/tools/splat/test/basic_app/expected/basic_app.ld +++ /dev/null @@ -1,70 +0,0 @@ -SECTIONS -{ - __romPos = 0; - _gp = 0x0; - header_ROM_START = __romPos; - header_VRAM = ADDR(.header); - .header : AT(header_ROM_START) SUBALIGN(16) - { - header_DATA_START = .; - header_s = .; - build/split/asm/header.s.o(.data); - header_DATA_END = .; - header_DATA_SIZE = ABSOLUTE(header_DATA_END - header_DATA_START); - } - __romPos += SIZEOF(.header); - header_ROM_END = __romPos; - header_VRAM_END = .; - - dummy_ipl3_ROM_START = __romPos; - dummy_ipl3_VRAM = ADDR(.dummy_ipl3); - .dummy_ipl3 0xA4000040 : AT(dummy_ipl3_ROM_START) SUBALIGN(16) - { - dummy_ipl3_DATA_START = .; - dummy_ipl3_bin = .; - build/split/assets/dummy_ipl3.bin.o(.data); - dummy_ipl3_DATA_END = .; - dummy_ipl3_DATA_SIZE = ABSOLUTE(dummy_ipl3_DATA_END - dummy_ipl3_DATA_START); - } - __romPos += SIZEOF(.dummy_ipl3); - __romPos = ALIGN(__romPos, 16); - dummy_ipl3_ROM_END = __romPos; - dummy_ipl3_VRAM_END = .; - - boot_ROM_START = __romPos; - boot_VRAM = ADDR(.boot); - .boot 0x80000400 : AT(boot_ROM_START) SUBALIGN(16) - { - boot_TEXT_START = .; - build/split/src/main.c.o(.text); - build/split/asm/handwritten.s.o(.text); - boot_TEXT_END = .; - boot_TEXT_SIZE = ABSOLUTE(boot_TEXT_END - boot_TEXT_START); - boot_DATA_START = .; - main_data__s = .; - build/split/asm/data/main.data.s.o(.data); - boot_DATA_END = .; - boot_DATA_SIZE = ABSOLUTE(boot_DATA_END - boot_DATA_START); - boot_RODATA_START = .; - build/split/asm/data/main.rodata.s.o(.rodata); - boot_RODATA_END = .; - boot_RODATA_SIZE = ABSOLUTE(boot_RODATA_END - boot_RODATA_START); - } - boot_bss_VRAM = ADDR(.boot_bss); - .boot_bss (NOLOAD) : SUBALIGN(16) - { - boot_BSS_START = .; - build/split/asm/data/main.bss.s.o(.bss); - boot_BSS_END = .; - boot_BSS_SIZE = ABSOLUTE(boot_BSS_END - boot_BSS_START); - } - __romPos += SIZEOF(.boot); - __romPos = ALIGN(__romPos, 16); - boot_ROM_END = __romPos; - boot_VRAM_END = .; - - /DISCARD/ : - { - *(*); - } -} diff --git a/tools/splat/test/basic_app/expected/src/main.c b/tools/splat/test/basic_app/expected/src/main.c deleted file mode 100644 index 41b2e70..0000000 --- a/tools/splat/test/basic_app/expected/src/main.c +++ /dev/null @@ -1,5 +0,0 @@ -#include "common.h" - -INCLUDE_ASM(const s32, "main", func_80000400); - -INCLUDE_ASM(const s32, "main", func_800004A0); diff --git a/tools/splat/test/basic_app/expected/undefined_funcs_auto.txt b/tools/splat/test/basic_app/expected/undefined_funcs_auto.txt deleted file mode 100644 index e69de29..0000000 --- a/tools/splat/test/basic_app/expected/undefined_funcs_auto.txt +++ /dev/null diff --git a/tools/splat/test/basic_app/expected/undefined_syms_auto.txt b/tools/splat/test/basic_app/expected/undefined_syms_auto.txt deleted file mode 100644 index 8b96b0c..0000000 --- a/tools/splat/test/basic_app/expected/undefined_syms_auto.txt +++ /dev/null @@ -1 +0,0 @@ -jtbl_80000518 = 0x80000518; diff --git a/tools/splat/test/basic_app/handwritten.s b/tools/splat/test/basic_app/handwritten.s deleted file mode 100644 index 1a5f482..0000000 --- a/tools/splat/test/basic_app/handwritten.s +++ /dev/null @@ -1,12 +0,0 @@ -.section .text -.balign 16 -.set noreorder - -.global asm_func -.type asm_func, @function -.ent asm_func -asm_func: - add $v0, $a0, $a1 # Instruction that a compiler will never emit - jr $ra - nop -.end asm_func diff --git a/tools/splat/test/basic_app/header.s b/tools/splat/test/basic_app/header.s deleted file mode 100644 index 1e19f59..0000000 --- a/tools/splat/test/basic_app/header.s +++ /dev/null @@ -1,16 +0,0 @@ -.section .data - -.word 0x80371240 /* PI BSB Domain 1 register */ -.word 0x0000000F /* Clockrate setting */ -.word bootproc /* Entrypoint address */ -.word 0x00001444 /* Revision */ -.word 0x00000000 /* Checksum 1 */ -.word 0x00000000 /* Checksum 2 */ -.word 0x00000000 /* Unknown 1 */ -.word 0x00000000 /* Unknown 2 */ -.ascii "SPLAT TEST ROM " /* Internal name */ -.word 0x00000000 /* Unknown 3 */ -.word 0x0000004E /* Cartridge */ -.ascii "SP" /* Cartridge ID */ -.ascii "E" /* Country code */ -.byte 0x00 /* Version */ diff --git a/tools/splat/test/basic_app/main.c b/tools/splat/test/basic_app/main.c deleted file mode 100644 index cf119f7..0000000 --- a/tools/splat/test/basic_app/main.c +++ /dev/null @@ -1,45 +0,0 @@ -// .data -volatile int test = 1; - -// bin (.rodata) -const char bin_data[] = {0,1,2,3,4,5,6,7}; - -// .bss -unsigned long long bss_data[0x10]; - -// .data -volatile int switch_arg = 0; - -int do_switch() -{ - switch(switch_arg) - { - case 0: - return 7; - case 1: - return 6; - case 2: - return 5; - case 3: - return 4; - case 4: - return 3; - case 5: - return 2; - case 6: - return 1; - case 7: - return 0; - } - return 0; -} - -// c -void bootproc() -{ - do_switch(); - for (;;) - { - test++; - } -} diff --git a/tools/splat/test/basic_app/splat.yaml b/tools/splat/test/basic_app/splat.yaml deleted file mode 100644 index c1d2cf9..0000000 --- a/tools/splat/test/basic_app/splat.yaml +++ /dev/null @@ -1,38 +0,0 @@ -options: - platform: n64 - basename: basic_app - base_path: . - build_path: build - target_path: build/basic_app.bin - asm_path: split/asm - src_path: split/src - ld_script_path: split/basic_app.ld - cache_path: split/.splache - symbol_addrs_path: split/generated.symbols.txt - undefined_funcs_auto_path: split/undefined_funcs_auto.txt - undefined_syms_auto_path: split/undefined_syms_auto.txt - asset_path: split/assets - asm_generated_by: False - compiler: GCC -segments: - - name: header - type: header - start: 0x00 - - name: dummy_ipl3 - type: code - start: 0x40 - vram: 0xA4000040 - subsegments: - - [0x0040, bin, dummy_ipl3] - - name: boot - type: code - start: 0x1000 - vram: 0x80000400 - bss_size: 0x80 - subsegments: - - [0x1000, c, main] - - [0x10F0, hasm, handwritten] - - [0x1100, data, main] - - [0x1110, rodata, main] - - { type: bss, vram: 0x80000540, name: main } - - [0x1138] diff --git a/tools/splat/test_gen_expected.sh b/tools/splat/test_gen_expected.sh deleted file mode 100644 index 1b4a629..0000000 --- a/tools/splat/test_gen_expected.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker run --rm -it -v $(pwd):/splat -w /splat/test/basic_app splat-build sh build.sh && \ -rm -rf $(pwd)/test/basic_app/expected && \ -cp -r $(pwd)/test/basic_app/split $(pwd)/test/basic_app/expected diff --git a/tools/splat/util/__init__.py b/tools/splat/util/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/tools/splat/util/__init__.py +++ /dev/null diff --git a/tools/splat/util/color.py b/tools/splat/util/color.py deleted file mode 100644 index 6d75cbd..0000000 --- a/tools/splat/util/color.py +++ /dev/null @@ -1,19 +0,0 @@ -from math import ceil - -from util import options - - -# RRRRRGGG GGBBBBBA -def unpack_color(data): - s = int.from_bytes(data[0:2], byteorder=options.opts.endianness) - - r = (s >> 11) & 0x1F - g = (s >> 6) & 0x1F - b = (s >> 1) & 0x1F - a = (s & 1) * 0xFF - - r = ceil(0xFF * (r / 31)) - g = ceil(0xFF * (g / 31)) - b = ceil(0xFF * (b / 31)) - - return r, g, b, a diff --git a/tools/splat/util/compiler.py b/tools/splat/util/compiler.py deleted file mode 100644 index e1666dc..0000000 --- a/tools/splat/util/compiler.py +++ /dev/null @@ -1,43 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - - -@dataclass -class Compiler: - name: str - asm_function_macro: str = "glabel" - asm_jtbl_label_macro: str = "glabel" - asm_data_macro: str = "glabel" - asm_end_label: str = "" - c_newline: str = "\n" - asm_inc_header: str = "" - include_macro_inc: bool = True - asm_emit_size_directive: Optional[bool] = None - - -GCC = Compiler( - "GCC", - asm_inc_header=".set noat /* allow manual use of $at */\n.set noreorder /* don't insert nops after branches */\n\n", -) - -SN64 = Compiler( - "SN64", - asm_function_macro=".globl", - asm_jtbl_label_macro=".globl", - asm_data_macro=".globl", - asm_end_label=".end", - c_newline="\r\n", - include_macro_inc=False, - asm_emit_size_directive=False, -) - -IDO = Compiler("IDO", include_macro_inc=False, asm_emit_size_directive=False) - -compiler_for_name = {"GCC": GCC, "SN64": SN64, "IDO": IDO} - - -def for_name(name: str) -> Compiler: - name = name.upper() - if name in compiler_for_name: - return compiler_for_name[name] - return Compiler(name) diff --git a/tools/splat/util/floats.py b/tools/splat/util/floats.py deleted file mode 100644 index 009725e..0000000 --- a/tools/splat/util/floats.py +++ /dev/null @@ -1,63 +0,0 @@ -import math -import struct - - -# From mips_to_c: https://github.com/matt-kempster/mips_to_c/blob/d208400cca045113dada3e16c0d59c50cdac4529/src/translate.py#L2085 -def format_f32_imm(num: int) -> str: - packed = struct.pack(">I", num & (2**32 - 1)) - value = struct.unpack(">f", packed)[0] - - if not value or value == 4294967296.0: - # Zero, negative zero, nan, or INT_MAX. - return str(value) - - # Write values smaller than 1e-7 / greater than 1e7 using scientific notation, - # and values in between using fixed point. - if abs(math.log10(abs(value))) > 6.9: - fmt_char = "e" - elif abs(value) < 1: - fmt_char = "f" - else: - fmt_char = "g" - - def fmt(prec: int) -> str: - """Format 'value' with 'prec' significant digits/decimals, in either scientific - or regular notation depending on 'fmt_char'.""" - ret = ("{:." + str(prec) + fmt_char + "}").format(value) - if fmt_char == "e": - return ret.replace("e+", "e").replace("e0", "e").replace("e-0", "e-") - if "e" in ret: - # The "g" format character can sometimes introduce scientific notation if - # formatting with too few decimals. If this happens, return an incorrect - # value to prevent the result from being used. - # - # Since the value we are formatting is within (1e-7, 1e7) in absolute - # value, it will at least be possible to format with 7 decimals, which is - # less than float precision. Thus, this annoying Python limitation won't - # lead to us outputting numbers with more precision than we really have. - return "0" - return ret - - # 20 decimals is more than enough for a float. Start there, then try to shrink it. - prec = 20 - while prec > 0: - prec -= 1 - value2 = float(fmt(prec)) - if struct.pack(">f", value2) != packed: - prec += 1 - break - - if prec == 20: - # Uh oh, even the original value didn't format correctly. Fall back to str(), - # which ought to work. - return str(value) - - ret = fmt(prec) - if "." not in ret and "e" not in ret: - ret += ".0" - return ret - - -def format_f64_imm(num: int) -> str: - (value,) = struct.unpack(">d", struct.pack(">Q", num & (2**64 - 1))) - return str(value) diff --git a/tools/splat/util/gc/gcfst.py b/tools/splat/util/gc/gcfst.py deleted file mode 100644 index e50420f..0000000 --- a/tools/splat/util/gc/gcfst.py +++ /dev/null @@ -1,181 +0,0 @@ -import struct -from pathlib import Path -from typing import List, Optional - -from segtypes.gc.segment import GCSegment - -from util import options -from util.gc.gcutil import read_string_from_bytes - - -# Represents the info for either a directory or a file within a GameCube disc image's file system. -class GCFSTEntry: - def __init__(self, flags: bool, name_offset, offset, length): - self.flags = flags - self.name_offset = name_offset - self.offset = offset - self.length = length - - self.name = "" - self.parent: Optional[GCFSTEntry] = None - self.children: List[GCFSTEntry] = [] - - def populate_children_recursive( - self, root_dir: "GCFSTEntry", current_node_offset, fst_bytes, string_table_bytes - ): - self.parent = root_dir - self.name = read_string_from_bytes(self.name_offset, string_table_bytes) - - # This node is a file, so we don't do anything but return that we read 1 node. - if self.flags == False: - return 1 - - nodes_read = 1 - next_child_offset = current_node_offset + 0x0C - - # Directory nodes contain the index of the next node that is NOT its child, meaning the index of their next sibling node. - # We can figure out when we're done reading child nodes by comparing the offset of the next node to read to the - # offset of the next sibling node. We stop reading when the next node offset is >= the offset of the next sibling node. - while next_child_offset < self.length * 0x0C: - new_entry = GCFSTEntry( - bool(fst_bytes[next_child_offset + 0x0000]), - struct.unpack_from( - ">I", fst_bytes[next_child_offset : next_child_offset + 0x0004] - )[0] - & 0x00FFFFFF, - struct.unpack_from(">I", fst_bytes, next_child_offset + 0x0004)[0], - struct.unpack_from(">I", fst_bytes, next_child_offset + 0x0008)[0], - ) - - self.children.append(new_entry) - nodes_read += new_entry.populate_children_recursive( - self, next_child_offset, fst_bytes, string_table_bytes - ) - - next_child_offset = current_node_offset + nodes_read * 0x0C - - return nodes_read - - # Builds this entry's full path within the filesystem from its parents' names. - def get_full_name(self): - path_components: List[str] = [] - - entry = self - while entry.parent != None: - path_components.insert(0, entry.name) - - if entry.parent is None: - break - entry = entry.parent - - return Path(*path_components) - - # Emits this entry to the filesystem. - def emit(self, filesystem_dir: Path, iso_bytes): - full_path = filesystem_dir / self.get_full_name() - - # If this is a directory, we just need to make the directory on disk. - if self.flags == True: - full_path.mkdir(parents=True, exist_ok=True) - return - - file_bytes = iso_bytes[self.offset : self.offset + self.length] - with open(full_path, "wb") as f: - f.write(file_bytes) - - def emit_recursive(self, filesystem_dir: Path, iso_bytes): - # Don't emit if this is the root directory. - if self.parent != None: - self.emit(filesystem_dir, iso_bytes) - - for e in self.children: - e.emit_recursive(filesystem_dir, iso_bytes) - - -# Splits the ISO into its component parts - header info, apploader, DOL, FST metadata, and the individual files in the filesystem. -def split_iso(iso_bytes): - split_sys_info(iso_bytes) - split_content(iso_bytes) - - -# Splits the header info, apploader, DOL, and FST metadata from the ISO. -def split_sys_info(iso_bytes): - assert options.opts.filesystem_path is not None - - sys_path = options.opts.filesystem_path / "sys" - sys_path.mkdir(parents=True, exist_ok=True) - - # Split boot.info. Always at 0x0000 and 0x0440 bytes long. - with open(sys_path / "boot.bin", "wb") as f: - f.write(iso_bytes[0x0000:0x0440]) - - # Split bi2.info. Always at 0x0440 and 0x2000 bytes long. - with open(sys_path / "bi2.bin", "wb") as f: - f.write(iso_bytes[0x0440:0x2440]) - - # Split apploader.img. Always at 0x2440 and size is listed at 0x0400. - apploader_size = struct.unpack_from(">I", iso_bytes, 0x0400)[0] - with open(sys_path / "apploader.img", "wb") as f: - f.write(iso_bytes[0x2440 : 0x2440 + apploader_size]) - - # Split main.dol. Offset specified explicitly at 0x0420, but size must be calculated. - dol_offset = struct.unpack_from(">I", iso_bytes, 0x0420)[0] - fst_offset = struct.unpack_from(">I", iso_bytes, 0x0424)[0] - - dol_size = fst_offset - dol_offset - with open(sys_path / "main.dol", "wb") as f: - f.write(iso_bytes[dol_offset : dol_offset + dol_size]) - - # Split fst.bin. Offset specified at 0x0424 and size specified at 0x402C. - fst_size = struct.unpack_from(">I", iso_bytes, 0x0428)[0] - with open(sys_path / "fst.bin", "wb") as f: - f.write(iso_bytes[fst_offset : fst_offset + fst_size]) - - -# Splits the ISO's filesystem into individual files. -def split_content(iso_bytes): - assert options.opts.filesystem_path is not None - - fst_path = options.opts.filesystem_path / "sys" / "fst.bin" - assert fst_path.is_file() - - fst_bytes = fst_path.read_bytes() - fst_root_entry = populate_filesystem(fst_bytes) - - files_path = options.opts.filesystem_path / "files" - files_path.mkdir(parents=True, exist_ok=True) - fst_root_entry.emit_recursive(files_path, iso_bytes) - - -# Loads the FST data needed to split the filesystem. -def populate_filesystem(fst_bytes): - root_dir = GCFSTEntry( - bool(fst_bytes[0x0000]), - struct.unpack_from(">I", fst_bytes, 0x0000)[0] & 0x00FFFFFF, - struct.unpack_from(">I", fst_bytes, 0x0004)[0], - struct.unpack_from(">I", fst_bytes, 0x0008)[0], - ) - - string_table_bytes = fst_bytes[root_dir.length * 0x0C : len(fst_bytes)] - - # Parsing the filesystem is a bit tricky. The root directory's length property is the total number of nodes in the FST. - # So, we initialize nodes_read to 1, since the root is included in the number of nodes. - # We will rely on each directory and file on the root directory to tell us how many nodes were read while parsing them. - # We can stop reading the FST when our total number of nodes read is >= the number of nodes in the FST. - nodes_read = 1 - while nodes_read < root_dir.length: - current_offset = nodes_read * 0x0C - - new_entry = GCFSTEntry( - bool(fst_bytes[current_offset + 0x0000]), - struct.unpack_from(">I", fst_bytes, current_offset)[0] & 0x00FFFFFF, - struct.unpack_from(">I", fst_bytes, current_offset + 0x0004)[0], - struct.unpack_from(">I", fst_bytes, current_offset + 0x0008)[0], - ) - - root_dir.children.append(new_entry) - nodes_read += new_entry.populate_children_recursive( - root_dir, current_offset, fst_bytes, string_table_bytes - ) - - return root_dir diff --git a/tools/splat/util/gc/gcinfo.py b/tools/splat/util/gc/gcinfo.py deleted file mode 100644 index 78130df..0000000 --- a/tools/splat/util/gc/gcinfo.py +++ /dev/null @@ -1,88 +0,0 @@ -#! /usr/bin/env python3 - - -import argparse - -import hashlib - -from pathlib import Path -from typing import Optional - -parser = argparse.ArgumentParser( - description="Gives information on GameCube disc images" -) -parser.add_argument("iso", help="path to a GameCube disc image") - -system_codes = { - "D": "GameCube Demo", - "G": "GameCube", - "P": "GameCube Promotional", - "R": "Early Wii", - "S": "Later Wii", -} - -region_codes = {"E": "NTSC-U", "J": "NTSC-J", "P": "PAL"} - -publisher_codes = {"01": "Nintendo", "08": "Capcom", "8P": "Sega", "E9": "Natsume"} - - -def get_info(iso_path: Path, iso_bytes: Optional[bytes] = None): - if iso_bytes is None: - iso_bytes = iso_path.read_bytes() - - return get_info_bytes(iso_bytes) - - -def get_info_bytes(iso_bytes: bytes): - system_code = chr(iso_bytes[0x00]) - game_code = iso_bytes[0x01:0x03].decode("utf-8") - region_code = chr(iso_bytes[0x03]) - publisher_code = iso_bytes[0x04:0x06].decode("utf-8") - - name = str(iso_bytes[0x20:0x400], "utf-8").strip("\x00") - root = "filesystem" - - compiler = "mwcc" - sha1 = hashlib.sha1(iso_bytes).hexdigest() - - return GCIso( - name, - root, - system_code, - game_code, - region_code, - publisher_code, - compiler, - sha1, - ) - - -class GCIso: - def __init__( - self, - name: str, - root: str, - system_code, - game_code, - region_code, - publisher_code, - compiler, - sha1, - ): - self.name = name - self.root = root - self.system_code = system_code - self.game_code = game_code - self.region_code = region_code - self.publisher_code = publisher_code - self.compiler = compiler - self.sha1 = sha1 - - def get_system_name(self): - return system_codes[self.system_code] - - def get_publisher_name(self): - return publisher_codes[self.publisher_code] - - def get_region_name(self): - return region_codes[self.region_code] diff --git a/tools/splat/util/gc/gcutil.py b/tools/splat/util/gc/gcutil.py deleted file mode 100644 index ba78bc2..0000000 --- a/tools/splat/util/gc/gcutil.py +++ /dev/null @@ -1,11 +0,0 @@ -def read_string_from_bytes(name_offset, string_table_bytes): - bytes = bytearray() - - for offset in range(len(string_table_bytes) - name_offset): - cur_byte = string_table_bytes[name_offset + offset] - if cur_byte == 0x00: - break - - bytes.append(cur_byte) - - return bytes.decode("shift-jis") diff --git a/tools/splat/util/log.py b/tools/splat/util/log.py deleted file mode 100644 index 4f296da..0000000 --- a/tools/splat/util/log.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys -from typing import NoReturn, Optional -from pathlib import Path - -from colorama import Fore, init, Style - -init(autoreset=True) - -newline = True - -Status = Optional[str] - - -def write(*args, status=None, **kwargs): - global newline - - if not newline: - print("") - newline = True - - print(status_to_ansi(status) + str(args[0]), *args[1:], **kwargs) - - -def error(*args, **kwargs) -> NoReturn: - write(*args, **kwargs, status="error") - sys.exit(2) - - -# The line_num is expected to be zero-indexed -def parsing_error_preamble(path: Path, line_num: int, line: str): - write("") - write(f"error reading {path}, line {line_num + 1}:", status="error") - write(f"\t{line}") - - -def status_to_ansi(status: Status): - if status == "ok": - return Fore.GREEN - elif status == "warn": - return Fore.YELLOW + Style.BRIGHT - elif status == "error": - return Fore.RED + Style.BRIGHT - elif status == "skip": - return Style.DIM - else: - return "" diff --git a/tools/splat/util/n64/Mio0decompress.py b/tools/splat/util/n64/Mio0decompress.py deleted file mode 100644 index 64a059e..0000000 --- a/tools/splat/util/n64/Mio0decompress.py +++ /dev/null @@ -1,108 +0,0 @@ -import argparse -import struct -import sys - -try: - from .. import log - from .decompressor import Decompressor -except ImportError: - print(f"Run as python3 -m util.n64.Miodecompress") - sys.exit(1) - - -class GenericMio0Decompressor(Decompressor): - def __init__( - self, unpacked_offset, compressed_offset, uncompressed_offset, header_length - ): - self.unpacked_offset = unpacked_offset - self.compressed_offset = compressed_offset - self.uncompressed_offset = uncompressed_offset - self.header_length = header_length - - @staticmethod - def read_word(data, offset): - (res,) = struct.unpack(">I", data[offset : offset + 4]) - return res - - @staticmethod - def read_short(data, offset): - (res,) = struct.unpack(">H", data[offset : offset + 2]) - return res - - def decompress(self, in_bytes, byte_order="big") -> bytearray: - magic = in_bytes[0:4] - if magic != b"MIO0": - log.error(f"MIO0 magic is incorrect: {magic}") - - unpacked_size = self.read_word(in_bytes, self.unpacked_offset) - comp_offset = self.read_word(in_bytes, self.compressed_offset) - uncomp_offset = self.read_word(in_bytes, self.uncompressed_offset) - - layout_data = struct.iter_unpack(">I", in_bytes[self.header_length :]) - uncompressed_data = struct.iter_unpack(">B", in_bytes[uncomp_offset:]) - compressed_data = struct.iter_unpack(">H", in_bytes[comp_offset:]) - - idx = 0 - ret = bytearray(unpacked_size) - - mask_bit_counter = 0 - while idx < unpacked_size: - if mask_bit_counter == 0: - (current_mask,) = next(layout_data) - mask_bit_counter = 32 - - if current_mask & 0x80000000: - (ud,) = next(uncompressed_data) - ret[idx] = ud - idx += 1 - else: - (length_offset,) = next(compressed_data) - - length = (length_offset >> 12) + 3 - index = (length_offset & 0xFFF) + 1 - offset = idx - index - - if not (3 <= length <= 18): - log.error(f"Invalid length: {length}, corrupt data?") - - if not (1 <= index <= 4096): - log.error(f"Invalid index: {index}, corrupt data?") - - for i in range(length): - ret[idx] = ret[offset + i] - idx += 1 - - current_mask <<= 1 - mask_bit_counter -= 1 - - return ret - - -class Mio0Decompressor(GenericMio0Decompressor): - def __init__(self): - super().__init__( - unpacked_offset=4, - compressed_offset=8, - uncompressed_offset=12, - header_length=16, - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("infile") - parser.add_argument("outfile") - args = parser.parse_args() - - with open(args.infile, "rb") as f: - raw_bytes = f.read() - - miodecompress = Mio0Decompressor() - decompressed = miodecompress.decompress(raw_bytes) - - with open(args.outfile, "wb") as f: - f.write(decompressed) - - -if __name__ == "__main__": - main() diff --git a/tools/splat/util/n64/Yay0decompress.c b/tools/splat/util/n64/Yay0decompress.c deleted file mode 100644 index 250f347..0000000 --- a/tools/splat/util/n64/Yay0decompress.c +++ /dev/null @@ -1,44 +0,0 @@ -#include <stdio.h> -#include <stdint.h> -#include <stdbool.h> -#include <stdlib.h> - -typedef struct { - uint32_t magic; - uint32_t uncompressedLength; - uint32_t opPtr; - uint32_t dataPtr; -} Yay0Header; - -void decompress(Yay0Header* hdr, uint8_t* srcPtr, uint8_t* dstPtr, bool isBigEndian) { - uint8_t byte = 0, mask = 0; - uint8_t* ctrl, * ops, * data; - uint16_t copy, op; - uint32_t written = 0; - - ctrl = srcPtr + sizeof(Yay0Header); - ops = srcPtr + hdr->opPtr; - data = srcPtr + hdr->dataPtr; - - while (written < hdr->uncompressedLength) { - if ((mask >>= 1) == 0) { - byte = *ctrl++; - mask = 0x80; - } - - if (byte & mask) { - *dstPtr++ = *data++; - written++; - } else { - op = isBigEndian ? (ops[0] << 8) | ops[1] : (ops[1] << 8) | ops[0]; - ops += 2; - - written += copy = (op >> 12) ? (2 + (op >> 12)) : (18 + *data++); - - while (copy--) { - *dstPtr = dstPtr[-(op & 0xfff) - 1]; - dstPtr++; - } - } - } -} diff --git a/tools/splat/util/n64/Yay0decompress.py b/tools/splat/util/n64/Yay0decompress.py deleted file mode 100644 index 780d1cb..0000000 --- a/tools/splat/util/n64/Yay0decompress.py +++ /dev/null @@ -1,162 +0,0 @@ -import argparse -import os -import sys -from ctypes import CDLL, Structure, byref, c_bool, c_uint32, c_uint8, cdll -from struct import pack, unpack_from -from typing import Literal, Optional - -try: - from .. import log - from .decompressor import Decompressor -except ImportError: - print(f"Run as python3 -m util.n64.Yay0decompress") - sys.exit(1) - -tried_loading = False -lib: Optional[CDLL] = None - - -def setup_lib(): - global lib - global tried_loading - if lib: - return True - if tried_loading: - return False - tried_loading = True - try: - lib = cdll.LoadLibrary( - os.path.dirname(os.path.realpath(__file__)) + "/Yay0decompress" - ) - return True - except Exception: - log.write( - "Failed to load Yay0 C library; falling back to Python method", - status="warn", - ) - tried_loading = True - return False - - -class Yay0Decompressor(Decompressor): - @staticmethod - def decompress(in_bytes, byte_order: Literal["little", "big"] = "big") -> bytearray: - # attempt to load the library only once per execution - global lib - if not setup_lib(): - return Yay0Decompressor.decompress_python(in_bytes, byte_order) - assert lib is not None - - class Yay0(Structure): - _fields_ = [ - ("magic", c_uint32), - ("uncompressedLength", c_uint32), - ("opPtr", c_uint32), - ("dataPtr", c_uint32), - ] - - # read the file header - bigEndian = byte_order == "big" - if bigEndian: - # the struct is only a view, so when passed to C it will keep - # its BigEndian values and crash. Explicitly convert them here to little - hdr = Yay0.from_buffer_copy( - pack("<IIII", *unpack_from(">IIII", in_bytes, 0)) - ) - else: - hdr = Yay0.from_buffer_copy(in_bytes, 0) - - magic = getattr(hdr, hdr._fields_[0][0]) - if magic != int.from_bytes(str.encode("Yay0"), byteorder="big"): - log.error(f"Yay0 magic is incorrect: {magic}") - - # create the input/output buffers, copying data to in - src = (c_uint8 * len(in_bytes)).from_buffer_copy(in_bytes, 0) - dst = (c_uint8 * hdr.uncompressedLength)() - - # call decompress, equivilant to, in C: - # decompress(&hdr, &src, &dst, bigEndian) - lib.decompress(byref(hdr), byref(src), byref(dst), c_bool(bigEndian)) - - # other functions want the results back as a non-ctypes type - return bytearray(dst) - - @staticmethod - def decompress_python(in_bytes, byte_order: Literal["little", "big"] = "big"): - if in_bytes[:4] != b"Yay0": - log.error("Input file is not Yay0") - - decompressed_size = int.from_bytes(in_bytes[4:8], byteorder=byte_order) - link_table_offset = int.from_bytes(in_bytes[8:12], byteorder=byte_order) - chunk_offset = int.from_bytes(in_bytes[12:16], byteorder=byte_order) - - link_table_idx = link_table_offset - chunk_idx = chunk_offset - other_idx = 16 - - mask_bit_counter = 0 - current_mask = 0 - - # preallocate result and index into it - idx = 0 - ret = bytearray(decompressed_size) - - while idx < decompressed_size: - # If we're out of bits, get the next mask - if mask_bit_counter == 0: - current_mask = int.from_bytes( - in_bytes[other_idx : other_idx + 4], byteorder=byte_order - ) - other_idx += 4 - mask_bit_counter = 32 - - if current_mask & 0x80000000: - ret[idx] = in_bytes[chunk_idx] - idx += 1 - chunk_idx += 1 - else: - link = int.from_bytes( - in_bytes[link_table_idx : link_table_idx + 2], byteorder=byte_order - ) - link_table_idx += 2 - - offset = idx - (link & 0xFFF) - - count = link >> 12 - - if count == 0: - count_modifier = in_bytes[chunk_idx] - chunk_idx += 1 - count = count_modifier + 18 - else: - count += 2 - - # Copy the block - for i in range(count): - ret[idx] = ret[offset + i - 1] - idx += 1 - - current_mask <<= 1 - mask_bit_counter -= 1 - - return ret - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("infile") - parser.add_argument("outfile") - parser.add_argument("--byte-order", default="big", choices=["big", "little"]) - args = parser.parse_args() - - with open(args.infile, "rb") as f: - raw_bytes = f.read() - - decompressed = Yay0Decompressor.decompress(raw_bytes, args.byte_order) - - with open(args.outfile, "wb") as f: - f.write(decompressed) - - -if __name__ == "__main__": - main() diff --git a/tools/splat/util/n64/__init__.py b/tools/splat/util/n64/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/tools/splat/util/n64/__init__.py +++ /dev/null diff --git a/tools/splat/util/n64/decompressor.py b/tools/splat/util/n64/decompressor.py deleted file mode 100644 index 7df0de0..0000000 --- a/tools/splat/util/n64/decompressor.py +++ /dev/null @@ -1,10 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Literal - - -class Decompressor(ABC): - @abstractmethod - def decompress( - self, in_bytes, byte_order: Literal["little", "big"] = "big" - ) -> bytearray: - pass diff --git a/tools/splat/util/n64/find_code_length.py b/tools/splat/util/n64/find_code_length.py deleted file mode 100755 index 1f9053c..0000000 --- a/tools/splat/util/n64/find_code_length.py +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env python3 - -import argparse - -import rabbitizer -import spimdisasm - - -def int_any_base(x): - return int(x, 0) - - -parser = argparse.ArgumentParser( - description="Given a rom and start offset, find where the code ends" -) -parser.add_argument("rom", help="path to a .z64 rom") -parser.add_argument("start", help="start offset", type=int_any_base) -parser.add_argument("--end", help="end offset", default=None, type=int_any_base) -parser.add_argument( - "--vram", - help="vram address to start disassembly at", - default="0x80000000", - type=int_any_base, -) - - -def run(rom_bytes, start_offset, vram, end_offset=None): - rom_addr = start_offset - last_return = rom_addr - - wordList = spimdisasm.common.Utils.bytesToBEWords(rom_bytes[start_offset:]) - for word in wordList: - insn = rabbitizer.Instruction(word) - if not insn.isImplemented(): - break - - if insn.isJrRa(): - last_return = rom_addr - rom_addr += 4 - if end_offset and rom_addr >= end_offset: - break - - # align to next 0x10 boundary - end = last_return + 0x10 - end -= end % 0x10 - return end - - -def main(): - args = parser.parse_args() - - with open(args.rom, "rb") as f: - rom_bytes = f.read() - - start = args.start - end = args.end - vram = args.vram - - print(f"0x{run(rom_bytes, start, vram, end):X}") - - -if __name__ == "__main__": - main() diff --git a/tools/splat/util/n64/rominfo.py b/tools/splat/util/n64/rominfo.py deleted file mode 100755 index cede4de..0000000 --- a/tools/splat/util/n64/rominfo.py +++ /dev/null @@ -1,322 +0,0 @@ -#! /usr/bin/env python3 - -import argparse - -import hashlib -import itertools -import struct - -import sys -import zlib -from dataclasses import dataclass - -from pathlib import Path -from typing import Optional - -import rabbitizer -import spimdisasm - -parser = argparse.ArgumentParser(description="Gives information on N64 roms") -parser.add_argument("rom", help="path to an N64 rom") -parser.add_argument( - "--header-encoding", - dest="header_encoding", - help=( - "Text encoding the game header is using;" - " see docs.python.org/3/library/codecs.html#standard-encodings for valid encodings" - ), -) - -country_codes = { - 0x00: "Unknown", - 0x37: "Beta", - 0x41: "Asian (NTSC)", - 0x42: "Brazilian", - 0x43: "Chinese", - 0x44: "German", - 0x45: "North America", - 0x46: "French", - 0x47: "Gateway 64 (NTSC)", - 0x48: "Dutch", - 0x49: "Italian", - 0x4A: "Japanese", - 0x4B: "Korean", - 0x4C: "Gateway 64 (PAL)", - 0x4E: "Canadian", - 0x50: "European (basic spec.)", - 0x53: "Spanish", - 0x55: "Australian", - 0x57: "Scandinavian", - 0x58: "European", - 0x59: "European", -} - - -@dataclass -class CIC: - ntsc_name: str - pal_name: str - offset: int - - -crc_to_cic = { - 0x6170A4A1: CIC("6101", "7102", 0x000000), - 0x90BB6CB5: CIC("6102", "7101", 0x000000), - 0x0B050EE0: CIC("6103", "7103", 0x100000), - 0x98BC2C86: CIC("6105", "7105", 0x000000), - 0xACC8580A: CIC("6106", "7106", 0x200000), -} -unknown_cic = CIC("unknown", "unknown", 0x0000000) - - -@dataclass -class N64EntrypointInfo: - entry_size: int - bss_start_address: Optional[int] - bss_size: Optional[int] - main_address: Optional[int] - stack_top: int - - @staticmethod - def parse_rom_bytes( - rom_bytes, offset: int = 0x1000, size: int = 0x60 - ) -> "N64EntrypointInfo": - word_list = spimdisasm.common.Utils.bytesToWords( - rom_bytes, offset, offset + size - ) - nops_count = 0 - - register_values = [0 for _ in range(32)] - - register_bss_address: Optional[int] = None - register_bss_size: Optional[int] = None - register_main_address: Optional[int] = None - - size = 0 - for word in word_list: - insn = rabbitizer.Instruction(word) - if not insn.isImplemented(): - break - - if insn.isNop(): - nops_count += 1 - elif nops_count >= 3: - break - elif insn.canBeHi(): - register_values[insn.rt.value] = insn.getProcessedImmediate() << 16 - elif insn.canBeLo(): - if insn.isLikelyHandwritten(): - # Try to skip these instructions: - # addi $t0, $t0, 0x8 - # addi $t1, $t1, -0x8 - pass - elif insn.modifiesRt(): - register_values[insn.rt.value] = ( - register_values[insn.rs.value] + insn.getProcessedImmediate() - ) - elif insn.doesStore(): - if insn.rt == rabbitizer.RegGprO32.zero: - # Try to detect the zero-ing bss algorithm - # sw $zero, 0x0($t0) - register_bss_address = insn.rs.value - elif insn.isBranch(): - # lui $t1, 0x2 - # addiu $t1, $t1, -0x7220 - # ... - # addi $t1, $t1, -0x8 - # ... - # bnez $t1, label - register_bss_size = insn.rs.value - - elif insn.isJumptableJump() or insn.isReturn(): - # lui $t2, 0x8000 - # addiu $t2, $t2, 0x494 - # ... - # jr $t2 - register_main_address = insn.rs.value - - # print(f"{word:08X}", insn) - size += 4 - - # for i, val in enumerate(register_values): - # print(i, f"{val:08X}") - - bss_address = ( - register_values[register_bss_address] - if register_bss_address is not None - else None - ) - bss_size = ( - register_values[register_bss_size] - if register_bss_size is not None - else None - ) - main_address = ( - register_values[register_main_address] - if register_main_address is not None - else None - ) - stack_top = register_values[rabbitizer.RegGprO32.sp.value] - return N64EntrypointInfo(size, bss_address, bss_size, main_address, stack_top) - - -@dataclass -class N64Rom: - name: str - header_encoding: str - country_code: int - libultra_version: str - checksum: str - cic: CIC - entry_point: int - size: int - compiler: str - sha1: str - entrypoint_info: N64EntrypointInfo - - def get_country_name(self) -> str: - return country_codes[self.country_code] - - -def swap_bytes(data): - return bytes( - itertools.chain.from_iterable( - struct.pack(">H", x) for (x,) in struct.iter_unpack("<H", data) - ) - ) - - -def read_rom(rom_path: Path): - rom_bytes = rom_path.read_bytes() - - if rom_path.suffix.lower() == ".n64": - print("Warning: Input file has .n64 suffix, byte-swapping!") - rom_bytes = swap_bytes(rom_bytes) - as_z64 = rom_path.with_suffix(".z64") - if not as_z64.exists(): - print(f"Writing down {as_z64}") - as_z64.write_bytes(rom_bytes) - return rom_bytes - - -def get_cic(rom_bytes: bytes): - ipl3_crc = zlib.crc32(rom_bytes[0x40:0x1000]) - - return crc_to_cic.get(ipl3_crc, unknown_cic) - - -def get_entry_point(program_counter: int, cic: CIC): - return program_counter - cic.offset - - -def guess_header_encoding(rom_bytes: bytes): - header = rom_bytes[0x20:0x34] - encodings = ["ASCII", "shift_jis", "euc-jp"] - for encoding in encodings: - try: - header.decode(encoding) - return encoding - except UnicodeDecodeError: - # we guessed wrong... - pass - - sys.exit("Unknown header encoding, please raise an Issue with us") - - -def get_info( - rom_path: Path, rom_bytes: Optional[bytes] = None, header_encoding=None -) -> N64Rom: - if rom_bytes is None: - rom_bytes = read_rom(rom_path) - - if header_encoding is None: - header_encoding = guess_header_encoding(rom_bytes) - - return get_info_bytes(rom_bytes, header_encoding) - - -def get_info_bytes(rom_bytes: bytes, header_encoding: str) -> N64Rom: - (program_counter,) = struct.unpack(">I", rom_bytes[0x8:0xC]) - libultra_version = chr(rom_bytes[0xF]) - checksum = rom_bytes[0x10:0x18].hex().upper() - - try: - name = rom_bytes[0x20:0x34].decode(header_encoding).rstrip(" \0") or "empty" - except: - sys.exit( - "splat could not decode the game name;" - " try using a different encoding by passing the --header-encoding argument" - " (see docs.python.org/3/library/codecs.html#standard-encodings for valid encodings)" - ) - - country_code = rom_bytes[0x3E] - - cic = get_cic(rom_bytes) - entry_point = get_entry_point(program_counter, cic) - - compiler = get_compiler_info(rom_bytes, entry_point, print_result=False) - - sha1 = hashlib.sha1(rom_bytes).hexdigest() - - entrypoint_info = N64EntrypointInfo.parse_rom_bytes(rom_bytes) - - return N64Rom( - name, - header_encoding, - country_code, - libultra_version, - checksum, - cic, - entry_point, - len(rom_bytes), - compiler, - sha1, - entrypoint_info, - ) - - -def get_compiler_info(rom_bytes, entry_point, print_result=True): - jumps = 0 - branches = 0 - - word_list = spimdisasm.common.Utils.bytesToWords(rom_bytes[0x1000:]) - for word in word_list: - insn = rabbitizer.Instruction(word) - if not insn.isImplemented(): - break - - if insn.uniqueId == rabbitizer.InstrId.cpu_j: - jumps += 1 - elif insn.uniqueId == rabbitizer.InstrId.cpu_b: - branches += 1 - - compiler = "IDO" if branches > jumps else "GCC" - if print_result: - print( - f"{branches} branches and {jumps} jumps detected in the first code segment." - f" Compiler is most likely {compiler}" - ) - return compiler - - -def main(): - rabbitizer.config.pseudos_pseudoB = True - - args = parser.parse_args() - rom_bytes = read_rom(Path(args.rom)) - rom = get_info(Path(args.rom), rom_bytes, args.header_encoding) - - print("Image name: " + rom.name) - print("Country code: " + chr(rom.country_code) + " - " + rom.get_country_name()) - print("Libultra version: " + rom.libultra_version) - print("Checksum: " + rom.checksum) - print("CIC: " + rom.cic.ntsc_name + " / " + rom.cic.pal_name) - print("RAM entry point: " + hex(rom.entry_point)) - print("Header encoding: " + rom.header_encoding) - print("") - - get_compiler_info(rom_bytes, rom.entry_point) - - -if __name__ == "__main__": - main() diff --git a/tools/splat/util/options.py b/tools/splat/util/options.py deleted file mode 100644 index 2229cad..0000000 --- a/tools/splat/util/options.py +++ /dev/null @@ -1,476 +0,0 @@ -from dataclasses import dataclass -import os -from pathlib import Path -from typing import cast, Dict, List, Literal, Mapping, Optional, Set, Type, TypeVar - -from util import compiler -from util.compiler import Compiler - - -@dataclass -class SplatOpts: - # Debug / logging - verbose: bool - dump_symbols: bool - modes: List[str] - - # Project configuration - - # Determines the base path of the project. Everything is relative to this path - base_path: Path - # Determines the path to the target binary - target_path: Path - # Determines the platform of the target binary - platform: str - # Determines the compiler used to compile the target binary - compiler: Compiler - # Determines the endianness of the target binary - endianness: Literal["big", "little"] - # Determines the default section order of the target binary - # this can be overridden per-segment - section_order: List[str] - # Determines the code that is inserted by default in generated .c files - generated_c_preamble: str - # Determines the code that is inserted by default in generated .s files - generated_s_preamble: str - # Determines whether to use .o as the suffix for all binary files?... TODO document - use_o_as_suffix: bool - # the value of the $gp register to correctly calculate offset to %gp_rel relocs - gp: Optional[int] - - # Paths - asset_path: Path - # Determines the path to the symbol addresses file(s) - # A symbol_addrs file is to be updated/curated manually and contains addresses of symbols - # as well as optional metadata such as rom address, type, and more - # - # It's possible to use more than one file by supplying a list instead of a string - symbol_addrs_paths: List[Path] - reloc_addrs_paths: List[Path] - # Determines the path to the project build directory - build_path: Path - # Determines the path to the source code directory - src_path: Path - # Determines the path to the asm code directory - asm_path: Path - # Determines the path to the asm data directory - data_path: Path - # Determines the path to the asm nonmatchings directory - nonmatchings_path: Path - # Determines the path to the cache file (used when supplied --use-cache via the CLI) - cache_path: Path - - # Determines whether to create an automatically-generated undefined functions file - # this file stores all functions that are referenced in the code but are not defined as seen by splat - create_undefined_funcs_auto: bool - # Determines the path to the undefined_funcs_auto file - undefined_funcs_auto_path: Path - - # Determines whether to create an automatically-generated undefined symbols file - # this file stores all symbols that are referenced in the code but are not defined as seen by splat - create_undefined_syms_auto: bool - # Determines the path to the undefined_symbols_auto file - undefined_syms_auto_path: Path - - # Determines the path in which to search for custom splat extensions - extensions_path: Optional[Path] - - # Determines the path to library files that are to be linked into the target binary - lib_path: Path - - # TODO document - elf_section_list_path: Optional[Path] - - # Linker script - # Determines the default subalign value to be specified in the generated linker script - subalign: int - # The following option determines whether to automatically configure the linker script to link against - # specified sections for all "base" (asm/c) files when the yaml doesn't have manual configurations - # for these sections. - auto_all_sections: List[str] - # Determines the desired path to the linker script that splat will generate - ld_script_path: Path - # Determines the desired path to the linker symbol header, - # which exposes externed definitions for all segment ram/rom start/end locations - ld_symbol_header_path: Optional[Path] - # Determines whether to add a discard section to the linker script - ld_discard_section: bool - # Determines the list of section labels that are to be added to the linker script - ld_section_labels: List[str] - # Determines whether to add wildcards for section linking in the linker script (.rodata* for example) - ld_wildcard_sections: bool - # Determines whether to use "follows" settings to determine locations of overlays in the linker script. - # If disabled, this effectively ignores "follows" directives in the yaml. - ld_use_follows: bool - # If enabled, the end symbol for each segment will be placed before the alignment directive for the segment - segment_end_before_align: bool - # Controls the style of the auto-generated segment symbols in the linker script. Possible values: splat, makerom - segment_symbols_style: str - # Specifies the starting offset for rom address symbols in the linker script. - ld_rom_start: int - - ################################################################################ - # C file options - ################################################################################ - # Determines whether to create new c files if they don't exist - create_c_files: bool - # Determines whether to "auto-decompile" empty functions - auto_decompile_empty_functions: bool - # Determines whether to detect matched/unmatched functions in existing c files - # so we can avoid creating .s files for already-decompiled functions - do_c_func_detection: bool - # Determines the newline char(s) to be used in c files - c_newline: str - - ################################################################################ - # (Dis)assembly-related options - ################################################################################ - # The following options determine the format that symbols should be named by default - symbol_name_format: str - # Same as above but for symbols with no rom address - symbol_name_format_no_rom: str - # Determines whether to detect and hint to the user about likely file splits when disassembling - find_file_boundaries: bool - # Determines whether to detect and hint to the user about possible rodata sections corresponding to a text section - pair_rodata_to_text: bool - # Determines whether to attempt to automatically migrate rodata into functions - # (only works in certain circumstances) - migrate_rodata_to_functions: bool - # Determines the header to be used in every asm file that's included from c files - asm_inc_header: str - # Determines the macro used to declare functions in asm files - asm_function_macro: str - # Determines the macro used to declare jumptable labels in asm files - asm_jtbl_label_macro: str - # Determines the macro used to declare data symbols in asm files - asm_data_macro: str - # Determines the macro used at the end of a function, such as endlabel or .end - asm_end_label: str - # Toggles the .size directive emitted by the disassembler - asm_emit_size_directive: Optional[bool] - # Determines including the macro.inc file on non-migrated rodata variables - include_macro_inc: bool - # Determines the number of characters to left align before the TODO finish documenting - mnemonic_ljust: int - # Determines whether to pad the rom address - rom_address_padding: bool - # Determines which ABI names to use for general purpose registers - mips_abi_gpr: str - # Determines which ABI names to use for floating point registers - # Valid values: 'numeric', 'o32', 'n32', 'n64' - # o32 is highly recommended, as it provides logically named registers for floating point instructions - # For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d - mips_abi_float_regs: str - # Determines whether to add ".set gp=64" to asm/hasm files - add_set_gp_64: bool - # Generate .asmproc.d dependency files for each C file which still reference functions in assembly files - create_asm_dependencies: bool - # Global option for rodata string encoding. This can be overriden per segment - string_encoding: Optional[str] - # Global option for data string encoding. This can be overriden per segment - data_string_encoding: Optional[str] - # Global option for the rodata string guesser. 0 disables the guesser completely. - rodata_string_guesser_level: Optional[int] - # Global option for the data string guesser. 0 disables the guesser completely. - data_string_guesser_level: Optional[int] - # Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol - allow_data_addends: bool - # Determines whether to include the "Generated by spimdisasm" text in the asm - asm_generated_by: bool - # Tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data - disasm_unknown: bool - # Tries to detect redundant and unreferenced functions ends and merge them together. This option is ignored if the compiler is not set to IDO. - detect_redundant_function_end: bool - # Don't skip disassembling already matched functions and migrated sections - disassemble_all: bool - - ################################################################################ - # N64-specific options - ################################################################################ - # Determines the encoding of the header - header_encoding: str - # Determines the type gfx ucode (used by gfx segments) - # Valid options are ['f3d', 'f3db', 'f3dex', 'f3dexb', 'f3dex2'] - gfx_ucode: str - # Use named libultra symbols by default. Those will need to be added to a linker script manually by the user - libultra_symbols: bool - # Use named hardware register symbols by default. Those will need to be added to a linker script manually by the user - hardware_regs: bool - - ################################################################################ - # Gamecube-specific options - ################################################################################ - # Path where the iso's filesystem will be extracted to - filesystem_path: Optional[Path] - - ################################################################################ - # Compiler-specific options - ################################################################################ - # Determines whether to use a legacy INCLUDE_ASM macro format in c files - # only applies to GCC/SN64 - use_legacy_include_asm: bool - - # Returns whether the given mode is currently enabled - def is_mode_active(self, mode: str) -> bool: - return mode in self.modes or "all" in self.modes - - -opts: SplatOpts - - -T = TypeVar("T") - - -class OptParser: - _read_opts: Set[str] - - def __init__(self, yaml: Mapping[str, object]) -> None: - self._yaml = yaml - self._read_opts = set() - - def parse_opt(self, opt: str, t: Type[T], default: Optional[T] = None) -> T: - if opt not in self._yaml: - if default is not None: - return default - raise ValueError(f"Missing required option {opt}") - self._read_opts.add(opt) - value = self._yaml[opt] - if isinstance(value, t): - return value - if t is float and isinstance(value, int): - return cast(T, float(value)) - raise ValueError(f"Expected {opt} to have type {t}, got {type(value)}") - - def parse_optional_opt(self, opt: str, t: Type[T]) -> Optional[T]: - if opt not in self._yaml: - return None - return self.parse_opt(opt, t) - - def parse_opt_within( - self, opt: str, t: Type[T], within: List[T], default: Optional[T] = None - ) -> T: - value = self.parse_opt(opt, t, default) - if value not in within: - raise ValueError(f"Invalid value for {opt}: {value}") - return value - - def parse_path( - self, base_path: Path, opt: str, default: Optional[str] = None - ) -> Path: - return Path(os.path.normpath(base_path / self.parse_opt(opt, str, default))) - - def parse_optional_path(self, base_path: Path, opt: str) -> Optional[Path]: - if opt not in self._yaml: - return None - return self.parse_path(base_path, opt) - - def parse_path_list(self, base_path: Path, opt: str, default: str) -> List[Path]: - paths = self.parse_opt(opt, object, default) - - if isinstance(paths, str): - return [base_path / paths] - elif isinstance(paths, list): - return [base_path / path for path in paths] - else: - raise ValueError(f"Expected str or list for '{opt}', got {type(paths)}") - - def check_no_unread_opts(self) -> None: - opts = [opt for opt in self._yaml if opt not in self._read_opts] - if opts: - raise ValueError(f"Unrecognized YAML option(s): {', '.join(opts)}") - - -def _parse_yaml( - yaml: Dict, - config_paths: List[str], - modes: List[str], - verbose: bool = False, - disasm_all: bool = False, -) -> SplatOpts: - p = OptParser(yaml) - - basename = p.parse_opt("basename", str) - platform = p.parse_opt_within("platform", str, ["n64", "psx", "gc", "ps2"]) - comp = compiler.for_name(p.parse_opt("compiler", str, "IDO")) - - base_path = Path( - os.path.normpath(Path(config_paths[0]).parent / p.parse_opt("base_path", str)) - ) - asm_path: Path = p.parse_path(base_path, "asm_path", "asm") - - asm_emit_size_directive = p.parse_optional_opt("asm_emit_size_directive", bool) - # If option not provided then use the compiler default - if asm_emit_size_directive is None: - asm_emit_size_directive = comp.asm_emit_size_directive - - def parse_endianness() -> Literal["big", "little"]: - endianness = p.parse_opt_within( - "endianness", - str, - ["big", "little"], - "little" if platform in ["psx", "ps2"] else "big", - ) - - if endianness == "big": - return "big" - elif endianness == "little": - return "little" - else: - raise ValueError(f"Invalid endianness: {endianness}") - - ret = SplatOpts( - verbose=verbose, - dump_symbols=p.parse_opt("dump_symbols", bool, False), - modes=modes, - base_path=base_path, - target_path=p.parse_path(base_path, "target_path"), - platform=platform, - compiler=comp, - endianness=parse_endianness(), - section_order=p.parse_opt( - "section_order", list, [".text", ".data", ".rodata", ".bss"] - ), - generated_c_preamble=p.parse_opt( - "generated_c_preamble", str, '#include "common.h"' - ), - generated_s_preamble=p.parse_opt("generated_s_preamble", str, ""), - use_o_as_suffix=p.parse_opt("o_as_suffix", bool, False), - gp=p.parse_opt("gp_value", int, 0), - asset_path=p.parse_path(base_path, "asset_path", "assets"), - symbol_addrs_paths=p.parse_path_list( - base_path, "symbol_addrs_path", "symbol_addrs.txt" - ), - reloc_addrs_paths=p.parse_path_list( - base_path, "reloc_addrs_path", "reloc_addrs.txt" - ), - build_path=p.parse_path(base_path, "build_path", "build"), - src_path=p.parse_path(base_path, "src_path", "src"), - asm_path=asm_path, - data_path=p.parse_path(asm_path, "data_path", "data"), - nonmatchings_path=p.parse_path(asm_path, "nonmatchings_path", "nonmatchings"), - cache_path=p.parse_path(base_path, "cache_path", ".splache"), - create_undefined_funcs_auto=p.parse_opt( - "create_undefined_funcs_auto", bool, True - ), - undefined_funcs_auto_path=p.parse_path( - base_path, "undefined_funcs_auto_path", "undefined_funcs_auto.txt" - ), - create_undefined_syms_auto=p.parse_opt( - "create_undefined_syms_auto", bool, True - ), - undefined_syms_auto_path=p.parse_path( - base_path, "undefined_syms_auto_path", "undefined_syms_auto.txt" - ), - extensions_path=p.parse_optional_path(base_path, "extensions_path"), - lib_path=p.parse_path(base_path, "lib_path", "lib"), - elf_section_list_path=p.parse_optional_path(base_path, "elf_section_list_path"), - subalign=p.parse_opt("subalign", int, 16), - auto_all_sections=p.parse_opt( - "auto_all_sections", list, [".data", ".rodata", ".bss"] - ), - ld_script_path=p.parse_path(base_path, "ld_script_path", f"{basename}.ld"), - ld_symbol_header_path=p.parse_optional_path(base_path, "ld_symbol_header_path"), - ld_discard_section=p.parse_opt("ld_discard_section", bool, True), - ld_section_labels=p.parse_opt( - "ld_section_labels", - list, - [".text", ".data", ".rodata", ".bss"], - ), - ld_wildcard_sections=p.parse_opt("ld_wildcard_sections", bool, False), - ld_use_follows=p.parse_opt("ld_use_follows", bool, True), - segment_end_before_align=p.parse_opt("segment_end_before_align", bool, False), - segment_symbols_style=p.parse_opt_within( - "segment_symbols_style", str, ["splat", "makerom"], "splat" - ), - ld_rom_start=p.parse_opt("ld_rom_start", int, 0), - create_c_files=p.parse_opt("create_c_files", bool, True), - auto_decompile_empty_functions=p.parse_opt( - "auto_decompile_empty_functions", bool, True - ), - do_c_func_detection=p.parse_opt("do_c_func_detection", bool, True), - c_newline=p.parse_opt("c_newline", str, comp.c_newline), - symbol_name_format=p.parse_opt("symbol_name_format", str, "$VRAM"), - symbol_name_format_no_rom=p.parse_opt( - "symbol_name_format_no_rom", str, "$VRAM_$SEG" - ), - find_file_boundaries=p.parse_opt("find_file_boundaries", bool, True), - pair_rodata_to_text=p.parse_opt("pair_rodata_to_text", bool, True), - migrate_rodata_to_functions=p.parse_opt( - "migrate_rodata_to_functions", bool, True - ), - asm_inc_header=p.parse_opt("asm_inc_header", str, comp.asm_inc_header), - asm_function_macro=p.parse_opt( - "asm_function_macro", str, comp.asm_function_macro - ), - asm_jtbl_label_macro=p.parse_opt( - "asm_jtbl_label_macro", str, comp.asm_jtbl_label_macro - ), - asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro), - asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label), - asm_emit_size_directive=asm_emit_size_directive, - include_macro_inc=p.parse_opt( - "include_macro_inc", bool, comp.include_macro_inc - ), - mnemonic_ljust=p.parse_opt("mnemonic_ljust", int, 11), - rom_address_padding=p.parse_opt("rom_address_padding", bool, False), - mips_abi_gpr=p.parse_opt_within( - "mips_abi_gpr", - str, - ["numeric", "o32", "n32", "n64"], - "o32", - ), - mips_abi_float_regs=p.parse_opt_within( - "mips_abi_float_regs", - str, - ["numeric", "o32", "n32", "n64"], - "numeric", - ), - add_set_gp_64=p.parse_opt("add_set_gp_64", bool, True), - create_asm_dependencies=p.parse_opt("create_asm_dependencies", bool, False), - string_encoding=p.parse_optional_opt("string_encoding", str), - data_string_encoding=p.parse_optional_opt("data_string_encoding", str), - rodata_string_guesser_level=p.parse_optional_opt( - "rodata_string_guesser_level", int - ), - data_string_guesser_level=p.parse_optional_opt( - "data_string_guesser_level", int - ), - allow_data_addends=p.parse_opt("allow_data_addends", bool, True), - header_encoding=p.parse_opt("header_encoding", str, "ASCII"), - gfx_ucode=p.parse_opt_within( - "gfx_ucode", - str, - ["f3d", "f3db", "f3dex", "f3dexb", "f3dex2"], - "f3dex2", - ), - libultra_symbols=p.parse_opt("libultra_symbols", bool, False), - hardware_regs=p.parse_opt("hardware_regs", bool, False), - use_legacy_include_asm=p.parse_opt("use_legacy_include_asm", bool, True), - filesystem_path=p.parse_optional_path(base_path, "filesystem_path"), - asm_generated_by=p.parse_opt("asm_generated_by", bool, True), - disasm_unknown=p.parse_opt("disasm_unknown", bool, False), - detect_redundant_function_end=p.parse_opt( - "detect_redundant_function_end", bool, True - ), - # Command line argument takes precedence over yaml option - disassemble_all=disasm_all - if disasm_all - else p.parse_opt("disassemble_all", bool, False), - ) - p.check_no_unread_opts() - return ret - - -def initialize( - config: Dict, - config_paths: List[str], - modes: Optional[List[str]] = None, - verbose=False, - disasm_all=False, -): - global opts - - if not modes: - modes = ["all"] - - opts = _parse_yaml(config["options"], config_paths, modes, verbose, disasm_all) diff --git a/tools/splat/util/palettes.py b/tools/splat/util/palettes.py deleted file mode 100644 index 8ee1e9a..0000000 --- a/tools/splat/util/palettes.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Dict - -from segtypes.common.group import CommonSegGroup -from segtypes.n64.ci import N64SegCi -from segtypes.n64.palette import N64SegPalette as Palette - - -# Resolve Raster#palette and Palette#raster links -def initialize(all_segments): - def process(segments): - raster_map: Dict[str, N64SegCi] = {} - palette_map: Dict[str, Palette] = {} - - for segment in segments: - if isinstance(segment, Palette): - palette_map[segment.name] = segment - - if isinstance(segment, N64SegCi): - raster_map[segment.name] = segment - - if isinstance(segment, CommonSegGroup): - process(segment.subsegments) - - for raster_name in raster_map: - raster = raster_map[raster_name] - # print(f"{raster_name} -> {raster.palette_name}") - raster.palette = palette_map.get(raster.palette_name) - - for palette_name in palette_map: - palette = palette_map[palette_name] - # print(f"{palette_name} -> {palette.raster_name}") - palette.raster = raster_map.get(palette.raster_name) - - process(all_segments) diff --git a/tools/splat/util/progress_bar.py b/tools/splat/util/progress_bar.py deleted file mode 100644 index caec2ce..0000000 --- a/tools/splat/util/progress_bar.py +++ /dev/null @@ -1,8 +0,0 @@ -import tqdm -import sys - -out_file = sys.stderr - - -def get_progress_bar(elements: list) -> tqdm.tqdm: - return tqdm.tqdm(elements, total=len(elements), file=out_file) diff --git a/tools/splat/util/range.py b/tools/splat/util/range.py deleted file mode 100644 index 04aea6e..0000000 --- a/tools/splat/util/range.py +++ /dev/null @@ -1,17 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - - -@dataclass -class Range: - start: Optional[int] = None - end: Optional[int] = None - - def has_start(self): - return self.start is not None - - def has_end(self): - return self.end is not None - - def is_complete(self): - return self.has_start() and self.has_end() diff --git a/tools/splat/util/relocs.py b/tools/splat/util/relocs.py deleted file mode 100644 index 08e668f..0000000 --- a/tools/splat/util/relocs.py +++ /dev/null @@ -1,130 +0,0 @@ -from dataclasses import dataclass -from typing import Dict - -import spimdisasm - -from util import log, options, symbols, progress_bar - - -@dataclass -class Reloc: - rom_address: int - reloc_type: str - symbol_name: str - - addend: int = 0 - - -all_relocs: Dict[int, Reloc] = {} - - -def add_reloc(reloc: Reloc): - all_relocs[reloc.rom_address] = reloc - - -def initialize(): - global all_relocs - - all_relocs = {} - - for path in options.opts.reloc_addrs_paths: - if not path.exists(): - continue - - with path.open() as f: - sym_addrs_lines = f.readlines() - - prog_bar = progress_bar.get_progress_bar(sym_addrs_lines) - prog_bar.set_description(f"Loading relocs ({path.stem})") - line: str - for line_num, line in enumerate(prog_bar): - line = line.strip() - # Allow comments - line = line.split("//")[0] - line = line.strip() - - if line == "": - continue - - rom_addr = None - reloc_type = None - symbol_name = None - addend = None - - for info in line.split(" "): - if ":" not in info: - continue - - if info.count(":") > 1: - log.parsing_error_preamble(path, line_num, line) - log.write(f"Too many ':'s in '{info}'") - log.error("") - - attr_name, attr_val = info.split(":") - if attr_name == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute name in '{info}', is there extra whitespace?" - ) - log.error("") - if attr_val == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute value in '{info}', is there extra whitespace?" - ) - log.error("") - - # Non-Boolean attributes - try: - if attr_name == "rom": - rom_addr = int(attr_val, 0) - continue - if attr_name == "reloc": - reloc_type = attr_val - continue - if attr_name == "symbol": - symbol_name = attr_val - continue - if attr_name == "addend": - addend = int(attr_val, 0) - continue - except: - log.parsing_error_preamble(path, line_num, line) - log.write(f"value of attribute '{attr_name}' could not be read:") - log.write("") - raise - - if rom_addr is None: - log.parsing_error_preamble(path, line_num, line) - log.error(f"Missing required 'rom' attribute for reloc") - if reloc_type is None: - log.parsing_error_preamble(path, line_num, line) - log.error(f"Missing required 'reloc' attribute for reloc") - if symbol_name is None: - log.parsing_error_preamble(path, line_num, line) - log.error(f"Missing required 'symbol' attribute for reloc") - - reloc = Reloc(rom_addr, reloc_type, symbol_name) - if addend is not None: - reloc.addend = addend - - if reloc.rom_address in all_relocs: - log.parsing_error_preamble(path, line_num, line) - log.error( - f"Duplicated 'rom' address for reloc: 0x{reloc.rom_address:X}" - ) - add_reloc(reloc) - - -def initialize_spim_context(): - for rom_address, reloc in all_relocs.items(): - reloc_type = spimdisasm.common.RelocType.fromStr(reloc.reloc_type) - - if reloc_type is None: - log.error( - f"Reloc type '{reloc.reloc_type}' is not valid. Rom address: 0x{rom_address:X}" - ) - - symbols.spim_context.addGlobalReloc( - rom_address, reloc_type, reloc.symbol_name, addend=reloc.addend - ) diff --git a/tools/splat/util/symbols.py b/tools/splat/util/symbols.py deleted file mode 100644 index 4f5b682..0000000 --- a/tools/splat/util/symbols.py +++ /dev/null @@ -1,669 +0,0 @@ -from dataclasses import dataclass -import re -from typing import Dict, List, Optional, Set, TYPE_CHECKING - -import spimdisasm - -from intervaltree import IntervalTree -from disassembler import disassembler_instance -from pathlib import Path - -# circular import -if TYPE_CHECKING: - from segtypes.segment import Segment - -from util import log, options, progress_bar - -all_symbols: List["Symbol"] = [] -all_symbols_dict: Dict[int, List["Symbol"]] = {} -all_symbols_ranges = IntervalTree() -ignored_addresses: Set[int] = set() -to_mark_as_defined: Set[str] = set() -appears_after_overlays_syms: List["Symbol"] = [] - -# Initialize a spimdisasm context, used to store symbols and functions -spim_context = spimdisasm.common.Context() - -TRUEY_VALS = ["true", "on", "yes", "y"] -FALSEY_VALS = ["false", "off", "no", "n"] - -splat_sym_types = {"func", "jtbl", "jtbl_label", "label"} - - -def check_valid_type(typename: str) -> bool: - if typename[0].isupper(): - return True - - if typename in splat_sym_types: - return True - - if typename in disassembler_instance.get_instance().known_types(): - return True - - return False - - -def is_truey(str: str) -> bool: - return str.lower() in TRUEY_VALS - - -def is_falsey(str: str) -> bool: - return str.lower() in FALSEY_VALS - - -def add_symbol(sym: "Symbol"): - all_symbols.append(sym) - if sym.vram_start is not None: - if sym.vram_start not in all_symbols_dict: - all_symbols_dict[sym.vram_start] = [] - all_symbols_dict[sym.vram_start].append(sym) - - # For larger symbols, add their ranges to interval trees for faster lookup - if sym.size > 4: - all_symbols_ranges.addi(sym.vram_start, sym.vram_end, sym) - - -def to_cname(symbol_name: str) -> str: - symbol_name = re.sub(r"[^0-9a-zA-Z_]", "_", symbol_name) - - if symbol_name[0] in ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]: - symbol_name = "_" + symbol_name - - return symbol_name - - -def handle_sym_addrs( - path: Path, sym_addrs_lines: List[str], all_segments: "List[Segment]" -): - def get_seg_for_name(name: str) -> Optional["Segment"]: - for segment in all_segments: - if segment.name == name: - return segment - return None - - def get_seg_for_rom(rom: int) -> Optional["Segment"]: - for segment in all_segments: - if segment.contains_rom(rom): - return segment - return None - - prog_bar = progress_bar.get_progress_bar(sym_addrs_lines) - prog_bar.set_description(f"Loading symbols ({path.stem})") - line: str - for line_num, line in enumerate(prog_bar): - line = line.strip() - if not line == "" and not line.startswith("//"): - comment_loc = line.find("//") - line_main = line - line_ext = "" - - if comment_loc != -1: - line_ext = line[comment_loc + 2 :].strip() - line_main = line[:comment_loc].strip() - - try: - line_split = line_main.split("=") - name = line_split[0].strip() - addr = int(line_split[1].strip()[:-1], 0) - except: - log.parsing_error_preamble(path, line_num, line) - log.write("Line should be of the form") - log.write("<function_name> = <address> // attr0:val0 attr1:val1 [...]") - log.write("with <address> in hex preceded by 0x, or dec") - log.write("") - raise - - sym = Symbol(addr, given_name=name) - - ignore_sym = False - if line_ext: - for info in line_ext.split(" "): - if ":" in info: - if info.count(":") > 1: - log.parsing_error_preamble(path, line_num, line) - log.write(f"Too many ':'s in '{info}'") - log.error("") - - attr_name, attr_val = info.split(":") - if attr_name == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute name in '{info}', is there extra whitespace?" - ) - log.error("") - if attr_val == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute value in '{info}', is there extra whitespace?" - ) - log.error("") - - # Non-Boolean attributes - try: - if attr_name == "type": - if not check_valid_type(attr_val): - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Unrecognized symbol type in '{info}', it should be one of" - ) - log.write( - [ - *splat_sym_types, - *spimdisasm.common.gKnownTypes, - ] - ) - log.write( - "You may use a custom type that starts with a capital letter" - ) - log.error("") - type = attr_val - sym.type = type - continue - if attr_name == "size": - size = int(attr_val, 0) - sym.given_size = size - continue - if attr_name == "rom": - rom_addr = int(attr_val, 0) - sym.rom = rom_addr - continue - if attr_name == "segment": - seg = get_seg_for_name(attr_val) - if seg is None: - log.parsing_error_preamble(path, line_num, line) - log.write(f"Cannot find segment '{attr_val}'") - log.error("") - else: - # Add segment to symbol - sym.segment = seg - continue - if attr_name == "name_end": - sym.given_name_end = attr_val - continue - if attr_name == "appears_after_overlays_addr": - sym.appears_after_overlays_addr = int(attr_val, 0) - appears_after_overlays_syms.append(sym) - continue - except: - log.parsing_error_preamble(path, line_num, line) - log.write( - f"value of attribute '{attr_name}' could not be read:" - ) - log.write("") - raise - - # Boolean attributes - tf_val = ( - True - if is_truey(attr_val) - else False - if is_falsey(attr_val) - else None - ) - if tf_val is None: - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of" - ) - log.write([*TRUEY_VALS, *FALSEY_VALS]) - log.error("") - else: - if attr_name == "dead": - sym.dead = tf_val - continue - if attr_name == "defined": - sym.defined = tf_val - continue - if attr_name == "extract": - sym.extract = tf_val - continue - if attr_name == "ignore": - ignore_sym = tf_val - continue - if attr_name == "force_migration": - sym.force_migration = tf_val - continue - if attr_name == "force_not_migration": - sym.force_not_migration = tf_val - continue - if attr_name == "allow_addend": - sym.allow_addend = tf_val - continue - if attr_name == "dont_allow_addend": - sym.dont_allow_addend = tf_val - continue - - if ignore_sym: - if sym.given_size is None or sym.given_size == 0: - ignored_addresses.add(sym.vram_start) - else: - spim_context.addBannedSymbolRangeBySize( - sym.vram_start, sym.given_size - ) - - continue - - if sym.segment is None and sym.rom is not None: - sym.segment = get_seg_for_rom(sym.rom) - - if sym.segment: - sym.segment.add_symbol(sym) - - sym.user_declared = True - add_symbol(sym) - - -def initialize(all_segments: "List[Segment]"): - global all_symbols - global all_symbols_dict - global all_symbols_ranges - - all_symbols = [] - all_symbols_dict = {} - all_symbols_ranges = IntervalTree() - - # Manual list of func name / addrs - for path in options.opts.symbol_addrs_paths: - if path.exists(): - with open(path) as f: - sym_addrs_lines = f.readlines() - handle_sym_addrs(path, sym_addrs_lines, all_segments) - - -def initialize_spim_context(all_segments: "List[Segment]") -> None: - global_vrom_start = None - global_vrom_end = None - global_vram_start = None - global_vram_end = None - overlay_segments: Set[spimdisasm.common.SymbolsSegment] = set() - - spim_context.bannedSymbols |= ignored_addresses - - from segtypes.common.code import CommonSegCode - - for segment in all_segments: - if not isinstance(segment, CommonSegCode): - # We only care about the VRAMs of code segments - continue - - if segment.special_vram_segment: - # Special segments which should not be accounted in the global VRAM calculation, like N64's IPL3 - continue - - if ( - not isinstance(segment.vram_start, int) - or not isinstance(segment.vram_end, int) - or not isinstance(segment.rom_start, int) - or not isinstance(segment.rom_end, int) - ): - continue - - ram_id = segment.get_exclusive_ram_id() - - if ram_id is None: - if global_vram_start is None: - global_vram_start = segment.vram_start - elif segment.vram_start < global_vram_start: - global_vram_start = segment.vram_start - - if global_vram_end is None: - global_vram_end = segment.vram_end - elif global_vram_end < segment.vram_end: - global_vram_end = segment.vram_end - - if global_vrom_start is None: - global_vrom_start = segment.rom_start - elif segment.rom_start < global_vrom_start: - global_vrom_start = segment.rom_start - - if global_vrom_end is None: - global_vrom_end = segment.rom_end - elif global_vrom_end < segment.rom_end: - global_vrom_end = segment.rom_end - - else: - spim_segment = spim_context.addOverlaySegment( - ram_id, - segment.rom_start, - segment.rom_end, - segment.vram_start, - segment.vram_end, - ) - # Add the segment-specific symbols first - for symbols_list in segment.seg_symbols.values(): - for sym in symbols_list: - add_symbol_to_spim_segment(spim_segment, sym) - - overlay_segments.add(spim_segment) - - if ( - global_vram_start is not None - and global_vram_end is not None - and global_vrom_start is not None - and global_vrom_end is not None - ): - spim_context.changeGlobalSegmentRanges( - global_vrom_start, global_vrom_end, global_vram_start, global_vram_end - ) - - # Check the vram range of the global segment does not overlap with any overlay segment - for ovl_segment in overlay_segments: - assert ( - ovl_segment.vramStart <= ovl_segment.vramEnd - ), f"{ovl_segment.vramStart:X} {ovl_segment.vramEnd:X}" - if ( - ovl_segment.vramEnd > global_vram_start - and global_vram_end > ovl_segment.vramStart - ): - log.write( - f"Warning: the vram range ([0x{ovl_segment.vramStart:X}, 0x{ovl_segment.vramEnd:X}]) of the non-global segment at rom address 0x{ovl_segment.vromStart:X} overlaps with the global vram range ([0x{global_vram_start:X}, 0x{global_vram_end:X}])", - status="warn", - ) - - # pass the global symbols to spimdisasm - for segment in all_segments: - if not isinstance(segment, CommonSegCode): - # We only care about the VRAMs of code segments - continue - - ram_id = segment.get_exclusive_ram_id() - if ram_id is not None: - continue - - for symbols_list in segment.seg_symbols.values(): - for sym in symbols_list: - add_symbol_to_spim_segment(spim_context.globalSegment, sym) - - -def add_symbol_to_spim_segment( - segment: spimdisasm.common.SymbolsSegment, sym: "Symbol" -) -> spimdisasm.common.ContextSymbol: - if sym.type == "func": - context_sym = segment.addFunction( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - elif sym.type == "jtbl": - context_sym = segment.addJumpTable( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - elif sym.type == "jtbl_label": - context_sym = segment.addJumpTableLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - elif sym.type == "label": - context_sym = segment.addBranchLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - else: - context_sym = segment.addSymbol( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - if sym.type is not None: - context_sym.type = sym.type - - if sym.user_declared: - context_sym.isUserDeclared = True - if sym.defined: - context_sym.isDefined = True - if sym.rom is not None: - context_sym.vromAddress = sym.rom - if sym.given_size is not None: - context_sym.size = sym.size - if sym.force_migration: - context_sym.forceMigration = True - if sym.force_not_migration: - context_sym.forceNotMigration = True - if sym.allow_addend: - context_sym.allowedToReferenceAddends = True - if sym.dont_allow_addend: - context_sym.notAllowedToReferenceAddends = True - context_sym.setNameGetCallbackIfUnset(lambda _: sym.name) - if sym.given_name_end: - context_sym.nameEnd = sym.given_name_end - - return context_sym - - -def add_symbol_to_spim_section( - section: spimdisasm.mips.sections.SectionBase, sym: "Symbol" -) -> spimdisasm.common.ContextSymbol: - if sym.type == "func": - context_sym = section.addFunction( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "jtbl": - context_sym = section.addJumpTable( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "jtbl_label": - context_sym = section.addJumpTableLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "label": - context_sym = section.addBranchLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - else: - context_sym = section.addSymbol( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - if sym.type is not None: - context_sym.type = sym.type - - if sym.user_declared: - context_sym.isUserDeclared = True - if sym.defined: - context_sym.isDefined = True - if sym.rom is not None: - context_sym.vromAddress = sym.rom - if sym.given_size is not None: - context_sym.size = sym.size - if sym.force_migration: - context_sym.forceMigration = True - if sym.force_not_migration: - context_sym.forceNotMigration = True - context_sym.setNameGetCallbackIfUnset(lambda _: sym.name) - if sym.given_name_end: - context_sym.nameEnd = sym.given_name_end - - return context_sym - - -def create_symbol_from_spim_symbol( - segment: "Segment", context_sym: spimdisasm.common.ContextSymbol -) -> "Symbol": - in_segment = False - - sym_type = None - if context_sym.type == spimdisasm.common.SymbolSpecialType.jumptable: - in_segment = True - sym_type = "jtbl" - elif context_sym.type == spimdisasm.common.SymbolSpecialType.function: - sym_type = "func" - elif context_sym.type == spimdisasm.common.SymbolSpecialType.branchlabel: - in_segment = True - sym_type = "label" - elif context_sym.type == spimdisasm.common.SymbolSpecialType.jumptablelabel: - in_segment = True - sym_type = "jtbl_label" - - if not in_segment: - if ( - context_sym.overlayCategory is None - and segment.get_exclusive_ram_id() is None - ): - in_segment = segment.contains_vram(context_sym.vram) - elif context_sym.overlayCategory == segment.get_exclusive_ram_id(): - if context_sym.vromAddress is not None: - in_segment = segment.contains_rom(context_sym.vromAddress) - else: - in_segment = segment.contains_vram(context_sym.vram) - - sym = segment.create_symbol( - context_sym.vram, in_segment, type=sym_type, reference=True - ) - - if sym.given_name is None and context_sym.name is not None: - sym.given_name = context_sym.name - - # To keep the symbol name in sync between splat and spimdisasm - context_sym.setNameGetCallback(lambda _: sym.name) - - if context_sym.size is not None: - sym.given_size = context_sym.getSize() - if context_sym.vromAddress is not None: - sym.rom = context_sym.getVrom() - if context_sym.isDefined: - sym.defined = True - if context_sym.referenceCounter > 0: - sym.referenced = True - - return sym - - -def mark_c_funcs_as_defined(): - for symbol in all_symbols: - if len(to_mark_as_defined) == 0: - return - sym_name = symbol.name - if sym_name in to_mark_as_defined: - symbol.defined = True - to_mark_as_defined.remove(sym_name) - - -@dataclass -class Symbol: - vram_start: int - - given_name: Optional[str] = None - given_name_end: Optional[str] = None - rom: Optional[int] = None - type: Optional[str] = None - given_size: Optional[int] = None - segment: Optional["Segment"] = None - - defined: bool = False - referenced: bool = False - dead: bool = False - extract: bool = True - user_declared: bool = False - - force_migration: bool = False - force_not_migration: bool = False - - allow_addend: bool = False - dont_allow_addend: bool = False - - linker_section: Optional[str] = None - - _generated_default_name: Optional[str] = None - _last_type: Optional[str] = None - - appears_after_overlays_addr: Optional[int] = None - - def __str__(self): - return self.name - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Symbol): - return False - return self.vram_start == other.vram_start and self.segment == other.segment - - # https://stackoverflow.com/a/56915493/6292472 - def __hash__(self): - return hash((self.vram_start, self.segment)) - - def format_name(self, format: str) -> str: - ret = format - - ret = ret.replace("$VRAM", f"{self.vram_start:08X}") - - if "$ROM" in ret: - if not isinstance(self.rom, int): - log.error( - f"Attempting to rom-name a symbol with no ROM address: {self.vram_start:X} typed {self.type}" - ) - ret = ret.replace("$ROM", f"{self.rom:X}") - - if "$SEG" in ret: - if self.segment is None: - # This probably is fine - we can't expect every symbol to have a segment. Fall back to just the ram address - return f"{self.vram_start:X}" - assert self.segment is not None - ret = ret.replace("$SEG", self.segment.name) - - return ret - - @property - def default_name(self) -> str: - if self._generated_default_name is not None: - if self.type == self._last_type: - return self._generated_default_name - - if self.segment: - if isinstance(self.rom, int): - suffix = self.format_name(self.segment.symbol_name_format) - else: - suffix = self.format_name(self.segment.symbol_name_format_no_rom) - else: - if isinstance(self.rom, int): - suffix = self.format_name(options.opts.symbol_name_format) - else: - suffix = self.format_name(options.opts.symbol_name_format_no_rom) - - if self.type == "func": - prefix = "func" - elif self.type == "jtbl": - prefix = "jtbl" - elif self.type in {"jtbl_label", "label"}: - return f".L{suffix}" - else: - prefix = "D" - - self._last_type = self.type - self._generated_default_name = f"{prefix}_{suffix}" - return self._generated_default_name - - @property - def rom_end(self): - return None if not self.rom else self.rom + self.size - - @property - def vram_end(self): - return self.vram_start + self.size - - @property - def name(self) -> str: - return self.given_name if self.given_name else self.default_name - - @property - def size(self) -> int: - if self.given_size is not None: - return self.given_size - return 4 - - def contains_vram(self, offset): - return offset >= self.vram_start and offset < self.vram_end - - def contains_rom(self, offset): - return offset >= self.rom and offset < self.rom_end - - -def get_all_symbols(): - global all_symbols - return all_symbols - - -def reset_symbols(): - global all_symbols - global all_symbols_dict - global all_symbols_ranges - global ignored_addresses - global to_mark_as_defined - global appears_after_overlays_syms - all_symbols = [] - all_symbols_dict = {} - all_symbols_ranges = IntervalTree() - ignored_addresses = set() - to_mark_as_defined = set() - appears_after_overlays_syms = [] |