diff options
51 files changed, 4017 insertions, 229 deletions
diff --git a/include/macro.inc b/include/macro.inc index 1ee4aa4..e55380f 100644 --- a/include/macro.inc +++ b/include/macro.inc @@ -1,18 +1,63 @@ -.macro glabel label - .global \label +# Evaluate this file only once in case it's included more than once +.ifndef _MACRO_INC_GUARD +.internal _MACRO_INC_GUARD +.set _MACRO_INC_GUARD, 1 + +# A function symbol. +.macro glabel label, visibility=global + .\visibility \label .type \label, @function \label: + .ent \label .endm -.macro dlabel label - .global \label - \label: +# The end of a function symbol. +.macro endlabel label + .size \label, . - \label + .end \label .endm -.macro jlabel label - .global \label +# An alternative entry to a function. +.macro alabel label, visibility=global + .\visibility \label .type \label, @function \label: + .aent \label +.endm + +# A label referenced by an error handler table. +.macro ehlabel label, visibility=global + .\visibility \label + \label: +.endm + + +# A label referenced by a jumptable. +.macro jlabel label, visibility=global + .\visibility \label + \label: +.endm + + +# A data symbol. +.macro dlabel label, visibility=global + .\visibility \label + .type \label, @object + \label: +.endm + +# End of a data symbol. +.macro enddlabel label + .size \label, . - \label +.endm + + +# Label to signal the symbol haven't been matched yet. +.macro nonmatching label, size=1 + .global \label\().NON_MATCHING + .type \label\().NON_MATCHING, @object + .size \label\().NON_MATCHING, \size + \label\().NON_MATCHING: .endm # COP0 register aliases @@ -85,3 +130,5 @@ .set $fs4f, $f29 .set $fs5, $f30 .set $fs5f, $f31 + +.endif diff --git a/requirements.txt b/requirements.txt index accf75f..3d902cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ colorama libyaz0 ipl3checksum>=1.2.0,<2.0.0 -spimdisasm>=1.16.3,<2.0.0 +spimdisasm>=1.36.0,<2.0.0 diff --git a/tools/asm-processor/.github/workflows/release.yml b/tools/asm-processor/.github/workflows/release.yml new file mode 100644 index 0000000..dc9d1e7 --- /dev/null +++ b/tools/asm-processor/.github/workflows/release.yml @@ -0,0 +1,291 @@ +# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/ +# +# Copyright 2022-2024, axodotdev +# SPDX-License-Identifier: MIT or Apache-2.0 +# +# CI that: +# +# * checks for a Git Tag that looks like a release +# * builds artifacts with dist (archives, installers, hashes) +# * uploads those artifacts to temporary workflow zip +# * on success, uploads the artifacts to a GitHub Release +# +# Note that the GitHub Release will be created with a generated +# title/body based on your changelogs. + +name: Release +permissions: + "contents": "write" + +# This task will run whenever you push a git tag that looks like a version +# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. +# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where +# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION +# must be a Cargo-style SemVer Version (must have at least major.minor.patch). +# +# If PACKAGE_NAME is specified, then the announcement will be for that +# package (erroring out if it doesn't have the given version or isn't dist-able). +# +# If PACKAGE_NAME isn't specified, then the announcement will be for all +# (dist-able) packages in the workspace with that version (this mode is +# intended for workspaces with only one dist-able package, or with all dist-able +# packages versioned/released in lockstep). +# +# If you push multiple tags at once, separate instances of this workflow will +# spin up, creating an independent announcement for each one. However, GitHub +# will hard limit this to 3 tags per commit, as it will assume more tags is a +# mistake. +# +# If there's a prerelease-style suffix to the version, then the release(s) +# will be marked as a prerelease. +on: + pull_request: + push: + tags: + - '**[0-9]+.[0-9]+.[0-9]+*' + +jobs: + # Run 'dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: "ubuntu-20.04" + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ !github.event.pull_request && github.ref_name || '' }} + tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} + publishing: ${{ !github.event.pull_request }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install dist + # we specify bash to get pipefail; it guards against the `curl` command + # failing. otherwise `sh` won't catch that `curl` returned non-0 + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.28.0/cargo-dist-installer.sh | sh" + - name: Cache dist + uses: actions/upload-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/dist + # sure would be cool if github gave us proper conditionals... + # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible + # functionality based on whether this is a pull_request, and whether it's from a fork. + # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* + # but also really annoying to build CI around when it needs secrets to work right.) + - id: plan + run: | + dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "dist ran successfully" + cat plan-dist-manifest.json + echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v4 + with: + name: artifacts-plan-dist-manifest + path: plan-dist-manifest.json + + # Build and packages all the platform-specific things + build-local-artifacts: + name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) + # Let the initial task tell us to not run (currently very blunt) + needs: + - plan + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + strategy: + fail-fast: false + # Target platforms/runners are computed by dist in create-release. + # Each member of the matrix has the following arguments: + # + # - runner: the github runner + # - dist-args: cli flags to pass to dist + # - install-dist: expression to run to install dist on the runner + # + # Typically there will be: + # - 1 "global" task that builds universal installers + # - N "local" tasks that build each platform's binaries and platform-specific installers + matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container && matrix.container.image || null }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + steps: + - name: enable windows longpaths + run: | + git config --global core.longpaths true + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install Rust non-interactively if not already installed + if: ${{ matrix.container }} + run: | + if ! command -v cargo > /dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - name: Install dist + run: ${{ matrix.install_dist.run }} + # Get the dist-manifest + - name: Fetch local artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - name: Build artifacts + run: | + # Actually do builds and make zips and whatnot + dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "dist ran successfully" + - id: cargo-dist + name: Post-build + # We force bash here just because github makes it really hard to get values up + # to "real" actions without writing to env-vars, and writing to env-vars has + # inconsistent syntax between shell and powershell. + shell: bash + run: | + # Parse out what we just built and upload it to scratch storage + echo "paths<<EOF" >> "$GITHUB_OUTPUT" + dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v4 + with: + name: artifacts-build-local-${{ join(matrix.targets, '_') }} + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + # Build and package all the platform-agnostic(ish) things + build-global-artifacts: + needs: + - plan + - build-local-artifacts + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + # Get all the local artifacts for the global tasks to use (for e.g. checksums) + - name: Fetch local artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: cargo-dist + shell: bash + run: | + dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<<EOF" >> "$GITHUB_OUTPUT" + jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v4 + with: + name: artifacts-build-global + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + # Determines if we should publish/announce + host: + needs: + - plan + - build-local-artifacts + - build-global-artifacts + # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-20.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + # Fetch artifacts from scratch-storage + - name: Fetch artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: host + shell: bash + run: | + dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v4 + with: + # Overwrite the previous copy + name: artifacts-dist-manifest + path: dist-manifest.json + # Create a GitHub Release while uploading all files to it + - name: "Download GitHub Artifacts" + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: artifacts + merge-multiple: true + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create GitHub Release + env: + PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}" + ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}" + ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}" + RELEASE_COMMIT: "${{ github.sha }}" + run: | + # Write and read notes from a file to avoid quoting breaking things + echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt + + gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/* + + announce: + needs: + - plan + - host + # use "always() && ..." to allow us to wait for all publish jobs while + # still allowing individual publish jobs to skip themselves (for prereleases). + # "host" however must run to completion, no skipping allowed! + if: ${{ always() && needs.host.result == 'success' }} + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive diff --git a/tools/asm-processor/.github/workflows/test.yml b/tools/asm-processor/.github/workflows/test.yml new file mode 100644 index 0000000..f2071b1 --- /dev/null +++ b/tools/asm-processor/.github/workflows/test.yml @@ -0,0 +1,31 @@ +name: Build, Lint, and Test +on: + push: + pull_request: +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: sudo apt-get install binutils-mips-linux-gnu -y + - uses: actions/checkout@v4 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Build + run: cargo build --release + working-directory: rust + - name: Format + run: cargo fmt -- --check + working-directory: rust + - name: Check + run: cargo check + working-directory: rust + - name: Test + run: | + wget https://github.com/decompals/ido-static-recomp/releases/download/v1.2/ido-7.1-recomp-linux.tar.gz + tar -xf ido-7.1-recomp-linux.tar.gz + export MIPS_CC=./cc + ./run-tests.sh diff --git a/tools/asm-processor/.gitignore b/tools/asm-processor/.gitignore index cc5bba4..d47a5f2 100644 --- a/tools/asm-processor/.gitignore +++ b/tools/asm-processor/.gitignore @@ -1,2 +1,4 @@ *.o *.py[cod] +rust/target/ +.vscode/ diff --git a/tools/asm-processor/.gitrepo b/tools/asm-processor/.gitrepo index fd4c97a..4b3a8d1 100644 --- a/tools/asm-processor/.gitrepo +++ b/tools/asm-processor/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = git@github.com:simonlindholm/asm-processor.git branch = main - commit = 6dd4d3adcbfb3ee607b9a1e2bba3502d27217d44 - parent = 39a2e453bb882b2498c7c4aa9073a9f85d048f0c + commit = 5c3984fc9cb964ba995b2d455f71a8f0d6a2f155 + parent = 8c6ae625aef6d07741cd8925f422f43ae24b198c method = merge - cmdver = 0.4.3 + cmdver = 0.4.9 diff --git a/tools/asm-processor/README.md b/tools/asm-processor/README.md index 009147c..89a0b66 100644 --- a/tools/asm-processor/README.md +++ b/tools/asm-processor/README.md @@ -2,8 +2,33 @@ Pre-process .c files and post-process .o files to enable embedding MIPS assembly into IDO-compiled C. +This repository contains both the original Python implementation and rewrite in Rust that is designed to be 1:1 behavorially equivalent with the existing Python version, but faster. + +## Installation + +Most projects traditionally have included the `asm-processor` repo as a [submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules), [subrepo](https://github.com/ingydotnet/git-subrepo), or plain copy inside their project. +This is recommended, as it ensures consistency for all project users. + +### Rust +After vendoring this repo into your repository, you will want to add a step to your project setup procedure that builds the asm-processor binary with the [Rust toolchain](https://www.rust-lang.org/tools/install). +Presuming this repo is available at `tools/asm-processor/`, the following command can be run to build the project: + +``` +cargo build --release --manifest-path tools/asm-processor/rust/Cargo.toml +``` + +This will generate the executable at `tools/asm-processor/rust/target/release/asm-processor`. The build system for your project can then be configured to run `asm-processor` from this location. + +If you prefer not to build the project yourself or require downstream users to do so, we also provide release binaries that can either be downloaded at build time or included directly in your project's repo. + +### Python +Simply vendor this repo into your repository as described above and use `build.py`. + + ## Usage +The Python `build.py` script and Rust `asm-processor` binary accept the same syntax and command line flags. If using the Rust implementation, substitute `build.py` with `asm-processor` in the below guide. + Let's say you have a file compiled with `-g` on the IDO compiler, that looks like this: ```c float func4(void) { @@ -35,13 +60,15 @@ nop ) ``` -To compile the file, run `python3 build.py $CC -- $AS $ASFLAGS -- $CFLAGS -o out.o in.c`, where $CC points to an IDO binary (5.3/7.1 and recomp/qemu all supported), $AS is e.g. `mips-linux-gnu-as`, $ASFLAGS e.g. `-march=vr4300 -mabi=32` and $CFLAGS e.g. `-Wab,-r4300_mul -non_shared -G 0 -Xcpluscomm -g`. build.py may be customized as needed. +To compile the file, run `build.py $CC -- $AS $ASFLAGS -- $CFLAGS -o out.o in.c`, where $CC points to an IDO binary (5.3/7.1 and recomp/qemu all supported), $AS is e.g. `mips-linux-gnu-as`, $ASFLAGS e.g. `-march=vr4300 -mabi=32` and $CFLAGS e.g. `-Wab,-r4300_mul -non_shared -G 0 -Xcpluscomm -g`. -In addition to an .o file, build.py also generates a .d file with Makefile dependencies for .s files referenced by the input .c file. -This functionality may be removed if not needed. +In addition to an .o file, asm-processor also generates a .d file with Makefile dependencies for .s files referenced by the input .c file. +This functionality can be disabled by passing the `--no-dep-file` flag. Reading assembly from file is also supported, by either `GLOBAL_ASM("file.s")` or `#pragma GLOBAL_ASM("file.s")`. +For compatibility with common GCC macros, `INCLUDE_ASM("folder", functionname);` and `INCLUDE_RODATA("folder", functionname);` are also allowed, and equivalent to `GLOBAL_ASM("folder/functionname.s")`. + ### What is supported? `.text`, `.data`, `.bss` and `.rodata` sections, `.word`/`.incbin`, `.ascii`/`.asciz`, and `-g`, `-g3`, `-O1`, `-O2`, `-framepointer` and `-mips1`/`-mips2` flags to the IDO compiler. @@ -103,7 +130,7 @@ For example if asm-processor is cloned in the same directory as [ido static reco MIPS_CC=../ido-static-recomp/build/7.1/out/cc ./run-tests.sh ``` -Or using [qemu-irix](https://github.com/zeldaret/oot/releases/tag/0.1q) (don't forget `chmod u+x qemu-irix`) to emulate IDO: +Or using qemu-irix (don't forget `chmod u+x qemu-irix`) to emulate IDO: ```sh MIPS_CC='./qemu-irix -silent -L ../ido-static-recomp/ido/7.1/ ../ido-static-recomp/ido/7.1/usr/bin/cc' ./run-tests.sh diff --git a/tools/asm-processor/add-test.sh b/tools/asm-processor/add-test.sh index 708548e..7aef817 100755 --- a/tools/asm-processor/add-test.sh +++ b/tools/asm-processor/add-test.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +WD=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + for A in "$@"; do OBJDUMPFLAGS="-srt" - ./compile-test.sh "$A" && mips-linux-gnu-objdump $OBJDUMPFLAGS "${A%.*}.o" > "${A%.*}.objdump" + "$WD/compile-test.sh" "$A" python && mips-linux-gnu-objdump $OBJDUMPFLAGS "${A%.*}.o" > "${A%.*}.objdump" done diff --git a/tools/asm-processor/asm_processor.py b/tools/asm-processor/asm_processor.py index 23f6102..63103c6 100644 --- a/tools/asm-processor/asm_processor.py +++ b/tools/asm-processor/asm_processor.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import argparse -import tempfile -import struct -import copy -import sys -import re -import os from collections import namedtuple from io import StringIO +import os +from pathlib import Path +import re +import struct +import sys +import tempfile MAX_FN_SIZE = 100 SLOW_CHECKS = False @@ -82,7 +82,14 @@ R_MIPS_HI16 = 5 R_MIPS_LO16 = 6 MIPS_DEBUG_ST_STATIC = 2 +MIPS_DEBUG_ST_PROC = 6 +MIPS_DEBUG_ST_BLOCK = 7 +MIPS_DEBUG_ST_END = 8 +MIPS_DEBUG_ST_FILE = 11 MIPS_DEBUG_ST_STATIC_PROC = 14 +MIPS_DEBUG_ST_STRUCT = 26 +MIPS_DEBUG_ST_UNION = 27 +MIPS_DEBUG_ST_ENUM = 28 class ElfFormat: @@ -97,6 +104,17 @@ class ElfFormat: return struct.unpack(self.struct_char + fmt, data) +class Encoding: + def __init__(self, encoding): + self.encoding = encoding + self.is_euc_jp = encoding.lower().replace("_", "").replace("-", "") == "eucjp" + + def encode(self, s): + if self.is_euc_jp: + s = s.replace("～", "〜") + return s.encode(self.encoding) + + class ElfHeader: """ typedef struct { @@ -151,10 +169,10 @@ class Symbol: self.fmt = fmt self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx = fmt.unpack('IIIBBH', data) assert self.st_shndx != SHN_XINDEX, "too many sections (SHN_XINDEX not supported)" - self.bind = st_info >> 4 - self.type = st_info & 15 + self.st_bind = st_info >> 4 + self.st_type = st_info & 0xf self.name = name if name is not None else strtab.lookup_str(self.st_name) - self.visibility = self.st_other & 3 + self.st_visibility = self.st_other & 3 @staticmethod def from_parts(fmt, st_name, st_value, st_size, st_info, st_other, st_shndx, strtab, name): @@ -162,7 +180,7 @@ class Symbol: return Symbol(fmt, header, strtab, name) def to_bin(self): - st_info = (self.bind << 4) | self.type + st_info = (self.st_bind << 4) | self.st_type return self.fmt.pack('IIIBBH', self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx) @@ -171,18 +189,18 @@ class Relocation: self.fmt = fmt self.sh_type = sh_type if sh_type == SHT_REL: - self.r_offset, self.r_info = fmt.unpack('II', data) + self.r_offset, r_info = fmt.unpack('II', data) else: - self.r_offset, self.r_info, self.r_addend = fmt.unpack('III', data) - self.sym_index = self.r_info >> 8 - self.rel_type = self.r_info & 0xff + self.r_offset, r_info, self.r_addend = fmt.unpack('III', data) + self.sym_index = r_info >> 8 + self.rel_type = r_info & 0xff def to_bin(self): - self.r_info = (self.sym_index << 8) | self.rel_type + r_info = (self.sym_index << 8) | self.rel_type if self.sh_type == SHT_REL: - return self.fmt.pack('II', self.r_offset, self.r_info) + return self.fmt.pack('II', self.r_offset, r_info) else: - return self.fmt.pack('III', self.r_offset, self.r_info, self.r_addend) + return self.fmt.pack('III', self.r_offset, r_info, self.r_addend) class Section: @@ -239,36 +257,6 @@ class Section: self.sh_size = len(self.data) return self.fmt.pack('IIIIIIIIII', self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize) - def late_init(self, sections): - if self.sh_type == SHT_SYMTAB: - self.init_symbols(sections) - elif self.is_rel(): - self.rel_target = sections[self.sh_info] - self.rel_target.relocated_by.append(self) - self.init_relocs() - - def find_symbol(self, name): - assert self.sh_type == SHT_SYMTAB - for s in self.symbol_entries: - if s.name == name: - return (s.st_shndx, s.st_value) - return None - - def find_symbol_in_section(self, name, section): - pos = self.find_symbol(name) - assert pos is not None - assert pos[0] == section.index - return pos[1] - - def init_symbols(self, sections): - assert self.sh_type == SHT_SYMTAB - assert self.sh_entsize == 16 - self.strtab = sections[self.sh_link] - entries = [] - for i in range(0, self.sh_size, self.sh_entsize): - entries.append(Symbol(self.fmt, self.data[i:i+self.sh_entsize], self.strtab)) - self.symbol_entries = entries - def init_relocs(self): assert self.is_rel() entries = [] @@ -276,14 +264,6 @@ class Section: entries.append(Relocation(self.fmt, self.data[i:i+self.sh_entsize], self.sh_type)) self.relocations = entries - def local_symbols(self): - assert self.sh_type == SHT_SYMTAB - return self.symbol_entries[:self.sh_info] - - def global_symbols(self): - assert self.sh_type == SHT_SYMTAB - return self.symbol_entries[self.sh_info:] - def relocate_mdebug(self, original_offset): assert self.sh_type == SHT_MIPS_DEBUG new_data = bytearray(self.data) @@ -300,17 +280,17 @@ class Section: assert hdrr_magic == 0x7009, "Invalid magic value for .mdebug symbolic header" - hdrr_cbLineOffset += shift_by - hdrr_cbDnOffset += shift_by - hdrr_cbPdOffset += shift_by - hdrr_cbSymOffset += shift_by - hdrr_cbOptOffset += shift_by - hdrr_cbAuxOffset += shift_by - hdrr_cbSsOffset += shift_by - hdrr_cbSsExtOffset += shift_by - hdrr_cbFdOffset += shift_by - hdrr_cbRfdOffset += shift_by - hdrr_cbExtOffset += shift_by + if hdrr_cbLine: hdrr_cbLineOffset += shift_by + if hdrr_idnMax: hdrr_cbDnOffset += shift_by + if hdrr_ipdMax: hdrr_cbPdOffset += shift_by + if hdrr_isymMax: hdrr_cbSymOffset += shift_by + if hdrr_ioptMax: hdrr_cbOptOffset += shift_by + if hdrr_iauxMax: hdrr_cbAuxOffset += shift_by + if hdrr_issMax: hdrr_cbSsOffset += shift_by + if hdrr_issExtMax: hdrr_cbSsExtOffset += shift_by + if hdrr_ifdMax: hdrr_cbFdOffset += shift_by + if hdrr_crfd: hdrr_cbRfdOffset += shift_by + if hdrr_iextMax: hdrr_cbExtOffset += shift_by new_data[0:0x60] = self.fmt.pack("HHIIIIIIIIIIIIIIIIIIIIIII", hdrr_magic, hdrr_vstamp, hdrr_ilineMax, hdrr_cbLine, \ hdrr_cbLineOffset, hdrr_idnMax, hdrr_cbDnOffset, hdrr_ipdMax, \ @@ -346,11 +326,36 @@ class ElfFile: symtab = s assert symtab is not None self.symtab = symtab + self.sym_strtab = self.sections[symtab.sh_link] + self.symbol_entries = ElfFile.init_symbols(symtab, self.sym_strtab) shstr = self.sections[self.elf_header.e_shstrndx] for s in self.sections: s.name = shstr.lookup_str(s.sh_name) - s.late_init(self.sections) + if s.is_rel(): + self.sections[s.sh_info].relocated_by.append(s) + s.init_relocs() + + @staticmethod + def init_symbols(symtab, strtab): + assert symtab.sh_type == SHT_SYMTAB + assert symtab.sh_entsize == 16 + syms = [] + for i in range(0, symtab.sh_size, symtab.sh_entsize): + syms.append(Symbol(symtab.fmt, symtab.data[i:i+symtab.sh_entsize], strtab)) + return syms + + def find_symbol(self, name): + for s in self.symbol_entries: + if s.name == name: + return (s.st_shndx, s.st_value) + return None + + def find_symbol_in_section(self, name, section): + pos = self.find_symbol(name) + assert pos is not None + assert pos[0] == section.index + return pos[1] def find_section(self, name): for s in self.sections: @@ -367,7 +372,6 @@ class ElfFile: index=len(self.sections)) self.sections.append(s) s.name = name - s.late_init(self.sections) return s def drop_mdebug_gptab(self): @@ -523,7 +527,7 @@ class GlobalAsmBlock: raise Failure(message + "\nwithin " + context) def count_quoted_size(self, line, z, real_line, output_enc): - line = line.encode(output_enc).decode('latin1') + line = output_enc.encode(line).decode('latin1') in_quote = False has_comma = True num_parts = 0 @@ -609,7 +613,7 @@ class GlobalAsmBlock: self.text_glabels.append(line.split()[1]) if not line: pass # empty line - elif line.startswith('glabel ') or line.startswith('dlabel ') or line.startswith('jlabel ') or line.startswith('endlabel ') or (' ' not in line and line.endswith(':')): + elif line.startswith('glabel ') or line.startswith('dlabel ') or line.startswith('jlabel ') or line.startswith('alabel ') or line.startswith('endlabel ') or line.startswith('enddlabel ') or line.startswith('nonmatching ') or (' ' not in line and line.endswith(':')): pass # label elif line.startswith('.section') or line in ['.text', '.data', '.rdata', '.rodata', '.bss', '.late_rodata']: # section change @@ -666,9 +670,11 @@ class GlobalAsmBlock: self.add_sized(self.count_quoted_size(line, z, real_line, output_enc), real_line) elif line.startswith('.byte'): self.add_sized(len(line.split(',')), real_line) - elif line.startswith('.half') or line.startswith('.hword'): + elif line.startswith('.half') or line.startswith('.hword') or line.startswith(".short"): self.align2() self.add_sized(2*len(line.split(',')), real_line) + elif line.startswith('.size'): + pass elif line.startswith('.'): # .macro, ... self.fail("asm directive not supported", real_line) @@ -736,7 +742,7 @@ class GlobalAsmBlock: size - i >= jtbl_min_rodata_size and num_instr - len(late_rodata_fn_output) >= jtbl_size + 1): if state.pascal: - cases = " ".join("{}: ;".format(case) for case in range(size - i)) + cases = "\n".join("{}: ;".format(case) for case in range(size - i)) line = "case 0 of " + cases + " otherwise end;" else: cases = " ".join("case {}:".format(case) for case in range(size - i)) @@ -875,7 +881,7 @@ float_regexpr = re.compile(r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?f") def repl_float_hex(m): return str(struct.unpack(">I", struct.pack(">f", float(m.group(0).strip().rstrip("f"))))[0]) -Opts = namedtuple('Opts', ['opt', 'framepointer', 'mips1', 'kpic', 'pascal', 'input_enc', 'output_enc']) +Opts = namedtuple('Opts', ['opt', 'framepointer', 'mips1', 'kpic', 'pascal', 'input_enc', 'output_enc', 'encode_cutscene_data_floats']) def parse_source(f, opts, out_dependencies, print_source=None): if opts.opt in ['O1', 'O2']: @@ -927,8 +933,9 @@ def parse_source(f, opts, out_dependencies, print_source=None): global_asm = None asm_functions = [] + base_fname = f.name output_lines = [ - '#line 1 "' + f.name + '"' + '#line 1 "' + base_fname + '"' ] is_cutscene_data = False @@ -946,27 +953,64 @@ def parse_source(f, opts, out_dependencies, print_source=None): if global_asm is not None: if line.startswith(')'): src, fn = global_asm.finish(state) + if state.pascal: + # Pascal has a 1600-character line length limit, so some + # of the lines we emit may be broken up. Correct for that + # using a #line directive. + src[-1] += '\n#line ' + str(line_no + 1) for i, line2 in enumerate(src): output_lines[start_index + i] = line2 asm_functions.append(fn) global_asm = None else: global_asm.process_line(raw_line, output_enc) - elif line in ['GLOBAL_ASM(', '#pragma GLOBAL_ASM(']: + elif line in ("GLOBAL_ASM(", "#pragma GLOBAL_ASM("): global_asm = GlobalAsmBlock("GLOBAL_ASM block at line " + str(line_no)) start_index = len(output_lines) - elif ((line.startswith('GLOBAL_ASM("') or line.startswith('#pragma GLOBAL_ASM("')) - and line.endswith('")')): - fname = line[line.index('(') + 2 : -2] - out_dependencies.append(fname) - global_asm = GlobalAsmBlock(fname) - with open(fname, encoding=opts.input_enc) as f: + elif ( + (line.startswith('GLOBAL_ASM("') or line.startswith('#pragma GLOBAL_ASM("')) + and line.endswith('")') + ) or ( + (line.startswith('INCLUDE_ASM("') or line.startswith('INCLUDE_RODATA("')) + and '",' in line + and line.endswith(");") + ): + prologue = [] + if line.startswith("INCLUDE_"): + # INCLUDE_ASM("path/to", functionname); + before, after = line.split('",', 1) + fname = before[before.index("(") + 2 :] + "/" + after.strip()[:-2] + ".s" + if line.startswith("INCLUDE_RODATA"): + prologue = [".section .rodata"] + else: + # GLOBAL_ASM("path/to/file.s") + fname = line[line.index("(") + 2 : -2] + ext_global_asm = GlobalAsmBlock(fname) + for line2 in prologue: + ext_global_asm.process_line(line2, output_enc) + try: + f = open(fname, encoding=opts.input_enc) + except FileNotFoundError: + # The GLOBAL_ASM block might be surrounded by an ifdef, so it's + # not clear whether a missing file actually represents a compile + # error. Pass the responsibility for determining that on to the + # compiler by emitting a bad include directive. (IDO treats + # #error as a warning for some reason.) + output_lines[-1] = '#include "GLOBAL_ASM:' + fname + '"' + continue + with f: for line2 in f: - global_asm.process_line(line2.rstrip(), output_enc) - src, fn = global_asm.finish(state) - output_lines[-1] = ''.join(src) + ext_global_asm.process_line(line2.rstrip(), output_enc) + src, fn = ext_global_asm.finish(state) + if state.pascal: + # Pascal has a 1600-character line length limit, so avoid putting + # everything on the same line. + src.append('#line ' + str(line_no + 1)) + output_lines[-1] = '\n'.join(src) + else: + output_lines[-1] = ''.join(src) asm_functions.append(fn) - global_asm = None + out_dependencies.append(fname) elif line == '#pragma asmproc recurse': # C includes qualified as # #pragma asmproc recurse @@ -977,25 +1021,26 @@ def parse_source(f, opts, out_dependencies, print_source=None): # Previous line was a #pragma asmproc recurse is_early_include = False if not line.startswith("#include "): - raise Failure("#pragma asmproc recurse must be followed by an #include ") - fpath = os.path.dirname(f.name) + raise Failure("#pragma asmproc recurse must be followed by an #include") + fpath = os.path.dirname(base_fname) fname = os.path.join(fpath, line[line.index(' ') + 2 : -1]) out_dependencies.append(fname) include_src = StringIO() with open(fname, encoding=opts.input_enc) as include_file: parse_source(include_file, opts, out_dependencies, include_src) - include_src.write('#line ' + str(line_no + 1) + ' "' + f.name + '"') + include_src.write('#line ' + str(line_no + 1) + ' "' + base_fname + '"') output_lines[-1] = include_src.getvalue() include_src.close() else: - # This is a hack to replace all floating-point numbers in an array of a particular type - # (in this case CutsceneData) with their corresponding IEEE-754 hexadecimal representation - if cutscene_data_regexpr.search(line) is not None: - is_cutscene_data = True - elif line.endswith("};"): - is_cutscene_data = False - if is_cutscene_data: - raw_line = re.sub(float_regexpr, repl_float_hex, raw_line) + if opts.encode_cutscene_data_floats: + # This is a hack to replace all floating-point numbers in an array of a particular type + # (in this case CutsceneData) with their corresponding IEEE-754 hexadecimal representation + if cutscene_data_regexpr.search(line) is not None: + is_cutscene_data = True + elif line.endswith("};"): + is_cutscene_data = False + if is_cutscene_data: + raw_line = re.sub(float_regexpr, repl_float_hex, raw_line) output_lines[-1] = raw_line if print_source: @@ -1003,10 +1048,10 @@ def parse_source(f, opts, out_dependencies, print_source=None): for line in output_lines: print_source.write(line + '\n') else: - newline_encoded = "\n".encode(output_enc) + newline_encoded = output_enc.encode("\n") for line in output_lines: try: - line_encoded = line.encode(output_enc) + line_encoded = output_enc.encode(line) except UnicodeEncodeError: print("Failed to encode a line to", output_enc) print("The line:", line) @@ -1055,7 +1100,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d if temp_name is None: continue assert size > 0 - loc = objfile.symtab.find_symbol(temp_name) + loc = objfile.find_symbol(temp_name) if loc is None: ifdefed = True break @@ -1115,7 +1160,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d try: s_file.write(asm_prelude + b'\n') for line in asm: - s_file.write(line.encode(output_enc) + b'\n') + s_file.write(output_enc.encode(line) + b'\n') s_file.close() ret = os.system(assembler + " " + s_name + " -o " + o_name) if ret != 0: @@ -1149,8 +1194,8 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d source = asm_objfile.find_section(sectype) assert source is not None, "didn't find source section: " + sectype for (pos, count, temp_name, fn_desc) in to_copy[sectype]: - loc1 = asm_objfile.symtab.find_symbol_in_section(temp_name + '_asm_start', source) - loc2 = asm_objfile.symtab.find_symbol_in_section(temp_name + '_asm_end', source) + loc1 = asm_objfile.find_symbol_in_section(temp_name + '_asm_start', source) + loc2 = asm_objfile.find_symbol_in_section(temp_name + '_asm_end', source) assert loc1 == pos, "assembly and C files don't line up for section " + sectype + ", " + fn_desc if loc2 - loc1 != count: raise Failure("incorrectly computed size for section " + sectype + ", " + fn_desc + ". If using .double, make sure to provide explicit alignment padding.") @@ -1176,8 +1221,8 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d if any(all_late_rodata_dummy_bytes) or any(all_jtbl_rodata_size): source = asm_objfile.find_section('.late_rodata') target = objfile.find_section('.rodata') - source_pos = asm_objfile.symtab.find_symbol_in_section(late_rodata_source_name_start, source) - source_end = asm_objfile.symtab.find_symbol_in_section(late_rodata_source_name_end, source) + source_pos = asm_objfile.find_symbol_in_section(late_rodata_source_name_start, source) + source_end = asm_objfile.find_symbol_in_section(late_rodata_source_name_end, source) if source_end - source_pos != sum(map(len, all_late_rodata_dummy_bytes)) * 4 + sum(all_jtbl_rodata_size): raise Failure("computed wrong size of .late_rodata") new_data = list(target.data) @@ -1215,27 +1260,26 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d target.data = bytes(new_data) # Merge strtab data. - strtab_adj = len(objfile.symtab.strtab.data) - objfile.symtab.strtab.data += asm_objfile.symtab.strtab.data + strtab_adj = len(objfile.sym_strtab.data) + objfile.sym_strtab.data += asm_objfile.sym_strtab.data # Find relocated symbols relocated_symbols = set() for sectype in SECTIONS + ['.late_rodata']: - for obj in [asm_objfile, objfile]: - sec = obj.find_section(sectype) - if sec is None: - continue - for reltab in sec.relocated_by: - for rel in reltab.relocations: - relocated_symbols.add(obj.symtab.symbol_entries[rel.sym_index]) + sec = asm_objfile.find_section(sectype) + if sec is None: + continue + for reltab in sec.relocated_by: + for rel in reltab.relocations: + relocated_symbols.add(asm_objfile.symbol_entries[rel.sym_index]) # Move over symbols, deleting the temporary function labels. # Skip over new local symbols that aren't relocated against, to # avoid conflicts. - empty_symbol = objfile.symtab.symbol_entries[0] - new_syms = [s for s in objfile.symtab.symbol_entries[1:] if not is_temp_name(s.name)] + empty_symbol = objfile.symbol_entries[0] + new_syms = [s for s in objfile.symbol_entries[1:] if not is_temp_name(s.name)] - for i, s in enumerate(asm_objfile.symtab.symbol_entries): + for i, s in enumerate(asm_objfile.symbol_entries): is_local = (i < asm_objfile.symtab.sh_info) if is_local and s not in relocated_symbols: continue @@ -1255,7 +1299,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d s.st_shndx = objfile_section.index # glabel's aren't marked as functions, making objdump output confusing. Fix that. if s.name in all_text_glabels: - s.type = STT_FUNC + s.st_type = STT_FUNC if s.name in func_sizes: s.st_size = func_sizes[s.name] if section_name == '.late_rodata': @@ -1273,7 +1317,8 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d # Add static symbols from .mdebug, so they can be referred to from GLOBAL_ASM if mdebug_section and convert_statics != "no": - strtab_index = len(objfile.symtab.strtab.data) + static_name_count = {} + strtab_index = len(objfile.sym_strtab.data) new_strtab_data = [] ifd_max, cb_fd_offset = fmt.unpack('II', mdebug_section.data[18*4 : 20*4]) cb_sym_offset, = fmt.unpack('I', mdebug_section.data[9*4 : 10*4]) @@ -1281,20 +1326,28 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d for i in range(ifd_max): offset = cb_fd_offset + 18*4*i iss_base, _, isym_base, csym = fmt.unpack('IIII', objfile.data[offset + 2*4 : offset + 6*4]) + scope_level = 0 for j in range(csym): offset2 = cb_sym_offset + 12 * (isym_base + j) iss, value, st_sc_index = fmt.unpack('III', objfile.data[offset2 : offset2 + 12]) st = (st_sc_index >> 26) sc = (st_sc_index >> 21) & 0x1f - if st in [MIPS_DEBUG_ST_STATIC, MIPS_DEBUG_ST_STATIC_PROC]: + if st in (MIPS_DEBUG_ST_STATIC, MIPS_DEBUG_ST_STATIC_PROC): symbol_name_offset = cb_ss_offset + iss_base + iss symbol_name_offset_end = objfile.data.find(b'\0', symbol_name_offset) assert symbol_name_offset_end != -1 - symbol_name = objfile.data[symbol_name_offset : symbol_name_offset_end + 1] + symbol_name = objfile.data[symbol_name_offset : symbol_name_offset_end] + if scope_level > 1: + # For in-function statics, append an increasing counter to + # the name, to avoid duplicate conflicting symbols. + count = static_name_count.get(symbol_name, 0) + 1 + static_name_count[symbol_name] = count + symbol_name += b":" + str(count).encode("utf-8") emitted_symbol_name = symbol_name if convert_statics == "global-with-filename": # Change the emitted symbol name to include the filename, - # but don't let that affect deduplication logic. + # but don't let that affect deduplication logic (we still + # want to be able to reference statics from GLOBAL_ASM). emitted_symbol_name = objfile_name.encode("utf-8") + b":" + symbol_name section_name = {1: '.text', 2: '.data', 3: '.bss', 15: '.rodata'}[sc] section = objfile.find_section(section_name) @@ -1308,12 +1361,25 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d st_info=(binding << 4 | symtype), st_other=STV_DEFAULT, st_shndx=section.index, - strtab=objfile.symtab.strtab, - name=symbol_name[:-1].decode('latin1')) - strtab_index += len(emitted_symbol_name) - new_strtab_data.append(emitted_symbol_name) + strtab=objfile.sym_strtab, + name=symbol_name.decode('latin1')) + strtab_index += len(emitted_symbol_name) + 1 + new_strtab_data.append(emitted_symbol_name + b'\0') new_syms.append(sym) - objfile.symtab.strtab.data += b''.join(new_strtab_data) + if st in ( + MIPS_DEBUG_ST_FILE, + MIPS_DEBUG_ST_STRUCT, + MIPS_DEBUG_ST_UNION, + MIPS_DEBUG_ST_ENUM, + MIPS_DEBUG_ST_BLOCK, + MIPS_DEBUG_ST_PROC, + MIPS_DEBUG_ST_STATIC_PROC, + ): + scope_level += 1 + if st == MIPS_DEBUG_ST_END: + scope_level -= 1 + assert scope_level == 0 + objfile.sym_strtab.data += b''.join(new_strtab_data) # Get rid of duplicate symbols, favoring ones that are not UNDEF. # Skip this for unnamed local symbols though. @@ -1323,11 +1389,11 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d name_to_sym = {} for s in new_syms: if s.name == "_gp_disp": - s.type = STT_OBJECT - if s.bind == STB_LOCAL and s.st_shndx == SHN_UNDEF: + s.st_type = STT_OBJECT + if s.st_bind == STB_LOCAL and s.st_shndx == SHN_UNDEF: raise Failure("local symbol \"" + s.name + "\" is undefined") if not s.name: - if s.bind != STB_LOCAL: + if s.st_bind != STB_LOCAL: raise Failure("global symbol with no name") newer_syms.append(s) else: @@ -1335,7 +1401,9 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d if not existing: name_to_sym[s.name] = s newer_syms.append(s) - elif s.st_shndx != SHN_UNDEF: + elif s.st_shndx != SHN_UNDEF and not ( + existing.st_shndx == s.st_shndx and existing.st_value == s.st_value + ): raise Failure("symbol \"" + s.name + "\" defined twice") else: s.replace_by = existing @@ -1345,8 +1413,8 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d # Put local symbols in front, with the initial dummy entry first, and # _gp_disp at the end if it exists. new_syms.insert(0, empty_symbol) - new_syms.sort(key=lambda s: (s.bind != STB_LOCAL, s.name == "_gp_disp")) - num_local_syms = sum(1 for s in new_syms if s.bind == STB_LOCAL) + new_syms.sort(key=lambda s: (s.st_bind != STB_LOCAL, s.name == "_gp_disp")) + num_local_syms = sum(1 for s in new_syms if s.st_bind == STB_LOCAL) for i, s in enumerate(new_syms): s.new_index = i @@ -1368,7 +1436,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d sectype == '.rodata' and rel.r_offset in jtbl_rodata_positions): # don't include relocations for late_rodata dummy code continue - rel.sym_index = objfile.symtab.symbol_entries[rel.sym_index].new_index + rel.sym_index = objfile.symbol_entries[rel.sym_index].new_index nrels.append(rel) reltab.relocations = nrels reltab.data = b''.join(rel.to_bin() for rel in nrels) @@ -1382,28 +1450,20 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d target_sectype = '.rodata' if sectype == '.late_rodata' else sectype target = objfile.find_section(target_sectype) assert target is not None, target_sectype - target_reltab = objfile.find_section('.rel' + target_sectype) - target_reltaba = objfile.find_section('.rela' + target_sectype) for reltab in source.relocated_by: for rel in reltab.relocations: - rel.sym_index = asm_objfile.symtab.symbol_entries[rel.sym_index].new_index + rel.sym_index = asm_objfile.symbol_entries[rel.sym_index].new_index if sectype == '.late_rodata': rel.r_offset = moved_late_rodata[rel.r_offset] new_data = b''.join(rel.to_bin() for rel in reltab.relocations) - if reltab.sh_type == SHT_REL: - if not target_reltab: - target_reltab = objfile.add_section('.rel' + target_sectype, - sh_type=SHT_REL, sh_flags=0, - sh_link=objfile.symtab.index, sh_info=target.index, - sh_addralign=4, sh_entsize=8, data=b'') - target_reltab.data += new_data - else: - if not target_reltaba: - target_reltaba = objfile.add_section('.rela' + target_sectype, - sh_type=SHT_RELA, sh_flags=0, - sh_link=objfile.symtab.index, sh_info=target.index, - sh_addralign=4, sh_entsize=12, data=b'') - target_reltaba.data += new_data + prefix, sh_entsize = ('.rel', 8) if reltab.sh_type == SHT_REL else ('.rela', 12) + target_reltab = objfile.find_section(prefix + target_sectype) + if not target_reltab: + target_reltab = objfile.add_section(prefix + target_sectype, + sh_type=reltab.sh_type, sh_flags=0, + sh_link=objfile.symtab.index, sh_info=target.index, + sh_addralign=4, sh_entsize=sh_entsize, data=b'') + target_reltab.data += new_data objfile.write(objfile_name) finally: @@ -1415,16 +1475,20 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d pass def run_wrapped(argv, outfile, functions): + dir_path = Path(__file__).resolve().parent parser = argparse.ArgumentParser(description="Pre-process .c files and post-process .o files to enable embedding assembly into C.") parser.add_argument('filename', help="path to .c code") parser.add_argument('--post-process', dest='objfile', help="path to .o file to post-process") parser.add_argument('--assembler', dest='assembler', help="assembler command (e.g. \"mips-linux-gnu-as -march=vr4300 -mabi=32\")") - parser.add_argument('--asm-prelude', dest='asm_prelude', help="path to a file containing a prelude to the assembly file (with .set and .macro directives, e.g.)") + parser.add_argument('--asm-prelude', dest='asm_prelude', type=Path, default=dir_path / "prelude.inc", help="path to a file containing a prelude to the assembly file (with .set and .macro directives, e.g.)") parser.add_argument('--input-enc', default='latin1', help="input encoding (default: %(default)s)") parser.add_argument('--output-enc', default='latin1', help="output encoding (default: %(default)s)") parser.add_argument('--drop-mdebug-gptab', dest='drop_mdebug_gptab', action='store_true', help="drop mdebug and gptab sections") parser.add_argument('--convert-statics', dest='convert_statics', choices=["no", "local", "global", "global-with-filename"], default="local", help="change static symbol visibility (default: %(default)s)") parser.add_argument('--force', dest='force', action='store_true', help="force processing of files without GLOBAL_ASM blocks") + parser.add_argument('--keep-preprocessed', dest='keep_output_dir', type=Path, help="emit temporary files to this directory (build.py only)") + parser.add_argument('--no-dep-file', action='store_true', help="don't generate a .d make dependency file (build.py only)") + parser.add_argument('--encode-cutscene-data-floats', dest='encode_cutscene_data_floats', action='store_true', default=False, help="Replace floats with their encoded hexadecimal representation in CutsceneData data") parser.add_argument('-framepointer', dest='framepointer', action='store_true') parser.add_argument('-mips1', dest='mips1', action='store_true') parser.add_argument('-g3', dest='g3', action='store_true') @@ -1445,13 +1509,14 @@ def run_wrapped(argv, outfile, functions): raise Failure("-mips1 is only supported together with -O1 or -O2") if pascal and opt not in ('O1', 'O2', 'g3'): raise Failure("Pascal is only supported together with -O1, -O2 or -O2 -g3") - opts = Opts(opt, args.framepointer, args.mips1, args.kpic, pascal, args.input_enc, args.output_enc) + output_enc = Encoding(args.output_enc) + opts = Opts(opt, args.framepointer, args.mips1, args.kpic, pascal, args.input_enc, output_enc, args.encode_cutscene_data_floats) if args.objfile is None: with open(args.filename, encoding=args.input_enc) as f: deps = [] functions = parse_source(f, opts, out_dependencies=deps, print_source=outfile) - return functions, deps + return functions, deps, args.keep_output_dir else: if args.assembler is None: raise Failure("must pass assembler command") @@ -1464,7 +1529,7 @@ def run_wrapped(argv, outfile, functions): if args.asm_prelude: with open(args.asm_prelude, 'rb') as f: asm_prelude = f.read() - fixup_objfile(args.objfile, functions, asm_prelude, args.assembler, args.output_enc, args.drop_mdebug_gptab, args.convert_statics) + fixup_objfile(args.objfile, functions, asm_prelude, args.assembler, output_enc, args.drop_mdebug_gptab, args.convert_statics) def run(argv, outfile=sys.stdout.buffer, functions=None): try: diff --git a/tools/asm-processor/build.py b/tools/asm-processor/build.py index efbaade..79583e2 100644 --- a/tools/asm-processor/build.py +++ b/tools/asm-processor/build.py @@ -7,35 +7,68 @@ import tempfile import uuid import asm_processor -# Boolean for debugging purposes -# Preprocessed files are temporary, set to True to keep a copy -keep_preprocessed_files = False - -dir_path = Path(__file__).resolve().parent -asm_prelude_path = dir_path / "prelude.inc" +# Preprocessed files are temporary. For debugging purposes, you can set this +# variable or pass --keep-preprocessed path/to/dir/ to keep a copy. +keep_output_dir = None +# keep_output_dir = Path("./asm_processor_preprocessed") +progname = sys.argv[0] all_args = sys.argv[1:] -sep0 = next(index for index, arg in enumerate(all_args) if not arg.startswith("-")) -sep1 = all_args.index("--") -sep2 = all_args.index("--", sep1 + 1) -asmproc_flags = all_args[:sep0] -compiler = all_args[sep0:sep1] +i = 0 +asmproc_flags = [] +while i < len(all_args): + arg = all_args[i] + if arg == "--": + i += 1 + break + if not arg.startswith("-"): + break + i += 1 + asmproc_flags.append(arg) + if arg in ("--input-enc", "--output-enc", "--asm-prelude", "--convert-statics", "--keep-preprocessed", "--no-dep-file") and i < len(all_args): + asmproc_flags.append(all_args[i]) + i += 1 + +sep0 = i +try: + sep1 = all_args.index("--", sep0) + sep2 = all_args.index("--", sep1 + 1) +except ValueError: + print(f"Usage: {progname} [options] <compiler...> -- <assembler...> -- <compiler flags...>") + sys.exit(1) +compiler = all_args[sep0:sep1] assembler_args = all_args[sep1 + 1 : sep2] +compile_args = all_args[sep2 + 1 :] + assembler_sh = " ".join(shlex.quote(x) for x in assembler_args) -compile_args = all_args[sep2 + 1 :] +def fail_parse(msg): + print(f"Failed to parse compiler flags: {msg}") + sys.exit(1) -in_file = Path(compile_args[-1]) -del compile_args[-1] +try: + out_ind = compile_args.index("-o") +except ValueError: + fail_parse("missing -o argument") + +try: + out_file = Path(compile_args[out_ind + 1]) +except IndexError: + fail_parse("missing argument after -o") -out_ind = compile_args.index("-o") -out_file = Path(compile_args[out_ind + 1]) del compile_args[out_ind + 1] del compile_args[out_ind] +try: + in_file = Path(compile_args[-1]) +except IndexError: + fail_parse("missing input file argument") + +del compile_args[-1] + in_dir = in_file.resolve().parent opt_flags = [ @@ -59,13 +92,13 @@ with tempfile.TemporaryDirectory(prefix="asm_processor") as tmpdirname: preprocessed_filename = "preprocessed_" + uuid.uuid4().hex + in_file.suffix preprocessed_path = tmpdir_path / preprocessed_filename - with preprocessed_path.open("wb") as f: - functions, deps = asm_processor.run(asmproc_flags, outfile=f) + with preprocessed_path.open("wb") as outfile: + functions, deps, keep_output_dir2 = asm_processor.run(asmproc_flags, outfile=outfile) - if keep_preprocessed_files: + keep_output_dir = keep_output_dir or keep_output_dir2 + if keep_output_dir: import shutil - keep_output_dir = Path("./asm_processor_preprocessed") keep_output_dir.mkdir(parents=True, exist_ok=True) shutil.copy( @@ -95,14 +128,12 @@ with tempfile.TemporaryDirectory(prefix="asm_processor") as tmpdirname: str(out_file), "--assembler", assembler_sh, - "--asm-prelude", - str(asm_prelude_path), ], functions=functions, ) deps_file = out_file.with_suffix(".asmproc.d") - if deps: + if deps and "--no-dep-file" not in asmproc_flags: with deps_file.open("w") as f: f.write(str(out_file) + ": " + " \\\n ".join(deps) + "\n") for dep in deps: diff --git a/tools/asm-processor/compile-test.sh b/tools/asm-processor/compile-test.sh index 6551662..3039f8a 100755 --- a/tools/asm-processor/compile-test.sh +++ b/tools/asm-processor/compile-test.sh @@ -1,8 +1,12 @@ #!/bin/bash set -o pipefail -INPUT="$1" -OUTPUT="${INPUT%.*}.o" +INPUT=$(readlink -f "$1") + +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" +WD=$(pwd) +INPUT=${INPUT#"$WD"/} +OUTPUT="${INPUT%.*}.o" rm -f "$OUTPUT" CC="$MIPS_CC" # ido 7.1 via recomp or qemu-irix @@ -23,4 +27,15 @@ if [[ "$OPTFLAGS" != *-KPIC* ]]; then fi set -e -python3 build.py --drop-mdebug-gptab $ASMPFLAGS $CC -- $AS $ASFLAGS -- $CFLAGS $OPTFLAGS $ISET -o "$OUTPUT" "$INPUT" + +if [[ "$2" == "python" ]]; then + PROG="python3 ./build.py" +elif [[ "$2" == "rust-release" ]]; then + PROG="./rust/target/release/asm-processor" +elif [[ "$2" == "rust-debug" ]]; then + PROG="./rust/target/debug/asm-processor" +else + echo "Usage: $0 input.c (python|rust-release|rust-debug)" +fi + +$PROG --drop-mdebug-gptab $ASMPFLAGS $CC -- $AS $ASFLAGS -- $CFLAGS $OPTFLAGS $ISET -o "$OUTPUT" "$INPUT" diff --git a/tools/asm-processor/dist-workspace.toml b/tools/asm-processor/dist-workspace.toml new file mode 100644 index 0000000..542b690 --- /dev/null +++ b/tools/asm-processor/dist-workspace.toml @@ -0,0 +1,13 @@ +[workspace] +members = ["cargo:rust/"] + +# Config for 'dist' +[dist] +# The preferred dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.28.0" +# CI backends to support +ci = "github" +# The installers to generate for each app +installers = [] +# Target platforms to build apps for (Rust target-triple syntax) +targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] diff --git a/tools/asm-processor/mypy.ini b/tools/asm-processor/mypy.ini new file mode 100644 index 0000000..9a57d3c --- /dev/null +++ b/tools/asm-processor/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +files = asm_processor.py, build.py + diff --git a/tools/asm-processor/prelude.inc b/tools/asm-processor/prelude.inc index 9bc8939..fc08ac6 100644 --- a/tools/asm-processor/prelude.inc +++ b/tools/asm-processor/prelude.inc @@ -2,21 +2,51 @@ .set noreorder .set gp=64 -.macro glabel label - .global \label +# A function symbol (but some projects use it for everything). +.macro glabel label, visibility=global + .\visibility \label \label: .endm -.macro dlabel label - .global \label +# The end of a function symbol. +.macro endlabel label + .size \label, . - \label +.endm + +# An alternative entry to a function. +.macro alabel label, visibility=global + .\visibility \label + .type \label, @function + \label: +.endm + + +# A data symbol. +.macro dlabel label, visibility=global + .\visibility \label \label: .endm +# End of a data symbol. +.macro enddlabel label + .size \label, . - \label +.endm + + +# A label referenced by a jumptable. .macro jlabel label \label: .endm +# Label to signal the symbol haven't been matched yet. +.macro nonmatching label, size=1 + .global \label\().NON_MATCHING + .type \label\().NON_MATCHING, @object + .size \label\().NON_MATCHING, \size + \label\().NON_MATCHING: +.endm + # Float register aliases (o32 ABI, odd ones are rarely used) .set $fv0, $f0 diff --git a/tools/asm-processor/run-tests.sh b/tools/asm-processor/run-tests.sh index 5cae0a7..0448ffb 100755 --- a/tools/asm-processor/run-tests.sh +++ b/tools/asm-processor/run-tests.sh @@ -1,6 +1,23 @@ #!/usr/bin/env bash -for A in tests/*.c tests/*.p; do - OBJDUMPFLAGS=-srt - echo $A - ./compile-test.sh "$A" && mips-linux-gnu-objdump $OBJDUMPFLAGS "${A%.*}.o" | diff - "${A%.*}.objdump" || echo FAIL "$A" + +TESTS=${*:-python rust-release} + +if [[ -z "$MIPS_CC" ]]; then + echo "MIPS_CC not set" + exit 1 +fi + +cd "$(dirname "${BASH_SOURCE[0]}")" + +FAILED=0 + +OBJDUMPFLAGS=-srt +for typ in $TESTS; do + for A in tests/*.c tests/*.p; do + echo "$A ($typ)" + ./compile-test.sh "$A" $typ && mips-linux-gnu-objdump $OBJDUMPFLAGS "${A%.*}.o" | diff -w "${A%.*}.objdump" - || (echo FAIL "$A" && exit 1) + FAILED=$(( $? + $FAILED )) + done done + +exit $(( $FAILED != 0 )) diff --git a/tools/asm-processor/rust/Cargo.lock b/tools/asm-processor/rust/Cargo.lock new file mode 100644 index 0000000..46e46e9 --- /dev/null +++ b/tools/asm-processor/rust/Cargo.lock @@ -0,0 +1,236 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" + +[[package]] +name = "argp" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7409aa6f1dd8464eac2e56cf538e1e5f7f79678caa32f198d214a3db8d5075c1" +dependencies = [ + "argp_derive", +] + +[[package]] +name = "argp_derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d9b949411282939e3f7d8923127e3f18aa474b46da4e8bb0ddf2cb8c81f963a" +dependencies = [ + "proc-macro2", + "pulldown-cmark", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + +[[package]] +name = "asm-processor" +version = "1.0.0" +dependencies = [ + "anyhow", + "argp", + "binrw", + "encoding_rs", + "enum-map", + "regex-lite", + "shlex", + "temp-dir", +] + +[[package]] +name = "binrw" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d4bca59c20d6f40c2cc0802afbe1e788b89096f61bdf7aeea6bf00f10c2909b" +dependencies = [ + "array-init", + "binrw_derive", + "bytemuck", +] + +[[package]] +name = "binrw_derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8ba42866ce5bced2645bfa15e97eef2c62d2bdb530510538de8dd3d04efff3c" +dependencies = [ + "either", + "owo-colors", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "bytemuck" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" +dependencies = [ + "bitflags", + "getopts", + "memchr", + "unicase", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "temp-dir" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1ee6eef34f12f765cb94725905c6312b6610ab2b0940889cfe58dae7bc3c72" + +[[package]] +name = "unicase" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" diff --git a/tools/asm-processor/rust/Cargo.toml b/tools/asm-processor/rust/Cargo.toml new file mode 100644 index 0000000..3ef4479 --- /dev/null +++ b/tools/asm-processor/rust/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "asm-processor" +version = "1.0.0" +edition = "2021" +description = "Pre-process .c files and post-process .o files to enable embedding MIPS assembly into IDO-compiled C." +license = "Unlicense" +repository = "https://github.com/simonlindholm/asm-processor" +readme = "../README.md" +keywords = ["decompilation", "ido", "mips", "asm"] + +[dependencies] +anyhow = "1.0" +binrw = "0.14.1" +encoding_rs = "0.8.35" +enum-map = "2.7.3" +regex-lite = "0.1.6" +shlex = "1.3.0" +temp-dir = "0.1.14" +argp = "0.4.0" + +# The profile that 'dist' will build with +[profile.dist] +inherits = "release" +lto = "thin" diff --git a/tools/asm-processor/rust/src/main.rs b/tools/asm-processor/rust/src/main.rs new file mode 100644 index 0000000..8e214ee --- /dev/null +++ b/tools/asm-processor/rust/src/main.rs @@ -0,0 +1,518 @@ +mod postprocess; +mod preprocess; + +use std::{ + borrow::Cow, + ffi::OsString, + fmt::Display, + fs::{self, File}, + io::Write, + path::{Path, PathBuf}, + process::{exit, Command}, + str::FromStr, +}; + +use anyhow::Result; +use argp::{EarlyExit, FromArgs, HelpStyle}; +use encoding_rs::EUC_JP; +use enum_map::{Enum, EnumMap}; +use temp_dir::TempDir; + +use postprocess::fixup_objfile; +use preprocess::parse_source; + +#[derive(Copy, Clone, Eq, PartialEq, Debug, Enum)] +enum OutputSection { + Text, + Data, + Rodata, + Bss, +} + +impl OutputSection { + fn as_str(&self) -> &'static str { + match self { + OutputSection::Text => ".text", + OutputSection::Data => ".data", + OutputSection::Rodata => ".rodata", + OutputSection::Bss => ".bss", + } + } +} + +impl Display for OutputSection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +#[derive(Clone, Debug)] +struct Function { + text_glabels: Vec<String>, + asm_conts: Vec<String>, + late_rodata_dummy_bytes: Vec<[u8; 4]>, + jtbl_rodata_size: usize, + late_rodata_asm_conts: Vec<String>, + fn_desc: String, + data: EnumMap<OutputSection, (Option<String>, usize)>, +} + +#[derive(Clone, Copy, Debug)] +enum Encoding { + Latin1, + Custom(&'static encoding_rs::Encoding), +} + +impl Encoding { + fn encode<'a>(&self, s: &'a str) -> Result<Cow<'a, [u8]>> { + match self { + Encoding::Latin1 => { + if encoding_rs::mem::is_str_latin1(s) { + return Ok(encoding_rs::mem::encode_latin1_lossy(s)); + } + } + Encoding::Custom(enc) => { + if *enc == EUC_JP { + let s = s.replace("〜", "～"); + let (ret, _, failed) = enc.encode(&s); + if !failed { + return Ok(Cow::Owned(ret.into_owned())); + } + } else { + let (ret, _, failed) = enc.encode(s); + if !failed { + return Ok(ret); + } + } + } + } + Err(anyhow::anyhow!("Failed to encode string: {}", s)) + } + + fn decode<'a>(&self, bytes: &'a [u8]) -> Result<Cow<'a, str>> { + match self { + Encoding::Latin1 => Ok(encoding_rs::mem::decode_latin1(bytes)), + Encoding::Custom(enc) => { + let (ret, _, failed) = enc.decode(bytes); + if !failed { + Ok(ret) + } else { + Err(anyhow::anyhow!("Failed to decode string: {}", ret)) + } + } + } + } +} + +impl FromStr for Encoding { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + if s == "latin1" { + Ok(Encoding::Latin1) + } else { + match encoding_rs::Encoding::for_label(s.as_bytes()) { + Some(enc) => Ok(Encoding::Custom(enc)), + None => Err(format!("Unsupported encoding: {}", s)), + } + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq)] +enum ConvertStatics { + No, + Local, + Global, + GlobalWithFilename, +} + +impl FromStr for ConvertStatics { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + Ok(match s { + "no" => ConvertStatics::No, + "local" => ConvertStatics::Local, + "global" => ConvertStatics::Global, + "global-with-filename" => ConvertStatics::GlobalWithFilename, + _ => return Err("invalid value for symbol visibility".into()), + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +enum OptLevel { + O0, + O1, + O2, + G, + G3, +} + +/// Pre-process .c files and post-process .o files to enable embedding MIPS assembly into IDO-compiled C. +#[derive(FromArgs)] +struct AsmProcArgs { + /// Path to a file containing a prelude to the assembly file (with .set and .macro directives, e.g.) + #[argp(option, arg_name = "FILE")] + asm_prelude: Option<PathBuf>, + + /// Input encoding (default: latin1) + #[argp( + option, + default = "Encoding::Latin1", + from_str_fn(FromStr::from_str), + arg_name = "ENCODING" + )] + input_enc: Encoding, + + /// Output encoding (default: latin1) + #[argp( + option, + default = "Encoding::Latin1", + from_str_fn(FromStr::from_str), + arg_name = "ENCODING" + )] + output_enc: Encoding, + + /// Drop mdebug and gptab sections + #[argp(switch)] + drop_mdebug_gptab: bool, + + /// Change symbol visibility for static variables. Mode must be one of: + /// no, local, global, global-with-filename (default: local) + #[argp( + option, + default = "ConvertStatics::Local", + from_str_fn(FromStr::from_str), + arg_name = "MODE" + )] + convert_statics: ConvertStatics, + + /// Force processing of files without GLOBAL_ASM blocks + #[argp(switch)] + force: bool, + + /// Emit temporary files to this directory + #[argp(option, arg_name = "DIR")] + keep_preprocessed: Option<PathBuf>, + + /// Replace floats with their encoded hexadecimal representation in CutsceneData data + #[argp(switch)] + encode_cutscene_data_floats: bool, + + /// Don't generate a .d make dependency file + #[argp(switch)] + no_dep_file: bool, + + #[argp(positional, greedy)] + rest: Vec<String>, +} + +struct CompileOpts { + opt: OptLevel, + framepointer: bool, + mips1: bool, + kpic: bool, + pascal: bool, +} + +fn extract_compiler_input_output( + compile_args: &[String], +) -> Result<(PathBuf, PathBuf, Vec<String>), &'static str> { + let mut compile_args: Vec<String> = compile_args.to_vec(); + let out_ind = compile_args + .iter() + .position(|arg| arg == "-o") + .ok_or("missing -o argument")?; + let out_filename = compile_args + .get(out_ind + 1) + .ok_or("missing argument after -o")? + .clone(); + compile_args.remove(out_ind + 1); + compile_args.remove(out_ind); + + let in_file_str = compile_args + .last() + .ok_or("missing input file argument")? + .clone(); + compile_args.pop(); + + let out_file: PathBuf = out_filename.into(); + let in_file: PathBuf = in_file_str.into(); + + Ok((in_file, out_file, compile_args)) +} + +fn parse_compile_args( + compile_args: &[String], + in_file: &Path, +) -> Result<CompileOpts, &'static str> { + let mut opt_flags = vec![]; + for x in compile_args { + opt_flags.push(match x.as_str() { + "-g" => OptLevel::G, + "-O0" => OptLevel::O0, + "-O1" => OptLevel::O1, + "-O2" => OptLevel::O2, + _ => continue, + }); + } + + if opt_flags.len() != 1 { + return Err("exactly one of -g/-O0/-O1/-O2 must be passed"); + } + let mut opt = opt_flags[0]; + + let mips1 = !compile_args.contains(&"-mips2".to_string()); + let framepointer = compile_args.contains(&"-framepointer".to_string()); + let kpic = compile_args.contains(&"-KPIC".to_string()); + if compile_args.contains(&"-g3".to_string()) { + if opt != OptLevel::O2 { + return Err("-g3 is only supported together with -O2"); + } + opt = OptLevel::G3; + } + + if mips1 && (!matches!(opt, OptLevel::O1 | OptLevel::O2) || framepointer) { + return Err("-mips1 is only supported together with -O1 or -O2"); + } + + let in_file_str = in_file.to_string_lossy(); + let pascal = in_file_str.ends_with(".p") + || in_file_str.ends_with(".pas") + || in_file_str.ends_with(".pp"); + + if pascal && !matches!(opt, OptLevel::O1 | OptLevel::O2 | OptLevel::G3) { + return Err("Pascal is only supported together with -O1, -O2 or -O2 -g3"); + } + + Ok(CompileOpts { + opt, + framepointer, + mips1, + kpic, + pascal, + }) +} + +fn parse_rest(rest: &[String]) -> Option<(&[String], &[String], &[String])> { + let mut iter = rest.splitn(3, |x| *x == "--"); + let compiler = iter.next()?; + let assembler = iter.next()?; + let compile_args = iter.next()?; + assert!(iter.next().is_none()); + Some((compiler, assembler, compile_args)) +} + +struct ParsedArgs { + args: AsmProcArgs, + compiler: Vec<String>, + assembler: Vec<String>, + compile_args: Vec<String>, + in_file: PathBuf, + out_file: PathBuf, + opts: CompileOpts, +} + +/// Parse command line arguments while allowing --a=b syntax. +/// +/// This provides backward compatibility with Python argparse. +fn from_args_allow_eq(progname: &[&str], args: &[OsString]) -> Result<AsmProcArgs, EarlyExit> { + let base_res = AsmProcArgs::from_args(progname, args); + if base_res.is_ok() { + return base_res; + } + + // Try splitting up = chars successively, until we get a valid parse. + // This ensures we don't impact the compiler/assembler parts. + // + // Technically this might end up splitting --a --b=c into --a --b c even + // where --a is a flag that takes an argument --b=c, but this seems unlikely + // with our use case. + let mut i = 0; + let mut args = args.to_vec(); + while i < args.len() { + let arg = args[i].as_encoded_bytes(); + if arg.starts_with(b"--") { + if let Some(eq) = arg.iter().position(|&x| x == b'=') { + // SAFETY: splitting on ASCII still results in valid encoded bytes + let before = unsafe { OsString::from_encoded_bytes_unchecked(arg[0..eq].into()) }; + let after = unsafe { OsString::from_encoded_bytes_unchecked(arg[eq + 1..].into()) }; + args.splice(i..i + 1, [before, after]); + let new_res = AsmProcArgs::from_args(progname, &args); + if new_res.is_ok() { + return new_res; + } + i += 1; + } + } + i += 1; + } + + base_res +} + +fn parse_args_or_exit() -> ParsedArgs { + let argv: Vec<_> = std::env::args_os().collect(); + let help_style = HelpStyle { + short_usage: true, + ..HelpStyle::default() + }; + let progname = argv[0].to_string_lossy(); + + let args = from_args_allow_eq(&[&progname], &argv[1..]).unwrap_or_else(|early_exit| { + exit(match early_exit { + EarlyExit::Help(help) => { + println!( + "{}", + help.generate(&help_style).replace( + "[rest...]", + "<compiler...> -- <assembler...> -- <compiler flags...>" + ) + ); + 0 + } + EarlyExit::Err(err) => { + eprintln!("{}\nRun {} --help for more information.", err, progname); + 1 + } + }) + }); + + let Some((compiler, assembler, compile_args)) = parse_rest(&args.rest) else { + eprintln!( + "Usage: {} [options] <compiler...> -- <assembler...> -- <compiler flags...>", + progname + ); + eprintln!("Run {} --help for more information.", progname); + exit(1); + }; + + let (in_file, out_file, compile_args) = extract_compiler_input_output(compile_args) + .unwrap_or_else(|err| { + eprintln!("Failed to parse compiler flags: {}", err); + exit(1); + }); + + let opts = parse_compile_args(&compile_args, &in_file).unwrap_or_else(|err| { + eprintln!("Unsupported compiler flags: {}", err); + exit(1); + }); + + let compiler = compiler.into(); + let assembler = assembler.into(); + + ParsedArgs { + args, + compiler, + assembler, + compile_args, + in_file, + out_file, + opts, + } +} + +fn main() -> Result<()> { + let ParsedArgs { + args, + compiler, + assembler, + compile_args, + in_file, + out_file, + opts, + } = parse_args_or_exit(); + + let assembler_sh = assembler + .iter() + .map(|s| shlex::try_quote(s).unwrap().into_owned()) + .collect::<Vec<String>>() + .join(" "); + + let in_dir = fs::canonicalize(in_file.parent().unwrap().join("."))?; + + let temp_dir = TempDir::with_prefix("asm_processor")?; + let preprocessed_filename = format!( + "preprocessed_{}", + in_file.file_name().unwrap().to_str().unwrap() + ); + let preprocessed_path = temp_dir.path().join(&preprocessed_filename); + let mut preprocessed_file = File::create(&preprocessed_path)?; + + let res = parse_source(&in_file, &args, &opts, true)?; + preprocessed_file.write_all(&res.output)?; + + if let Some(keep_output_dir) = &args.keep_preprocessed { + fs::create_dir_all(keep_output_dir)?; + fs::copy( + &preprocessed_path, + keep_output_dir.join(&preprocessed_filename), + )?; + } + + // Run compiler + let mut compile_command = Command::new(&compiler[0]); + compile_command + .args(&compile_args) + .arg("-I") + .arg(in_dir) + .arg("-o") + .arg(&out_file) + .arg(&preprocessed_path); + + match compile_command.status() { + Ok(status) if status.success() => {} + _ => { + return Err(anyhow::anyhow!( + "Failed to compile file {}. Command line:\n\n{:?}\n", + in_file.display(), + compile_command + )); + } + } + + if !res.functions.is_empty() || args.force { + let prelude_str; + let asm_prelude = match &args.asm_prelude { + Some(prelude) => { + if let Ok(res) = fs::read_to_string(prelude) { + prelude_str = res; + &prelude_str + } else { + return Err(anyhow::anyhow!("Failed to read asm prelude")); + } + } + None => include_str!("../../prelude.inc"), + }; + + fixup_objfile( + &out_file, + &res.functions, + asm_prelude, + &assembler_sh, + &args.output_enc, + args.drop_mdebug_gptab, + args.convert_statics, + )?; + } + + if !res.deps.is_empty() && !args.no_dep_file { + let deps_file = out_file.with_extension("asmproc.d"); + let mut deps_file = File::create(&deps_file)?; + + writeln!( + deps_file, + "{}: {}", + out_file.to_str().unwrap(), + res.deps.join(" \\\n ") + )?; + + for dep in res.deps { + writeln!(deps_file, "\n{dep}:")?; + } + } + + Ok(()) +} diff --git a/tools/asm-processor/rust/src/postprocess.rs b/tools/asm-processor/rust/src/postprocess.rs new file mode 100644 index 0000000..60b782b --- /dev/null +++ b/tools/asm-processor/rust/src/postprocess.rs @@ -0,0 +1,1312 @@ +use anyhow::Result; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + fs::{self, File}, + io::{self, BufWriter, Cursor, Seek, SeekFrom, Write}, + path::Path, + process::Command, +}; + +use binrw::{binrw, BinRead, BinResult, BinWrite, Endian}; +use enum_map::EnumMap; +use temp_dir::TempDir; + +use crate::{ConvertStatics, Encoding, Function, OutputSection}; + +const EI_NIDENT: usize = 16; +const EI_CLASS: usize = 4; +const EI_DATA: usize = 5; + +const SHN_UNDEF: usize = 0; +const SHN_ABS: usize = 0xfff1; +const SHN_XINDEX: usize = 0xffff; + +const STT_OBJECT: u8 = 1; +const STT_FUNC: u8 = 2; + +const STB_LOCAL: u8 = 0; +const STB_GLOBAL: u8 = 1; + +const STV_DEFAULT: u8 = 0; + +const SHT_NULL: u32 = 0; +const SHT_SYMTAB: u32 = 2; +const SHT_STRTAB: u32 = 3; +const SHT_RELA: u32 = 4; +const SHT_NOBITS: u32 = 8; +const SHT_REL: u32 = 9; +const SHT_MIPS_GPTAB: u32 = 0x70000003; +const SHT_MIPS_DEBUG: u32 = 0x70000005; + +const SHF_LINK_ORDER: u32 = 0x80; + +const MIPS_DEBUG_ST_STATIC: usize = 2; +const MIPS_DEBUG_ST_PROC: usize = 6; +const MIPS_DEBUG_ST_BLOCK: usize = 7; +const MIPS_DEBUG_ST_END: usize = 8; +const MIPS_DEBUG_ST_FILE: usize = 11; +const MIPS_DEBUG_ST_STATIC_PROC: usize = 14; +const MIPS_DEBUG_ST_STRUCT: usize = 26; +const MIPS_DEBUG_ST_UNION: usize = 27; +const MIPS_DEBUG_ST_ENUM: usize = 28; + +#[binrw] +struct ElfHeader { + e_ident: [u8; EI_NIDENT], + e_type: u16, + e_machine: u16, + e_version: u32, + e_entry: u32, + e_phoff: u32, + e_shoff: u32, + e_flags: u32, + e_ehsize: u16, + e_phentsize: u16, + e_phnum: u16, + e_shentsize: u16, + e_shnum: u16, + e_shstrndx: u16, +} + +impl ElfHeader { + const SIZE: usize = 52; + + fn new(data: &[u8], endian: Endian) -> BinResult<Self> { + let mut cursor = Cursor::new(data); + + let header = Self::read_options(&mut cursor, endian, ())?; + + assert_eq!(header.e_ident[EI_CLASS], 1, "ELF must be 32-bit"); + assert_eq!(header.e_type, 1, "ELF must be relocatable"); + assert_eq!(header.e_machine, 8, "ELF must be MIPS 1"); + assert_eq!(header.e_phoff, 0, "ELF must not have program headers"); + assert_ne!(header.e_shoff, 0, "ELF must have section headers"); + assert_ne!( + header.e_shstrndx, SHN_UNDEF as u16, + "ELF must have a section header string table" + ); + + Ok(header) + } + + fn to_bin(&self, endian: Endian) -> [u8; Self::SIZE] { + let mut rv = [0; Self::SIZE]; + let mut cursor = Cursor::new(rv.as_mut_slice()); + + self.write_options(&mut cursor, endian, ()).unwrap(); + rv + } +} + +#[binrw] +struct SymbolData { + st_name: u32, + st_value: u32, + st_size: u32, + st_info: u8, + st_other: u8, + st_shndx: u16, +} + +#[derive(Clone)] +struct Symbol { + st_name: usize, + st_value: usize, + st_size: usize, + st_shndx: usize, + st_type: u8, + st_bind: u8, + st_visibility: u8, + name: Vec<u8>, +} + +impl Symbol { + fn new(data: &[u8], strtab: &Section, endian: Endian) -> BinResult<Self> { + let mut cursor = Cursor::new(data); + + let data = SymbolData::read_options(&mut cursor, endian, ())?; + if data.st_shndx == SHN_XINDEX as u16 { + panic!("too many sections (SHN_XINDEX not supported)"); + } + let st_type = data.st_info & 0xf; + let st_bind = data.st_info >> 4; + let st_visibility = data.st_other & 0x3; + let name = strtab.lookup_str(data.st_name as usize); + + Ok(Self { + st_name: data.st_name as usize, + st_value: data.st_value as usize, + st_size: data.st_size as usize, + st_shndx: data.st_shndx as usize, + st_type, + st_bind, + st_visibility, + name, + }) + } + + fn to_bin(&self) -> Vec<u8> { + let mut rv = vec![]; + let mut cursor = Cursor::new(&mut rv); + + SymbolData { + st_name: self.st_name as u32, + st_value: self.st_value as u32, + st_size: self.st_size as u32, + st_info: self.st_bind << 4 | self.st_type, + st_other: self.st_visibility, + st_shndx: self.st_shndx as u16, + } + .write_options(&mut cursor, Endian::Big, ()) + .unwrap(); + rv + } +} + +#[derive(Clone)] +struct Relocation { + r_offset: usize, + sym_index: usize, + rel_type: u32, + r_addend: Option<u32>, +} + +impl Relocation { + fn new(data: &[u8], sh_type: u32, endian: Endian) -> BinResult<Self> { + let mut cursor = Cursor::new(data); + + let r_offset = u32::read_options(&mut cursor, endian, ())? as usize; + let r_info = u32::read_options(&mut cursor, endian, ())?; + let r_addend = if sh_type == SHT_REL { + None + } else { + Some(u32::read_options(&mut cursor, endian, ())?) + }; + + let sym_index = (r_info >> 8) as usize; + let rel_type = r_info & 0xff; + + Ok(Self { + r_offset, + sym_index, + rel_type, + r_addend, + }) + } + + fn to_bin(&self, endian: Endian) -> Vec<u8> { + let mut rv = vec![]; + let mut cursor = Cursor::new(&mut rv); + let r_offset = self.r_offset as u32; + let r_info = ((self.sym_index as u32) << 8) | self.rel_type; + r_offset.write_options(&mut cursor, endian, ()).unwrap(); + r_info.write_options(&mut cursor, endian, ()).unwrap(); + self.r_addend + .write_options(&mut cursor, endian, ()) + .unwrap(); + rv + } +} + +#[binrw] +struct Hdrr { + magic: u16, + vstamp: u16, + iline_max: u32, + cb_line: u32, + cb_line_offset: u32, + idn_max: u32, + cb_dn_offset: u32, + ipd_max: u32, + cb_pd_offset: u32, + isym_max: u32, + cb_sym_offset: u32, + iopt_max: u32, + cb_opt_offset: u32, + iaux_max: u32, + cb_aux_offset: u32, + iss_max: u32, + cb_ss_offset: u32, + iss_ext_max: u32, + cb_ss_ext_offset: u32, + ifd_max: u32, + cb_fd_offset: u32, + crfd: u32, + cb_rfd_offset: u32, + iext_max: u32, + cb_ext_offset: u32, +} + +impl Hdrr { + const SIZE: usize = 96; +} + +#[binrw] +#[derive(Clone)] +struct SectionHeader { + sh_name: u32, + sh_type: u32, + sh_flags: u32, + sh_addr: u32, + sh_offset: u32, + sh_size: u32, + sh_link: u32, + sh_info: u32, + sh_addralign: u32, + sh_entsize: u32, +} + +impl SectionHeader { + const SIZE: usize = 40; +} + +#[derive(Clone)] +struct Section { + header: SectionHeader, + data: Vec<u8>, + index: usize, + relocated_by: Vec<usize>, + relocations: Vec<Relocation>, + name: String, +} + +impl Section { + fn new(data: &[u8], other_data: &[u8], index: usize, endian: Endian) -> BinResult<Self> { + let mut cursor = Cursor::new(data); + + let header = SectionHeader::read_options(&mut cursor, endian, ())?; + assert!(header.sh_flags & SHF_LINK_ORDER == 0); + if header.sh_entsize != 0 { + assert_eq!(header.sh_size % header.sh_entsize, 0); + } + + let data = if header.sh_type == SHT_NOBITS { + vec![] + } else { + other_data[header.sh_offset as usize..(header.sh_offset + header.sh_size) as usize] + .to_vec() + }; + Ok(Self { + header, + data, + index, + relocated_by: vec![], + relocations: vec![], + name: "".into(), + }) + } + + fn from_parts( + sh_name: u32, + fields: &HeaderFields, + data: &[u8], + index: usize, + endian: Endian, + ) -> Self { + let header = SectionHeader { + sh_name, + sh_type: fields.sh_type, + sh_flags: fields.sh_flags, + sh_addr: 0, + sh_offset: 0, + sh_size: data.len() as u32, + sh_link: fields.sh_link, + sh_info: fields.sh_info, + sh_addralign: fields.sh_addralign, + sh_entsize: fields.sh_entsize, + }; + + let mut rv = [0; SectionHeader::SIZE]; + let mut cursor = Cursor::new(rv.as_mut_slice()); + + header.write_options(&mut cursor, endian, ()).unwrap(); + + Self::new(&rv, data, index, endian).unwrap() + } + + fn lookup_str(&self, index: usize) -> Vec<u8> { + assert_eq!(self.header.sh_type, SHT_STRTAB); + let to = self.data[index..] + .iter() + .position(|&x| x == 0) + .expect("bad strtab index") + + index; + self.data[index..to].to_owned() + } + + fn add_str(&mut self, string: &[u8]) -> u32 { + assert_eq!(self.header.sh_type, SHT_STRTAB); + let index = self.data.len() as u32; + + self.data.extend_from_slice(string); + self.data.push(0); + index + } + + fn is_rel(&self) -> bool { + self.header.sh_type == SHT_REL || self.header.sh_type == SHT_RELA + } + + fn header_to_bin(&mut self, endian: Endian) -> [u8; SectionHeader::SIZE] { + if self.header.sh_type != SHT_NOBITS { + self.header.sh_size = self.data.len() as u32; + } + + let mut rv = [0; SectionHeader::SIZE]; + let mut cursor = Cursor::new(rv.as_mut_slice()); + + self.header.write_options(&mut cursor, endian, ()).unwrap(); + + rv + } + + fn init_relocs(&mut self, endian: Endian) { + assert!(self.is_rel()); + + let mut entries = vec![]; + for i in (0..self.header.sh_size).step_by(self.header.sh_entsize as usize) { + entries.push( + Relocation::new( + &self.data[i as usize..(i + self.header.sh_entsize) as usize], + self.header.sh_type, + endian, + ) + .unwrap(), + ); + } + self.relocations = entries; + } + + fn relocate_mdebug(&mut self, original_offset: u32, endian: Endian) { + assert_eq!(self.header.sh_type, SHT_MIPS_DEBUG); + let shift_by = self.header.sh_offset.wrapping_sub(original_offset); + + let mut hdrr = Hdrr::read_options(&mut Cursor::new(&self.data), endian, ()).unwrap(); + + assert_eq!(hdrr.magic, 0x7009); + + let relocate = |a, b: &mut u32| { + if a != 0 { + *b = b.wrapping_add(shift_by); + } + }; + relocate(hdrr.cb_line, &mut hdrr.cb_line_offset); + relocate(hdrr.idn_max, &mut hdrr.cb_dn_offset); + relocate(hdrr.ipd_max, &mut hdrr.cb_pd_offset); + relocate(hdrr.isym_max, &mut hdrr.cb_sym_offset); + relocate(hdrr.iopt_max, &mut hdrr.cb_opt_offset); + relocate(hdrr.iaux_max, &mut hdrr.cb_aux_offset); + relocate(hdrr.iss_max, &mut hdrr.cb_ss_offset); + relocate(hdrr.iss_ext_max, &mut hdrr.cb_ss_ext_offset); + relocate(hdrr.ifd_max, &mut hdrr.cb_fd_offset); + relocate(hdrr.crfd, &mut hdrr.cb_rfd_offset); + relocate(hdrr.iext_max, &mut hdrr.cb_ext_offset); + + let mut new_data = [0; Hdrr::SIZE]; + let mut cursor = Cursor::new(new_data.as_mut_slice()); + hdrr.write_options(&mut cursor, endian, ()).unwrap(); + + self.data = new_data.to_vec(); + } +} + +struct ElfFile { + data: Vec<u8>, + endian: Endian, + header: ElfHeader, + sections: Vec<Section>, + symbol_entries: Vec<Symbol>, + symtab: usize, + sym_strtab: usize, +} + +struct HeaderFields { + sh_type: u32, + sh_flags: u32, + sh_link: u32, + sh_info: u32, + sh_addralign: u32, + sh_entsize: u32, +} + +impl ElfFile { + fn new(data: &[u8]) -> BinResult<Self> { + let data = data.to_vec(); + assert_eq!(data[..4], [0x7f, b'E', b'L', b'F']); + + let endian: Endian = if data[EI_DATA] == 1 { + Endian::Little + } else if data[5] == 2 { + Endian::Big + } else { + panic!("Invalid ELF endianness"); + }; + let header = ElfHeader::new(&data[..ElfHeader::SIZE], endian).unwrap(); + let offset = header.e_shoff as usize; + let size = header.e_shentsize as usize; + let null_section = Section::new(&data[offset..offset + size], &data, 0, endian).unwrap(); + let num_sections = if header.e_shnum == 0 { + null_section.header.sh_size as usize + } else { + header.e_shnum as usize + }; + let mut sections = vec![null_section]; + + for i in 1..num_sections { + let ind = offset + i * size; + let section = Section::new(&data[ind..ind + size], &data, i, endian).unwrap(); + sections.push(section); + } + + let symtab_index = sections + .iter() + .position(|s| s.header.sh_type == SHT_SYMTAB) + .expect("missing symtab"); + let sym_strtab_index = sections[symtab_index].header.sh_link as usize; + + let symtab = §ions[symtab_index]; + let sym_strtab = §ions[sym_strtab_index]; + let symbol_entries = ElfFile::init_symbols(symtab, sym_strtab, endian); + + let shstr = sections[header.e_shstrndx as usize].clone(); + + for i in 0..sections.len() { + let s = &mut sections[i]; + s.name = String::from_utf8(shstr.lookup_str(s.header.sh_name as usize)).unwrap(); + assert!(s.name.is_ascii()); + + if s.is_rel() { + let target_index = s.header.sh_info as usize; + s.init_relocs(endian); + sections[target_index].relocated_by.push(i); + } + } + + Ok(ElfFile { + data, + endian, + header, + sections, + symbol_entries, + symtab: symtab_index, + sym_strtab: sym_strtab_index, + }) + } + + fn find_section(&self, name: &str) -> Option<&Section> { + self.sections.iter().find(|s| s.name == name) + } + + fn find_section_mut(&mut self, name: &str) -> Option<&mut Section> { + self.sections.iter_mut().find(|s| s.name == name) + } + + fn symtab(&self) -> &Section { + &self.sections[self.symtab] + } + + fn symtab_mut(&mut self) -> &mut Section { + &mut self.sections[self.symtab] + } + + fn sym_strtab(&self) -> &Section { + &self.sections[self.sym_strtab] + } + + fn sym_strtab_mut(&mut self) -> &mut Section { + &mut self.sections[self.sym_strtab] + } + + fn init_symbols(symtab: &Section, strtab: &Section, endian: Endian) -> Vec<Symbol> { + assert_eq!(symtab.header.sh_type, SHT_SYMTAB); + assert_eq!(symtab.header.sh_entsize, 16); + + let mut syms = Vec::new(); + for i in 0..(symtab.data.len() / 16) { + syms.push(Symbol::new(&symtab.data[i * 16..(i + 1) * 16], strtab, endian).unwrap()); + } + syms + } + + fn find_symbol(&self, name: &[u8]) -> Option<(usize, usize)> { + for s in &self.symbol_entries { + if s.name == name { + return Some((s.st_shndx, s.st_value)); + } + } + None + } + + fn find_symbol_in_section(&self, name: &[u8], section: &Section) -> usize { + let Some((st_shndx, st_value)) = self.find_symbol(name) else { + panic!("failed to find symbol: {}", String::from_utf8_lossy(name)); + }; + if st_shndx != section.index { + panic!( + "symbol {} is in wrong section", + String::from_utf8_lossy(name) + ); + } + st_value + } + + fn add_section(&mut self, name: &str, fields: &HeaderFields, data: &[u8], endian: Endian) { + let shstr = self + .sections + .get_mut(self.header.e_shstrndx as usize) + .expect("bad e_shstrndx"); + let sh_name = shstr.add_str(name.as_bytes()); + let mut s = Section::from_parts(sh_name, fields, data, self.sections.len(), endian); + s.name = name.to_string(); + self.sections.push(s); + } + + fn drop_mdebug_gptab(&mut self) { + // We can only drop sections at the end, since otherwise section + // references might be wrong. Luckily, these sections typically are. + while let Some(s) = self.sections.last() { + if s.header.sh_type != SHT_MIPS_DEBUG && s.header.sh_type != SHT_MIPS_GPTAB { + break; + } + self.sections.pop(); + } + } + + fn pad_out(writer: &mut BufWriter<&mut File>, align: usize) -> io::Result<()> { + let pos = writer.stream_position()? as usize; + + if align > 0 && pos % align != 0 { + let pad = align - (pos % align); + for _ in 0..pad { + writer.write_all(&[0])?; + } + } + Ok(()) + } + + fn write(&mut self, writer: &mut BufWriter<&mut File>) -> io::Result<()> { + self.header.e_shnum = self.sections.len() as u16; + writer.write_all(&self.header.to_bin(self.endian))?; + + for s in &mut self.sections { + if s.header.sh_type != SHT_NOBITS && s.header.sh_type != SHT_NULL { + Self::pad_out(writer, s.header.sh_addralign as usize)?; + let old_offset = s.header.sh_offset; + s.header.sh_offset = writer.stream_position()? as u32; + if s.header.sh_type == SHT_MIPS_DEBUG && s.header.sh_offset != old_offset { + // The .mdebug section has moved, relocate offsets + s.relocate_mdebug(old_offset, self.endian); + } + writer.write_all(&s.data)?; + } + } + + Self::pad_out(writer, 4)?; + self.header.e_shoff = writer.stream_position()? as u32; + + for s in &mut self.sections { + writer.write_all(&s.header_to_bin(self.endian))?; + } + + writer.seek(SeekFrom::Start(0))?; + writer.write_all(&self.header.to_bin(self.endian))?; + writer.flush()?; + Ok(()) + } +} + +pub(crate) fn fixup_objfile( + objfile_path: &Path, + functions: &[Function], + asm_prelude: &str, + assembler: &str, + output_enc: &Encoding, + drop_mdebug_gptab: bool, + convert_statics: ConvertStatics, +) -> Result<()> { + const OUTPUT_SECTIONS: [OutputSection; 4] = [ + OutputSection::Data, + OutputSection::Text, + OutputSection::Rodata, + OutputSection::Bss, + ]; + const INPUT_SECTION_NAMES: [&str; 5] = [".data", ".text", ".rodata", ".bss", ".late_rodata"]; + + let objfile_data = fs::read(objfile_path)?; + let mut objfile = ElfFile::new(&objfile_data)?; + let endian = objfile.endian; + + let mut prev_locs: EnumMap<OutputSection, usize> = EnumMap::default(); + + struct ToCopyData { + loc: usize, + size: usize, + temp_name: String, + fn_desc: String, + } + + let mut to_copy: EnumMap<OutputSection, Vec<ToCopyData>> = EnumMap::default(); + + let mut asm: Vec<String> = vec![]; + let mut all_late_rodata_dummy_bytes: Vec<Vec<[u8; 4]>> = vec![]; + let mut all_jtbl_rodata_size: Vec<usize> = vec![]; + let mut late_rodata_asm: Vec<String> = vec![]; + let late_rodata_source_name_start = "_asmpp_late_rodata_start"; + let late_rodata_source_name_end = "_asmpp_late_rodata_end"; + + // Generate an assembly file with all the assembly we need to fill in. For + // simplicity we pad with nops/.space so that addresses match exactly, so we + // don't have to fix up relocations/symbol references. + let mut all_text_glabels: HashSet<Vec<u8>> = HashSet::new(); + let mut func_sizes: HashMap<Vec<u8>, usize> = HashMap::new(); + + for function in functions.iter() { + let text_glabels = function + .text_glabels + .iter() + .map(|x| output_enc.encode(x)) + .collect::<Result<Vec<_>>>()?; + let mut ifdefed = false; + for (sectype, &(ref temp_name, size)) in function.data.iter() { + let Some(temp_name) = temp_name else { continue }; + if size == 0 { + panic!("Size of section {} is 0", sectype.as_str()); + } + let Some((_, loc)) = objfile.find_symbol(temp_name.as_bytes()) else { + ifdefed = true; + break; + }; + let prev_loc = prev_locs[sectype]; + if loc < prev_loc { + // If the dummy C generates too little asm, and we have two + // consecutive GLOBAL_ASM blocks, we detect that error here. + // On the other hand, if it generates too much, we don't have + // a good way of discovering that error: it's indistinguishable + // from a static symbol occurring after the GLOBAL_ASM block. + panic!( + "Wrongly computed size for section {} (diff {}). This is an asm-processor bug!", + sectype, + prev_loc - loc + ); + } + if loc != prev_loc { + asm.push(format!(".section {}", sectype)); + if sectype == OutputSection::Text { + for _ in 0..((loc - prev_loc) / 4) { + asm.push("nop".to_owned()); + } + } else { + asm.push(format!(".space {}", loc - prev_loc)); + } + } + to_copy[sectype].push(ToCopyData { + loc, + size, + temp_name: temp_name.clone(), + fn_desc: function.fn_desc.clone(), + }); + if !text_glabels.is_empty() && sectype == OutputSection::Text { + func_sizes.insert(text_glabels[0].to_vec(), size); + } + prev_locs[sectype] = loc + size; + } + + if !ifdefed { + all_text_glabels.extend(text_glabels.iter().map(|x| x.to_vec())); + all_late_rodata_dummy_bytes.push(function.late_rodata_dummy_bytes.clone()); + all_jtbl_rodata_size.push(function.jtbl_rodata_size); + late_rodata_asm.extend(function.late_rodata_asm_conts.iter().cloned()); + for (sectype, (temp_name, _)) in function.data.iter() { + if let Some(temp_name) = temp_name { + asm.push(format!(".section {}", sectype)); + asm.push(format!("glabel {}_asm_start", temp_name)); + } + } + asm.push(".text".to_owned()); + asm.extend(function.asm_conts.iter().cloned()); + for (sectype, (temp_name, _)) in function.data.iter() { + if let Some(temp_name) = temp_name { + asm.push(format!(".section {}", sectype)); + asm.push(format!("glabel {}_asm_end", temp_name)); + } + } + } + } + + if !late_rodata_asm.is_empty() { + asm.push(".section .late_rodata".to_string()); + // Put some padding at the start to avoid conflating symbols with + // references to the whole section. + asm.push(".word 0, 0".to_string()); + asm.push(format!("glabel {}", late_rodata_source_name_start)); + asm.extend(late_rodata_asm.iter().cloned()); + asm.push(format!("glabel {}", late_rodata_source_name_end)); + } + + let temp_dir = TempDir::with_prefix("asm_processor")?; + + let obj_stem = objfile_path.file_stem().unwrap().to_str().unwrap(); + + let o_file_path = temp_dir + .path() + .join(format!("asm_processor_{}.o", obj_stem)); + let s_file_path = temp_dir + .path() + .join(format!("asm_processor_{}.s", obj_stem)); + { + let mut s_file = File::create(&s_file_path)?; + s_file.write_all(&output_enc.encode(asm_prelude)?)?; + s_file.write_all(&output_enc.encode("\n")?)?; + + for line in asm { + s_file.write_all(&output_enc.encode(&line)?)?; + s_file.write_all(&output_enc.encode("\n")?)?; + } + } + + let status = Command::new("sh") + .arg("-c") + .arg(format!( + "{} {} -o {}", + assembler, + shlex::try_quote(s_file_path.to_str().unwrap()).unwrap(), + shlex::try_quote(o_file_path.to_str().unwrap()).unwrap(), + )) + .status() + .expect("Failed to run shell"); + if !status.success() { + return Err(anyhow::anyhow!("Failed to assemble")); + } + let asm_objfile = ElfFile::new(&fs::read(&o_file_path)?)?; + + // Remove clutter from objdump output for tests, and make the tests + // portable by avoiding absolute paths. Outside of tests .mdebug is + // useful for showing source together with asm, though. + let mdebug_section = objfile.find_section(".mdebug").cloned(); + if drop_mdebug_gptab { + objfile.drop_mdebug_gptab(); + } + + // Unify reginfo sections + if let Some(target_reginfo) = objfile.find_section_mut(".reginfo") { + let source_reginfo = &asm_objfile + .find_section(".reginfo") + .expect("couldn't find source .reginfo"); + for (s, t) in source_reginfo + .data + .iter() + .zip(target_reginfo.data.iter_mut()) + { + *t |= *s; + } + } + + // Move over section contents + let mut modified_text_positions = HashSet::new(); + let mut jtbl_rodata_positions: HashSet<usize> = HashSet::new(); + let mut last_rodata_pos = 0; + for sectype in OUTPUT_SECTIONS { + if to_copy[sectype].is_empty() { + continue; + } + let Some(source) = asm_objfile.find_section(sectype.as_str()) else { + panic!("didn't find source section: {}", sectype); + }; + for &ToCopyData { + loc, + size, + ref temp_name, + ref fn_desc, + } in to_copy[sectype].iter() + { + let loc1 = asm_objfile + .find_symbol_in_section(format!("{}_asm_start", &temp_name).as_bytes(), source); + let loc2 = asm_objfile + .find_symbol_in_section(format!("{}_asm_end", &temp_name).as_bytes(), source); + if loc1 != loc { + panic!( + "assembly and C files don't line up for section {}, {}", + sectype, fn_desc + ); + } + if loc2 - loc1 != size { + return Err(anyhow::anyhow!( + "incorrectly computed size for section {}, {}. If using .double, make sure to provide explicit alignment padding.", + sectype, + fn_desc + )); + } + } + + if sectype == OutputSection::Bss { + continue; + } + + let Some(target) = objfile.find_section_mut(sectype.as_str()) else { + panic!("didn't find target section: {}", sectype); + }; + + for &ToCopyData { loc, size, .. } in to_copy[sectype].iter() { + target.data[loc..loc + size].copy_from_slice(&source.data[loc..loc + size]); + + if sectype == OutputSection::Text { + assert_eq!(size % 4, 0); + assert_eq!(loc % 4, 0); + for j in 0..size / 4 { + modified_text_positions.insert(loc + 4 * j); + } + } else if sectype == OutputSection::Rodata { + last_rodata_pos = loc + size; + } + } + } + + // Move over late rodata. This is heuristic, sadly, since I can't think + // of another way of doing it. + let mut moved_late_rodata: HashMap<usize, usize> = HashMap::new(); + if all_late_rodata_dummy_bytes.iter().any(|b| !b.is_empty()) + || all_jtbl_rodata_size.iter().any(|&s| s > 0) + { + let source = asm_objfile + .find_section(".late_rodata") + .expect(".late_rodata source section should exist"); + let target = objfile + .find_section_mut(".rodata") + .expect(".rodata target section should exist"); + let mut source_pos = + asm_objfile.find_symbol_in_section(late_rodata_source_name_start.as_bytes(), source); + let source_end = + asm_objfile.find_symbol_in_section(late_rodata_source_name_end.as_bytes(), source); + let num_dummies: usize = all_late_rodata_dummy_bytes.iter().map(|x| x.len()).sum(); + let expected_size = num_dummies * 4 + all_jtbl_rodata_size.iter().sum::<usize>(); + + if source_end - source_pos != expected_size { + return Err(anyhow::anyhow!("computed wrong size of .late_rodata")); + } + let mut new_data = target.data.clone(); + + for (dummy_bytes_list, &jtbl_rodata_size) in all_late_rodata_dummy_bytes + .iter_mut() + .zip(all_jtbl_rodata_size.iter()) + { + let dummy_bytes_list_len = dummy_bytes_list.len(); + + for (index, dummy_bytes) in dummy_bytes_list.iter_mut().enumerate() { + if endian == Endian::Little { + dummy_bytes.reverse(); + } + + let mut pos = target.data[last_rodata_pos..] + .windows(4) + .position(|x| x == dummy_bytes) + .expect("failed to find dummy .late_rodata bytes") + + last_rodata_pos; + + if index == 0 + && dummy_bytes_list_len > 1 + && target.data[pos + 4..pos + 8] == *b"\0\0\0\0" + { + // Ugly hack to handle double alignment for non-matching builds. + // We were told by .late_rodata_alignment (or deduced from a .double) + // that a function's late_rodata started out 4 (mod 8), and emitted + // a float and then a double. But it was actually 0 (mod 8), so our + // double was moved by 4 bytes. To make them adjacent to keep jump + // tables correct, move the float by 4 bytes as well. + new_data[pos..pos + 4].copy_from_slice(b"\0\0\0\0"); + pos += 4; + } + new_data[pos..pos + 4].copy_from_slice(&source.data[source_pos..source_pos + 4]); + moved_late_rodata.insert(source_pos, pos); + last_rodata_pos = pos + 4; + source_pos += 4; + } + + if jtbl_rodata_size > 0 { + assert!(!dummy_bytes_list.is_empty()); + let pos = last_rodata_pos; + new_data[pos..pos + jtbl_rodata_size] + .copy_from_slice(&source.data[source_pos..source_pos + jtbl_rodata_size]); + for i in (0..jtbl_rodata_size).step_by(4) { + moved_late_rodata.insert(source_pos + i, pos + i); + jtbl_rodata_positions.insert(pos + i); + } + last_rodata_pos += jtbl_rodata_size; + source_pos += jtbl_rodata_size; + } + } + target.data = new_data; + } + + // Merge strtab data. + let strtab = objfile.sym_strtab_mut(); + let strtab_adj = strtab.data.len(); + strtab.data.extend(&asm_objfile.sym_strtab().data); + + // Find relocated symbols in asm_objfile + let mut relocated_symbols = HashSet::new(); + for sectype in INPUT_SECTION_NAMES.iter() { + if let Some(sec) = asm_objfile.find_section(sectype) { + for reltab_idx in &sec.relocated_by { + let reltab = &asm_objfile.sections[*reltab_idx]; + for rel in &reltab.relocations { + relocated_symbols.insert(rel.sym_index); + } + } + } + } + + enum SymInd { + Obj(usize), + Asm(usize), + } + + // Move over symbols, deleting the temporary function labels. + // Skip over new local symbols that aren't relocated against, to + // avoid conflicts. + let empty_symbol = objfile.symbol_entries[0].clone(); + let mut new_syms: Vec<(Symbol, Vec<SymInd>)> = objfile + .symbol_entries + .iter() + .enumerate() + .map(|(i, x)| (x, vec![SymInd::Obj(i)])) + .skip(1) + .filter(|(x, _)| !x.name.starts_with(b"_asmpp_")) + .map(|(x, inds)| (x.clone(), inds)) + .collect(); + + for (i, s) in asm_objfile.symbol_entries.iter().enumerate() { + let is_local = i < asm_objfile.symtab().header.sh_info as usize; + if is_local && !relocated_symbols.contains(&i) { + continue; + } + if s.name.starts_with(b"_asmpp_") { + assert!(!relocated_symbols.contains(&i)); + continue; + } + let mut s = s.clone(); + if s.st_shndx != SHN_UNDEF && s.st_shndx != SHN_ABS { + let section_name = asm_objfile.sections[s.st_shndx].name.clone(); + let mut target_section_name = section_name.clone(); + if section_name == ".late_rodata" { + target_section_name = ".rodata".to_string(); + } else if !INPUT_SECTION_NAMES.contains(§ion_name.as_str()) { + return Err(anyhow::anyhow!( + "generated assembly .o must only have symbols for .text, .data, .rodata, .late_rodata, ABS and UNDEF, but found {}", + section_name + )); + } + let Some(objfile_section) = objfile.find_section(&target_section_name) else { + return Err(anyhow::anyhow!( + "generated assembly .o has section that real objfile lacks: {}", + target_section_name + )); + }; + s.st_shndx = objfile_section.index; + // glabels aren't marked as functions, making objdump output confusing. Fix that. + if all_text_glabels.contains(&s.name) { + s.st_type = STT_FUNC; + if let Some(&size) = func_sizes.get(&s.name) { + s.st_size = size; + } + } + if section_name == ".late_rodata" { + if s.st_value == 0 { + // This must be a symbol corresponding to the whole .late_rodata + // section, being referred to from a relocation. + // Moving local symbols is tricky, because it requires fixing up + // lo16/hi16 relocation references to .late_rodata+<offset>. + // Just disallow it for now. + return Err(anyhow::anyhow!( + "local symbols in .late_rodata are not allowed" + )); + } + s.st_value = moved_late_rodata[&s.st_value]; + } + } + s.st_name += strtab_adj; + new_syms.push((s, vec![SymInd::Asm(i)])); + } + + // Add static symbols from .mdebug, so they can be referred to from GLOBAL_ASM + if mdebug_section.is_some() && convert_statics != ConvertStatics::No { + let mdebug_section = mdebug_section.unwrap(); + let mut static_name_count: HashMap<Vec<u8>, usize> = HashMap::new(); + let mut strtab_index = objfile.sym_strtab().data.len(); + let mut new_strtab_data = vec![]; + + let read_u32 = |data: &[u8], offset| { + u32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()) as usize + }; + + let ifd_max = read_u32(&mdebug_section.data, 18 * 4); + let cb_fd_offset = read_u32(&mdebug_section.data, 19 * 4); + let cb_sym_offset = read_u32(&mdebug_section.data, 9 * 4); + let cb_ss_offset = read_u32(&mdebug_section.data, 15 * 4); + + for i in 0..ifd_max { + let offset = cb_fd_offset + 18 * 4 * i; + let iss_base = read_u32(&objfile.data, offset + 2 * 4); + let isym_base = read_u32(&objfile.data, offset + 4 * 4); + let csym = read_u32(&objfile.data, offset + 5 * 4); + let mut scope_level = 0; + + for j in 0..csym { + let offset2 = cb_sym_offset + 12 * (isym_base + j); + let iss = read_u32(&objfile.data, offset2); + let value = read_u32(&objfile.data, offset2 + 4); + let st_sc_index = read_u32(&objfile.data, offset2 + 8); + let st = st_sc_index >> 26; + let sc = (st_sc_index >> 21) & 0x1F; + + if st == MIPS_DEBUG_ST_STATIC || st == MIPS_DEBUG_ST_STATIC_PROC { + let symbol_name_offset = cb_ss_offset + iss_base + iss; + let symbol_name_offset_end = objfile_data[symbol_name_offset..] + .iter() + .position(|x| *x == 0) + .expect("bad .mdebug strtab reference") + + symbol_name_offset; + let mut symbol_name = + objfile_data[symbol_name_offset..symbol_name_offset_end].to_owned(); + if scope_level > 1 { + // For in-function statics, append an increasing counter to + // the name, to avoid duplicate conflicting symbols. + let count = static_name_count.get(&symbol_name).unwrap_or(&0) + 1; + static_name_count.insert(symbol_name.clone(), count); + symbol_name.extend(format!(":{}", count).as_bytes()); + } + let mut emitted_symbol_name = symbol_name.clone(); + if convert_statics == ConvertStatics::GlobalWithFilename { + // Change the emitted symbol name to include the filename, + // but don't let that affect deduplication logic (we still + // want to be able to reference statics from GLOBAL_ASM). + let mut new_name = objfile_path.to_string_lossy().into_owned().into_bytes(); + new_name.push(b':'); + new_name.extend(emitted_symbol_name); + emitted_symbol_name = new_name; + }; + let section_name = match sc { + 1 => ".text", + 2 => ".data", + 3 => ".bss", + 15 => ".rodata", + _ => { + return Err(anyhow::anyhow!("unsupported MIPS_DEBUG_SC value: {}", sc)); + } + }; + let Some(section) = objfile.find_section(section_name) else { + panic!( + "couldn't find section referenced from .mdebug: {}", + section_name + ); + }; + let symtype = if sc == 1 { STT_FUNC } else { STT_OBJECT }; + let binding = match convert_statics { + ConvertStatics::Global | ConvertStatics::GlobalWithFilename => STB_GLOBAL, + _ => STB_LOCAL, + }; + let sym = Symbol { + st_name: strtab_index, + st_value: value, + st_size: 0, + st_bind: binding, + st_type: symtype, + st_visibility: STV_DEFAULT, + st_shndx: section.index, + name: symbol_name, + }; + strtab_index += emitted_symbol_name.len() + 1; + new_strtab_data.extend(&emitted_symbol_name); + new_strtab_data.push(b'\0'); + new_syms.push((sym, vec![])); + } + match st { + MIPS_DEBUG_ST_FILE + | MIPS_DEBUG_ST_STRUCT + | MIPS_DEBUG_ST_UNION + | MIPS_DEBUG_ST_ENUM + | MIPS_DEBUG_ST_BLOCK + | MIPS_DEBUG_ST_PROC + | MIPS_DEBUG_ST_STATIC_PROC => { + scope_level += 1; + } + MIPS_DEBUG_ST_END => { + scope_level -= 1; + } + _ => {} + } + } + assert_eq!(scope_level, 0); + } + + objfile.sym_strtab_mut().data.extend(new_strtab_data); + } + + // Get rid of duplicate symbols, favoring ones that are not UNDEF. + // Skip this for unnamed local symbols though. + new_syms.sort_by(|(a, _), (b, _)| { + if a.st_shndx != SHN_UNDEF && b.st_shndx == SHN_UNDEF { + Ordering::Less + } else { + Ordering::Greater + } + }); + + let new_syms_prev = new_syms; + let mut new_syms = vec![]; + let mut name_to_sym = HashMap::new(); + for (mut s, inds) in new_syms_prev { + if s.name == b"_gp_disp" { + s.st_type = STT_OBJECT; + } + if s.st_bind == STB_LOCAL && s.st_shndx == SHN_UNDEF { + return Err(anyhow::anyhow!( + "local symbol \"{}\" is undefined", + String::from_utf8_lossy(&s.name) + )); + } + if s.name.is_empty() { + if s.st_bind != STB_LOCAL { + return Err(anyhow::anyhow!("global symbol with no name")); + } + new_syms.push((s.clone(), inds)); + } else { + match name_to_sym.get(&s.name) { + None => { + name_to_sym.insert(s.name.clone(), new_syms.len()); + new_syms.push((s.clone(), inds)); + } + Some(&existing) => { + let (s2, inds2) = &mut new_syms[existing]; + if s.st_shndx != SHN_UNDEF + && !(s2.st_shndx == s.st_shndx && s2.st_value == s.st_value) + { + return Err(anyhow::anyhow!( + "symbol \"{}\" defined twice", + String::from_utf8_lossy(&s.name) + )); + } + inds2.extend(inds); + } + } + } + } + + // Put local symbols in front, with the initial dummy entry first, and + // _gp_disp at the end if it exists. + new_syms.insert(0, (empty_symbol.clone(), vec![])); + new_syms.sort_by_key(|(a, _)| (a.st_bind != STB_LOCAL, a.name == b"_gp_disp")); + + let mut obj_new_index: HashMap<usize, usize> = HashMap::new(); + let mut asm_new_index: HashMap<usize, usize> = HashMap::new(); + + for (new_index, (_, inds)) in new_syms.iter().enumerate() { + for i in inds { + match i { + SymInd::Obj(i) => obj_new_index.insert(*i, new_index), + SymInd::Asm(i) => asm_new_index.insert(*i, new_index), + }; + } + } + + let new_syms: Vec<_> = new_syms.iter().map(|(s, _)| s).collect(); + let num_local_syms = new_syms.iter().filter(|s| s.st_bind == STB_LOCAL).count(); + let new_sym_data: Vec<u8> = new_syms.iter().flat_map(|s| s.to_bin()).collect(); + + objfile.symtab_mut().data = new_sym_data; + objfile.symtab_mut().header.sh_info = num_local_syms as u32; + + // Fix up relocation symbol references + for sectype in OUTPUT_SECTIONS { + let target = objfile.find_section(sectype.as_str()).cloned(); + + if let Some(target) = target { + // fixup relocation symbol indices, since we butchered them above + for reltab in target.relocated_by.iter() { + let reltab = &mut objfile.sections[*reltab]; + let mut nrels = vec![]; + for rel in reltab.relocations.iter() { + let mut rel = rel.clone(); + if (sectype == OutputSection::Text + && modified_text_positions.contains(&rel.r_offset)) + || (sectype == OutputSection::Rodata + && jtbl_rodata_positions.contains(&rel.r_offset)) + { + // don't include relocations for late_rodata dummy code + continue; + } + rel.sym_index = obj_new_index[&rel.sym_index]; + nrels.push(rel); + } + reltab.data = nrels.iter().flat_map(|x| x.to_bin(endian)).collect(); + reltab.relocations = nrels; + } + } + } + + // Move over relocations + for sectype in INPUT_SECTION_NAMES.iter() { + if let Some(source) = asm_objfile.find_section(sectype) { + if source.data.is_empty() { + continue; + } + + let target_sectype = if *sectype == ".late_rodata" { + ".rodata" + } else { + sectype + }; + let target_index = objfile + .find_section(target_sectype) + .expect("didn't find target section") + .index; + for reltab in &source.relocated_by { + let reltab = &mut asm_objfile.sections[*reltab].clone(); + for rel in &mut reltab.relocations { + rel.sym_index = asm_new_index[&rel.sym_index]; + if *sectype == ".late_rodata" { + rel.r_offset = moved_late_rodata[&rel.r_offset]; + } + } + let new_data: Vec<u8> = reltab + .relocations + .iter() + .flat_map(|x| x.to_bin(endian)) + .collect(); + + let (prefix, sh_entsize) = if reltab.header.sh_type == SHT_REL { + (".rel", 8) + } else { + (".rela", 12) + }; + let rel_section_name = format!("{}{}", prefix, target_sectype); + + if let Some(target_reltab) = objfile.find_section_mut(&rel_section_name) { + target_reltab.data.extend(new_data); + } else { + objfile.add_section( + &rel_section_name, + &HeaderFields { + sh_type: reltab.header.sh_type, + sh_flags: 0, + sh_link: objfile.symtab().index as u32, + sh_info: target_index as u32, + sh_addralign: 4, + sh_entsize, + }, + &new_data, + endian, + ); + } + } + } + } + + let mut file = std::fs::File::create(objfile_path).expect("unable to write to .o file"); + let mut writer = BufWriter::new(&mut file); + objfile.write(&mut writer)?; + + fs::remove_file(s_file_path)?; + fs::remove_file(o_file_path)?; + Ok(()) +} diff --git a/tools/asm-processor/rust/src/preprocess.rs b/tools/asm-processor/rust/src/preprocess.rs new file mode 100644 index 0000000..73af653 --- /dev/null +++ b/tools/asm-processor/rust/src/preprocess.rs @@ -0,0 +1,969 @@ +use std::{fs, io::Write, iter, path::Path, sync::OnceLock}; + +use anyhow::Result; +use enum_map::{Enum, EnumMap}; +use regex_lite::Regex; + +use crate::{AsmProcArgs, CompileOpts, Encoding, Function, OptLevel, OutputSection}; + +#[derive(Copy, Clone, Eq, PartialEq, Debug, Enum)] +enum InputSection { + Text, + Data, + Rodata, + LateRodata, + Bss, +} + +impl InputSection { + fn from_str(name: &str) -> Option<InputSection> { + match name { + ".text" => Some(InputSection::Text), + ".data" => Some(InputSection::Data), + ".rodata" => Some(InputSection::Rodata), + ".late_rodata" => Some(InputSection::LateRodata), + ".bss" => Some(InputSection::Bss), + _ => None, + } + } +} + +#[derive(Clone, Debug)] +struct GlobalState { + late_rodata_hex: u32, + valuectr: usize, + namectr: usize, + min_instr_count: usize, + skip_instr_count: usize, + use_jtbl_for_rodata: bool, + prelude_if_late_rodata: usize, + mips1: bool, + pascal: bool, +} + +impl GlobalState { + fn new( + min_instr_count: usize, + skip_instr_count: usize, + use_jtbl_for_rodata: bool, + prelude_if_late_rodata: usize, + mips1: bool, + pascal: bool, + ) -> Self { + Self { + // A value that hopefully never appears as a 32-bit rodata constant (or we + // miscompile late rodata). Increases by 1 in each step. + late_rodata_hex: 0xE0123456, + valuectr: 0, + namectr: 0, + min_instr_count, + skip_instr_count, + use_jtbl_for_rodata, + prelude_if_late_rodata, + mips1, + pascal, + } + } + + fn next_late_rodata_hex(&mut self) -> [u8; 4] { + let dummy_bytes = self.late_rodata_hex.to_be_bytes(); + if (self.late_rodata_hex & 0xffff) == 0 { + // Avoid lui + self.late_rodata_hex += 1; + } + self.late_rodata_hex += 1; + dummy_bytes + } + + fn make_name(&mut self, cat: &str) -> String { + self.namectr += 1; + format!("_asmpp_{}{}", cat, self.namectr) + } + + fn func_prologue(&self, name: &str) -> String { + if self.pascal { + [ + &format!("procedure {}();", name), + "type", + " pi = ^integer;", + " pf = ^single;", + " pd = ^double;", + "var", + " vi: pi;", + " vf: pf;", + " vd: pd;", + "begin", + " vi := vi;", + " vf := vf;", + " vd := vd;", + ] + .join(" ") + } else { + format!("void {}(void) {{", name) + } + } + + fn func_epilogue(&self) -> String { + if self.pascal { + "end;".to_string() + } else { + '}'.to_string() + } + } + + fn pascal_assignment_float(&mut self, val: f32) -> String { + self.valuectr += 1; + let address = (8 * self.valuectr) & 0x7FFF; + format!("vf := pf({}); vf^ := {:?};", address, val) + } + + fn pascal_assignment_double(&mut self, val: f64) -> String { + self.valuectr += 1; + let address = (8 * self.valuectr) & 0x7FFF; + format!("vd := pd({}); vd^ := {:?};", address, val) + } + + fn pascal_assignment_int(&mut self, val: i32) -> String { + self.valuectr += 1; + let address = (8 * self.valuectr) & 0x7FFF; + format!("vi := pi({}); vi^ := {};", address, val) + } +} + +#[derive(Clone, Debug)] +struct GlobalAsmBlock { + fn_desc: String, + cur_section: InputSection, + asm_conts: Vec<String>, + late_rodata_asm_conts: Vec<String>, + late_rodata_alignment: usize, + late_rodata_alignment_from_context: bool, + text_glabels: Vec<String>, + fn_section_sizes: EnumMap<InputSection, usize>, + fn_ins_inds: Vec<(usize, usize)>, + glued_line: String, + num_lines: usize, +} + +impl GlobalAsmBlock { + fn new(fn_desc: String) -> Self { + Self { + fn_desc, + cur_section: InputSection::Text, + asm_conts: vec![], + late_rodata_asm_conts: vec![], + late_rodata_alignment: 0, + late_rodata_alignment_from_context: false, + text_glabels: vec![], + fn_section_sizes: EnumMap::default(), + fn_ins_inds: vec![], + glued_line: String::new(), + num_lines: 0, + } + } + + fn fail_without_line<T>(&self, msg: &str) -> Result<T> { + Err(anyhow::anyhow!("{}\nwithin {}", msg, self.fn_desc)) + } + + fn fail_at_line<T>(&self, msg: &str, line: &str) -> Result<T> { + Err(anyhow::anyhow!( + "{}\nwithin {} at line {}", + msg, + self.fn_desc, + line + )) + } + + fn count_quoted_size( + &self, + line: &str, + z: bool, + real_line: &str, + output_enc: &Encoding, + ) -> Result<usize> { + let line = output_enc.encode(line)?; + + let mut in_quote = false; + let mut has_comma = true; + let mut num_parts = 0; + let mut ret = 0; + let mut i = 0; + let digits = b"0123456789"; // 0-7 would be more sane, but this matches GNU as + let hexdigits = b"0123456789abcdefABCDEF"; + + while i < line.len() { + let c = line[i]; + i += 1; + if !in_quote { + if c == b'"' { + in_quote = true; + if z && !has_comma { + return self.fail_at_line(".asciiz with glued strings is not supported due to GNU as version diffs", real_line); + } + num_parts += 1; + } else if c == b',' { + has_comma = true; + } + } else { + if c == b'"' { + in_quote = false; + has_comma = false; + continue; + } + ret += 1; + if c != b'\\' { + continue; + } + if i == line.len() { + return self.fail_at_line("backslash at end of line not supported", real_line); + } + let c = line[i]; + i += 1; + // (if c is in "bfnrtv", we have a real escaped literal) + if c == b'x' { + // hex literal, consume any number of hex chars, possibly none + while i < line.len() && hexdigits.contains(&line[i]) { + i += 1; + } + } else if digits.contains(&c) { + // octal literal, consume up to two more digits + let mut it = 0; + while i < line.len() && digits.contains(&line[i]) && it < 2 { + i += 1; + it += 1; + } + } + } + } + + if in_quote { + return self.fail_at_line("unterminated string literal", real_line); + } + if num_parts == 0 { + return self.fail_at_line(".ascii with no string", real_line); + } + Ok(ret + if z { num_parts } else { 0 }) + } + + fn align(&mut self, n: usize) { + let size = &mut self.fn_section_sizes[self.cur_section]; + while *size % n != 0 { + *size += 1; + } + } + + fn add_sized(&mut self, size: isize, line: &str) -> Result<()> { + if (self.cur_section == InputSection::Text || self.cur_section == InputSection::LateRodata) + && size % 4 != 0 + { + return self.fail_at_line("size must be a multiple of 4", line); + } + + if size < 0 { + return self.fail_at_line("size cannot be negative", line); + } + + self.fn_section_sizes[self.cur_section] += size as usize; + + if self.cur_section == InputSection::Text { + if self.text_glabels.is_empty() { + return self.fail_at_line(".text block without an initial glabel", line); + } + self.fn_ins_inds + .push((self.num_lines - 1, size as usize / 4)); + } + + Ok(()) + } + + fn process_line(&mut self, line: &str, output_enc: &Encoding) -> Result<()> { + self.num_lines += 1; + if let Some(stripped) = line.strip_suffix("\\") { + self.glued_line = format!("{}{}", self.glued_line, stripped); + return Ok(()); + } + let mut line = self.glued_line.clone() + line; + self.glued_line = String::new(); + + static CACHE: OnceLock<(Regex, Regex)> = OnceLock::new(); + let (re_comment_or_string, re_label) = CACHE.get_or_init(|| { + ( + Regex::new(r#"#.*|/\*.*?\*/|"(?:\\.|[^\\"])*""#).unwrap(), + Regex::new(r"^[a-zA-Z0-9_]+:\s*").unwrap(), + ) + }); + + fn re_comment_replacer(caps: ®ex_lite::Captures) -> String { + let s = caps[0].to_string(); + if s.starts_with("/") || s.starts_with("#") { + " ".to_owned() + } else { + s + } + } + + let real_line = line.clone(); + line = re_comment_or_string + .replace_all(&line, re_comment_replacer) + .into_owned(); + line = line.trim().to_string(); + line = re_label.replace_all(&line, "").into_owned(); + let mut changed_section = false; + let mut emitting_double = false; + + if (line.starts_with("glabel ") || line.starts_with("jlabel ")) + && self.cur_section == InputSection::Text + { + self.text_glabels + .push(line.split_whitespace().nth(1).unwrap().to_string()); + } + if line.is_empty() { + // empty line + } else if line.starts_with("glabel ") + || line.starts_with("dlabel ") + || line.starts_with("jlabel ") + || line.starts_with("alabel ") + || line.starts_with("endlabel ") + || line.starts_with("enddlabel ") + || line.starts_with("nonmatching ") + || (!line.contains(" ") && line.ends_with(":")) + { + // label + } else if line.starts_with(".section") + || matches!( + line.as_str(), + ".text" | ".data" | ".rdata" | ".rodata" | ".bss" | ".late_rodata" + ) + { + // section change + self.cur_section = if line == ".rdata" { + InputSection::Rodata + } else { + let first_arg = line.split(',').next().unwrap().to_string(); + let name = first_arg.split_whitespace().last().unwrap(); + match InputSection::from_str(name) { + Some(s) => s, + None => { + return self.fail_at_line("unrecognized .section directive", &real_line) + } + } + }; + + changed_section = true; + } else if line.starts_with(".late_rodata_alignment") { + if self.cur_section != InputSection::LateRodata { + return self.fail_at_line( + ".late_rodata_alignment must occur within .late_rodata section", + &real_line, + ); + } + + let value = line.split_whitespace().nth(1).unwrap().parse::<usize>()?; + if value != 4 && value != 8 { + return self + .fail_at_line(".late_rodata_alignment argument must be 4 or 8", &real_line); + } + if self.late_rodata_alignment != 0 && self.late_rodata_alignment != value { + return self.fail_without_line( + ".late_rodata_alignment alignment assumption conflicts with earlier .double directive. Make sure to provide explicit alignment padding." + ); + } + self.late_rodata_alignment = value; + changed_section = true; + } else if line.starts_with(".incbin") { + let size = line + .split(',') + .next_back() + .unwrap() + .trim() + .parse::<isize>()?; + self.add_sized(size, &real_line)?; + } else if line.starts_with(".word") + || line.starts_with(".gpword") + || line.starts_with(".float") + { + self.align(4); + + self.add_sized(4 * line.split(',').count() as isize, &real_line)?; + } else if line.starts_with(".double") { + self.align(4); + + if self.cur_section == InputSection::LateRodata { + let align8 = self.fn_section_sizes[self.cur_section] % 8; + // Automatically set late_rodata_alignment, so the generated C code uses doubles. + // This gives us correct alignment for the transferred doubles even when the + // late_rodata_alignment is wrong, e.g. for non-matching compilation. + if self.late_rodata_alignment == 0 { + self.late_rodata_alignment = 8 - align8; + self.late_rodata_alignment_from_context = true; + } else if self.late_rodata_alignment != 8 - align8 { + if self.late_rodata_alignment_from_context { + return self.fail_at_line( + "found two .double directives with different start addresses mod 8. Make sure to provide explicit alignment padding.", + &real_line + ); + } else { + return self.fail_at_line( + ".double at address that is not 0 mod 8 (based on .late_rodata_alignment assumption). Make sure to provide explicit alignment padding.\n{}", + &real_line + ); + } + } + + self.add_sized(8 * line.split(',').count() as isize, &real_line)?; + emitting_double = true; + } + } else if line.starts_with(".space") { + let size = line.split_whitespace().nth(1).unwrap().parse::<isize>()?; + self.add_sized(size, &real_line)?; + } else if line.starts_with(".balign") { + let align = line.split_whitespace().nth(1).unwrap().parse::<isize>()?; + if align != 4 { + return self.fail_at_line("only .balign 4 is supported", &real_line); + } + self.align(4); + } else if line.starts_with(".align") { + let align = line.split_whitespace().nth(1).unwrap().parse::<isize>()?; + if align != 2 { + return self.fail_at_line("only .align 2 is supported", &real_line); + } + self.align(4); + } else if line.starts_with(".asci") { + let z = line.starts_with(".asciz") || line.starts_with(".asciiz"); + self.add_sized( + self.count_quoted_size(&line, z, &real_line, output_enc)? as isize, + &real_line, + )?; + } else if line.starts_with(".byte") { + self.add_sized(line.split(',').count() as isize, &real_line)?; + } else if line.starts_with(".half") + || line.starts_with(".hword") + || line.starts_with(".short") + { + self.align(2); + self.add_sized(2 * line.split(',').count() as isize, &real_line)?; + } else if line.starts_with(".size") { + } else if line.starts_with('.') { + return self.fail_at_line("asm directive not supported", &real_line); + } else { + // Unfortunately, macros are hard to support for .rodata -- + // we don't know how how space they will expand to before + // running the assembler, but we need that information to + // construct the C code. So if we need that we'll either + // need to run the assembler twice (at least in some rare + // cases), or change how this program is invoked. + // Similarly, we can't currently deal with pseudo-instructions + // that expand to several real instructions. + if self.cur_section != InputSection::Text { + return self.fail_at_line( + "instruction or macro call in non-.text section? not supported", + &real_line, + ); + } + self.add_sized(4, &real_line)?; + } + + if self.cur_section == InputSection::LateRodata { + if !changed_section { + if emitting_double { + self.late_rodata_asm_conts.push(".align 0".to_string()); + } + self.late_rodata_asm_conts.push(real_line.clone()); + if emitting_double { + self.late_rodata_asm_conts.push(".align 2".to_string()); + } + } + } else { + self.asm_conts.push(real_line.clone()); + } + + Ok(()) + } + + const MAX_FN_SIZE: usize = 100; + + fn finish(&self, state: &mut GlobalState) -> Result<(Vec<String>, Function)> { + let mut src = vec!["".to_owned(); self.num_lines + 1]; + let mut late_rodata_dummy_bytes = vec![]; + let mut jtbl_rodata_size = 0; + let mut late_rodata_fn_output = vec![]; + + let num_instr = self.fn_section_sizes[InputSection::Text] / 4; + + if self.fn_section_sizes[InputSection::LateRodata] > 0 { + // Generate late rodata by emitting unique float constants. + // This requires 3 instructions for each 4 bytes of rodata. + // If we know alignment, we can use doubles, which give 3 + // instructions for 8 bytes of rodata. + let size = self.fn_section_sizes[InputSection::LateRodata] / 4; + let mut skip_next = false; + let mut needs_double = self.late_rodata_alignment != 0; + let mut extra_mips1_nop = false; + let (jtbl_size, jtbl_min_rodata_size) = match (state.pascal, state.mips1) { + (true, true) => (9, 2), + (true, false) => (8, 2), + (false, true) => (11, 5), + (false, false) => (9, 5), + }; + + for i in 0..size { + if skip_next { + skip_next = false; + continue; + } + // Jump tables give 9 instructions (11 with -mips1) for >= 5 words of rodata, + // and should be emitted when: + // - -O2 or -O2 -g3 are used, which give the right codegen + // - we have emitted our first .float/.double (to ensure that we find the + // created rodata in the binary) + // - we have emitted our first .double, if any (to ensure alignment of doubles + // in shifted rodata sections) + // - we have at least 5 words of rodata left to emit (otherwise IDO does not + // generate a jump table) + // - we have at least 10 more instructions to go in this function (otherwise our + // function size computation will be wrong since the delay slot goes unused) + if !needs_double + && state.use_jtbl_for_rodata + && i >= 1 + && size - i >= jtbl_min_rodata_size + && num_instr - late_rodata_fn_output.len() >= jtbl_size + 1 + { + let line = if state.pascal { + let cases: String = (0..(size - i)) + .map(|case| format!("{}: ;", case)) + .collect::<Vec<String>>() + .join("\n"); + format!("case 0 of {} otherwise end;", cases) + } else { + let cases: String = (0..(size - i)) + .map(|case| format!("case {}:", case)) + .collect::<Vec<String>>() + .join(" "); + format!("switch (*(volatile int*)0) {{ {} ; }}", cases) + }; + late_rodata_fn_output.push(line); + late_rodata_fn_output.extend(iter::repeat_n("".to_owned(), jtbl_size - 1)); + jtbl_rodata_size = (size - i) * 4; + extra_mips1_nop = i != 2; + break; + } + + let dummy_bytes = state.next_late_rodata_hex(); + late_rodata_dummy_bytes.push(dummy_bytes); + if self.late_rodata_alignment == 4 * ((i + 1) % 2 + 1) && i + 1 < size { + let dummy_bytes2 = state.next_late_rodata_hex(); + late_rodata_dummy_bytes.push(dummy_bytes2); + let combined = [dummy_bytes, dummy_bytes2].concat().try_into().unwrap(); + let fval = f64::from_be_bytes(combined); + let line = if state.pascal { + state.pascal_assignment_double(fval) + } else { + format!("*(volatile double*)0 = {:?};", fval) + }; + late_rodata_fn_output.push(line); + skip_next = true; + needs_double = false; + if state.mips1 { + // mips1 does not have ldc1/sdc1 + late_rodata_fn_output.push("".to_owned()); + late_rodata_fn_output.push("".to_owned()); + } + extra_mips1_nop = false; + } else { + let fval = f32::from_be_bytes(dummy_bytes); + let line = if state.pascal { + state.pascal_assignment_float(fval) + } else { + format!("*(volatile float*)0 = {:?}f;", fval) + }; + late_rodata_fn_output.push(line); + extra_mips1_nop = true; + } + late_rodata_fn_output.push("".to_owned()); + late_rodata_fn_output.push("".to_owned()); + } + + if state.mips1 && extra_mips1_nop { + late_rodata_fn_output.push("".to_owned()); + } + } + + let mut text_name = None; + if self.fn_section_sizes[InputSection::Text] > 0 || !late_rodata_fn_output.is_empty() { + let new_name = state.make_name("func"); + src[0] = state.func_prologue(&new_name); + text_name = Some(new_name); + src[self.num_lines] = state.func_epilogue(); + let instr_count = self.fn_section_sizes[InputSection::Text] / 4; + if instr_count < state.min_instr_count { + return self.fail_without_line("too short .text block"); + } + let mut tot_emitted = 0; + let mut tot_skipped = 0; + let mut fn_emitted = 0; + let mut fn_skipped = 0; + let mut skipping = true; + let mut rodata_stack: Vec<String> = late_rodata_fn_output.clone(); + rodata_stack.reverse(); + + for &(line, count) in &self.fn_ins_inds { + for _ in 0..count { + if fn_emitted > Self::MAX_FN_SIZE + && instr_count - tot_emitted > state.min_instr_count + && (rodata_stack.is_empty() || !rodata_stack.last().unwrap().is_empty()) + { + // Don't let functions become too large. When a function reaches 284 + // instructions, and -O2 -framepointer flags are passed, the IRIX + // compiler decides it is a great idea to start optimizing more. + // Also, Pascal cannot handle too large functions before it runs out + // of unique statements to write. + fn_emitted = 0; + fn_skipped = 0; + skipping = true; + let large_func_name = state.make_name("large_func"); + src[line] += &format!( + " {} {} ", + state.func_epilogue(), + state.func_prologue(&large_func_name) + ); + } + + let skip_for_late_rodata = if !rodata_stack.is_empty() { + state.prelude_if_late_rodata + } else { + 0 + }; + if skipping && fn_skipped < state.skip_instr_count + skip_for_late_rodata { + fn_skipped += 1; + tot_skipped += 1; + } else { + skipping = false; + if let Some(entry) = rodata_stack.pop() { + src[line] += &entry; + } else if state.pascal { + src[line] += &state.pascal_assignment_int(0); + } else { + src[line] += "*(volatile int*)0 = 0;"; + } + } + tot_emitted += 1; + fn_emitted += 1; + } + } + + if !rodata_stack.is_empty() { + let size = late_rodata_fn_output.len() / 3; + let available = instr_count - tot_skipped; + return self.fail_without_line(&format!( + "late rodata to text ratio is too high: {} / {} must be <= 1/3\n + add .late_rodata_alignment (4|8) to the .late_rodata block + to double the allowed ratio.", + size, available + )); + } + } + + let mut rodata_name = None; + if self.fn_section_sizes[InputSection::Rodata] > 0 { + if state.pascal { + return self.fail_without_line(".rodata isn't supported with Pascal for now"); + } + let new_name = state.make_name("rodata"); + src[self.num_lines] += &format!( + " const char {}[{}] = {{1}};", + new_name, + self.fn_section_sizes[InputSection::Rodata] + ); + rodata_name = Some(new_name); + } + + let mut data_name = None; + if self.fn_section_sizes[InputSection::Data] > 0 { + let new_name = state.make_name("data"); + let line = if state.pascal { + format!( + " var {}: packed array[1..{}] of char := [otherwise: 0];", + new_name, + self.fn_section_sizes[InputSection::Data] + ) + } else { + format!( + " char {}[{}] = {{1}};", + new_name, + self.fn_section_sizes[InputSection::Data] + ) + }; + src[self.num_lines] += &line; + data_name = Some(new_name); + } + + let mut bss_name = None; + if self.fn_section_sizes[InputSection::Bss] > 0 { + let new_name = state.make_name("bss"); + if state.pascal { + return self.fail_without_line(".bss isn't supported with Pascal for now"); + } + src[self.num_lines] += &format!( + " char {}[{}];", + new_name, + self.fn_section_sizes[InputSection::Bss] + ); + bss_name = Some(new_name); + } + + let mut data = EnumMap::default(); + data[OutputSection::Text] = (text_name, self.fn_section_sizes[InputSection::Text]); + data[OutputSection::Data] = (data_name, self.fn_section_sizes[InputSection::Data]); + data[OutputSection::Rodata] = (rodata_name, self.fn_section_sizes[InputSection::Rodata]); + data[OutputSection::Bss] = (bss_name, self.fn_section_sizes[InputSection::Bss]); + let ret_fn = Function { + text_glabels: self.text_glabels.clone(), + asm_conts: self.asm_conts.clone(), + late_rodata_dummy_bytes, + jtbl_rodata_size, + late_rodata_asm_conts: self.late_rodata_asm_conts.clone(), + fn_desc: self.fn_desc.clone(), + data, + }; + + Ok((src, ret_fn)) + } +} + +/// Convert a float string to its hexadecimal representation +fn repl_float_hex(cap: ®ex_lite::Captures) -> String { + let float_str = cap[0].trim().trim_end_matches('f'); + let float_val = float_str.parse::<f32>().unwrap(); + let hex_val = f32::to_be_bytes(float_val); + format!("{}", u32::from_be_bytes(hex_val)) +} + +pub(crate) struct ParseSourceResult { + pub functions: Vec<Function>, + pub deps: Vec<String>, + pub output: Vec<u8>, +} + +pub(crate) fn parse_source( + infile_path: &Path, + args: &AsmProcArgs, + opts: &CompileOpts, + encode: bool, +) -> Result<ParseSourceResult> { + let (mut min_instr_count, mut skip_instr_count) = match opts.opt { + OptLevel::O0 => match opts.framepointer { + true => (8, 8), + false => (4, 4), + }, + OptLevel::O1 | OptLevel::O2 => match opts.framepointer { + true => (6, 5), + false => (2, 1), + }, + OptLevel::G => match opts.framepointer { + true => (7, 7), + false => (4, 4), + }, + OptLevel::G3 => match opts.framepointer { + true => (4, 4), + false => (2, 2), + }, + }; + + let mut prelude_if_late_rodata = 0; + if opts.kpic { + // Without optimizations, the PIC prelude always takes up 3 instructions. + // With optimizations, the prelude is optimized out if there's no late rodata. + if matches!(opts.opt, OptLevel::O2 | OptLevel::G3) { + prelude_if_late_rodata = 3; + } else { + min_instr_count += 3; + skip_instr_count += 3; + } + } + + let use_jtbl_for_rodata = + matches!(opts.opt, OptLevel::O2 | OptLevel::G3) && !opts.framepointer && !opts.kpic; + + let mut state = GlobalState::new( + min_instr_count, + skip_instr_count, + use_jtbl_for_rodata, + prelude_if_late_rodata, + opts.mips1, + opts.pascal, + ); + let input_enc = &args.input_enc; + let output_enc = &args.output_enc; + let mut global_asm: Option<(GlobalAsmBlock, usize)> = None; + let mut asm_functions: Vec<Function> = vec![]; + let mut output_lines: Vec<String> = vec![format!("#line 1 \"{}\"", infile_path.display())]; + let mut deps: Vec<String> = vec![]; + + let mut is_cutscene_data = false; + let mut is_early_include = false; + + let cutscene_re = Regex::new(r"CutsceneData (.|\n)*\[\] = \{")?; + let float_re = Regex::new(r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?f")?; + + let file_contents = fs::read(infile_path)?; + for (line_no, line) in input_enc.decode(&file_contents)?.lines().enumerate() { + let line_no = line_no + 1; + let mut raw_line = line.trim().to_owned(); + let line = raw_line.trim_start(); + + // Print exactly one output line per source line, to make compiler + // errors have correct line numbers. These will be overridden with + // reasonable content further down. + output_lines.push("".to_owned()); + + if let Some((ref mut gasm, start_index)) = global_asm { + if line.starts_with(')') { + let (mut src, fun) = gasm.finish(&mut state)?; + if state.pascal { + // Pascal has a 1600-character line length limit, so some + // of the lines we emit may be broken up. Correct for that + // using a #line directive. + *src.last_mut().unwrap() += &format!("\n#line {}", line_no + 1); + } + for (i, line2) in src.iter().enumerate() { + output_lines[start_index + i] = line2.clone(); + } + asm_functions.push(fun); + global_asm = None; + } else { + gasm.process_line(&raw_line, output_enc)?; + } + } else if line == "GLOBAL_ASM(" || line == "#pragma GLOBAL_ASM(" { + global_asm = Some(( + GlobalAsmBlock::new(format!("GLOBAL_ASM block at line {}", &line_no.to_string())), + output_lines.len(), + )); + } else if ((line.starts_with("GLOBAL_ASM(\"") || line.starts_with("#pragma GLOBAL_ASM(\"")) + && line.ends_with("\")")) + || ((line.starts_with("INCLUDE_ASM(\"") || line.starts_with("INCLUDE_RODATA(\"")) + && line.contains("\",") + && line.ends_with(");")) + { + let (prologue, fname) = if line.starts_with("INCLUDE_") { + // INCLUDE_ASM("path/to", functionname); + let (before, after) = line.split_once("\",").unwrap(); + let fname = format!( + "{}/{}.s", + before[before.find('(').unwrap() + 2..].to_owned(), + after.trim()[..after.len() - 3].trim() + ); + + if line.starts_with("INCLUDE_RODATA") { + (vec![".section .rodata".to_string()], fname) + } else { + (vec![], fname) + } + } else { + // GLOBAL_ASM("path/to/file.s") + let fname = line[line.find('(').unwrap() + 2..line.len() - 2].to_string(); + (vec![], fname) + }; + + let mut gasm = GlobalAsmBlock::new(fname.clone()); + for line2 in prologue { + gasm.process_line(line2.trim_end(), output_enc)?; + } + + if !Path::new(&fname).exists() { + // The GLOBAL_ASM block might be surrounded by an ifdef, so it's + // not clear whether a missing file actually represents a compile + // error. Pass the responsibility for determining that on to the + // compiler by emitting a bad include directive. (IDO treats + // #error as a warning for some reason.) + let output_lines_len = output_lines.len(); + output_lines[output_lines_len - 1] = format!("#include \"GLOBAL_ASM:{}\"", fname); + continue; + } + + let file_contents = fs::read(&fname)?; + for line2 in input_enc.decode(&file_contents)?.lines() { + gasm.process_line(line2.trim_end(), output_enc)?; + } + + let (mut src, fun) = gasm.finish(&mut state)?; + let output_lines_len = output_lines.len(); + if state.pascal { + // Pascal has a 1600-character line length limit, so avoid putting + // everything on the same line. + src.push(format!("#line {}", line_no + 1)); + output_lines[output_lines_len - 1] = src.join("\n"); + } else { + output_lines[output_lines_len - 1] = src.join(""); + } + asm_functions.push(fun); + deps.push(fname); + } else if line == "#pragma asmproc recurse" { + // C includes qualified as + // #pragma asmproc recurse + // #include "file.c" + // will be processed recursively when encountered + is_early_include = true; + } else if is_early_include { + // Previous line was a #pragma asmproc recurse + is_early_include = false; + if !line.starts_with("#include ") { + return Err(anyhow::anyhow!( + "#pragma asmproc recurse must be followed by an #include " + )); + } + let fpath = infile_path.parent().unwrap(); + let fname = fpath.join(line[line.find(' ').unwrap() + 2..].trim()); + deps.push(fname.to_str().unwrap().to_string()); + let mut res = parse_source(&fname, args, opts, false)?; + deps.append(&mut res.deps); + let res_str = format!( + "{}#line {} \"{}\"", + String::from_utf8(res.output).expect("nested calls generate utf-8"), + line_no + 1, + infile_path.file_name().unwrap().to_str().unwrap() + ); + + let output_lines_len = output_lines.len(); + output_lines[output_lines_len - 1] = res_str; + } else { + if args.encode_cutscene_data_floats { + // This is a hack to replace all floating-point numbers in an array of a particular type + // (in this case CutsceneData) with their corresponding IEEE-754 hexadecimal representation + if cutscene_re.is_match(line) { + is_cutscene_data = true; + } else if line.ends_with("};") { + is_cutscene_data = false; + } + + if is_cutscene_data { + raw_line = float_re.replace_all(&raw_line, repl_float_hex).into_owned(); + } + } + let output_lines_len = output_lines.len(); + output_lines[output_lines_len - 1] = raw_line.to_owned(); + } + } + + let out_data = if encode { + let newline_encoded = output_enc.encode("\n")?; + + let mut data = vec![]; + for line in output_lines { + let line_encoded = output_enc.encode(&line)?; + data.write_all(&line_encoded)?; + data.write_all(&newline_encoded)?; + } + data + } else { + let str = format!("{}\n", output_lines.join("\n")); + + str.as_bytes().to_vec() + }; + + Ok(ParseSourceResult { + functions: asm_functions, + deps, + output: out_data, + }) +} diff --git a/tools/asm-processor/tests/custom-prelude.c b/tools/asm-processor/tests/custom-prelude.c new file mode 100644 index 0000000..1b057d5 --- /dev/null +++ b/tools/asm-processor/tests/custom-prelude.c @@ -0,0 +1,17 @@ +// COMPILE-FLAGS: -O2 +// ASMP-FLAGS: --asm-prelude tests/custom-prelude.s + +GLOBAL_ASM( +glabel foo +nop +nop +nop +nop +nop +nop +nop +nop +nop +nop +endlabel foo +) diff --git a/tools/asm-processor/tests/custom-prelude.objdump b/tools/asm-processor/tests/custom-prelude.objdump new file mode 100644 index 0000000..37af655 --- /dev/null +++ b/tools/asm-processor/tests/custom-prelude.objdump @@ -0,0 +1,21 @@ + +tests/custom-prelude.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000030 .text +00000000 g F .text 00000028 foo +00000028 g .text 00000000 myendlabel + + +Contents of section .text: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00000000 00000000 00000000 ................ + 0020 00000000 00000000 00000000 00000000 ................ +Contents of section .options: + 0000 01200000 00000000 80000000 00000000 . .............. + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 80000000 00000000 00000000 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/custom-prelude.s b/tools/asm-processor/tests/custom-prelude.s new file mode 100644 index 0000000..75f955a --- /dev/null +++ b/tools/asm-processor/tests/custom-prelude.s @@ -0,0 +1,14 @@ +.set noat +.set noreorder +.set gp=64 + +.macro glabel label + .global \label + \label: +.endm + +.macro endlabel label + .global myendlabel + myendlabel: +.endm + diff --git a/tools/asm-processor/tests/euc_jp_wavedash.c b/tools/asm-processor/tests/euc_jp_wavedash.c new file mode 100644 index 0000000..d187d7d --- /dev/null +++ b/tools/asm-processor/tests/euc_jp_wavedash.c @@ -0,0 +1,3 @@ +// ASMP-FLAGS: --force --input-enc=utf-8 --output-enc=EUC-JP + +char *chars = "〜！？＠＃";
\ No newline at end of file diff --git a/tools/asm-processor/tests/euc_jp_wavedash.objdump b/tools/asm-processor/tests/euc_jp_wavedash.objdump new file mode 100644 index 0000000..5820e73 --- /dev/null +++ b/tools/asm-processor/tests/euc_jp_wavedash.objdump @@ -0,0 +1,26 @@ + +tests/euc_jp_wavedash.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .rodata 00000010 .rodata +00000000 l d .data 00000010 .data +00000000 g O .data 00000004 chars + + +RELOCATION RECORDS FOR [.data]: +OFFSET TYPE VALUE +00000000 R_MIPS_32 .rodata + + +Contents of section .rodata: + 0000 a1c1a1aa a1a9a1f7 a1f40000 00000000 ................ +Contents of section .data: + 0000 00000000 00000000 00000000 00000000 ................ +Contents of section .options: + 0000 01200000 00000000 00000000 00000000 . .............. + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 00000000 00000000 00000000 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/force.c b/tools/asm-processor/tests/force.c index 03b630c..32d8706 100644 --- a/tools/asm-processor/tests/force.c +++ b/tools/asm-processor/tests/force.c @@ -1,5 +1,5 @@ // COMPILE-FLAGS: -O2 -// ASMP-FLAGS: --convert-statics=global-with-filename --force +// ASMP-FLAGS: --convert-statics global-with-filename --force static int xtext(int a, int b, int c); const int rodata1[] = {1}; static const int rodata2[] = {2}; diff --git a/tools/asm-processor/tests/force.objdump b/tools/asm-processor/tests/force.objdump index 7ef7024..46941d3 100644 --- a/tools/asm-processor/tests/force.objdump +++ b/tools/asm-processor/tests/force.objdump @@ -17,7 +17,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000001c R_MIPS_HI16 .bss 00000034 R_MIPS_LO16 .bss 00000020 R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/include_file.asmproc.d b/tools/asm-processor/tests/include_file.asmproc.d new file mode 100644 index 0000000..80d2d66 --- /dev/null +++ b/tools/asm-processor/tests/include_file.asmproc.d @@ -0,0 +1,12 @@ +tests/include_file.o: tests/include_file_1.s \ + tests/include_file_2.s \ + tests/include_file_3.s \ + tests//include_file_4.s + +tests/include_file_1.s: + +tests/include_file_2.s: + +tests/include_file_3.s: + +tests//include_file_4.s: diff --git a/tools/asm-processor/tests/include_file.c b/tools/asm-processor/tests/include_file.c new file mode 100644 index 0000000..97eb45d --- /dev/null +++ b/tools/asm-processor/tests/include_file.c @@ -0,0 +1,8 @@ +GLOBAL_ASM("tests/include_file_1.s") +#pragma GLOBAL_ASM("tests/include_file_2.s") +INCLUDE_ASM("tests", include_file_3); +INCLUDE_RODATA("tests/", include_file_4); +#if 0 +GLOBAL_ASM("tests/nonexisting.s"); +INCLUDE_ASM("tests", nonexisting); +#endif diff --git a/tools/asm-processor/tests/include_file.objdump b/tools/asm-processor/tests/include_file.objdump new file mode 100644 index 0000000..c2b172b --- /dev/null +++ b/tools/asm-processor/tests/include_file.objdump @@ -0,0 +1,27 @@ + +tests/include_file.o: file format elf32-tradbigmips + +SYMBOL TABLE: +00000000 l d .text 00000050 .text +00000000 l d .rodata 00000010 .rodata +00000000 g F .text 00000018 file_1 +00000018 g F .text 00000018 file_2 +00000030 g F .text 00000018 file_3 + + +Contents of section .text: + 0000 24050000 24050001 24050002 24050003 $...$...$...$... + 0010 24050004 24050005 24060000 24060001 $...$...$...$... + 0020 24060002 24060003 24060004 24060005 $...$...$...$... + 0030 24070000 24070001 24070002 24070003 $...$...$...$... + 0040 24070004 24070005 00000000 00000000 $...$........... +Contents of section .rodata: + 0000 00000005 00000006 00000007 00000008 ................ +Contents of section .options: + 0000 01200000 00000000 80000000 00000000 . .............. + 0010 00000000 00000000 00000000 00007ff0 ................ + 0020 07100000 00000000 00000000 00000000 ................ + 0030 08100000 00000000 00000000 00000000 ................ +Contents of section .reginfo: + 0000 800000e0 00000000 00000000 00000000 ................ + 0010 00000000 00007ff0 ........ diff --git a/tools/asm-processor/tests/include_file_1.s b/tools/asm-processor/tests/include_file_1.s new file mode 100644 index 0000000..24a67f9 --- /dev/null +++ b/tools/asm-processor/tests/include_file_1.s @@ -0,0 +1,7 @@ +glabel file_1 +li $a1, 0 +li $a1, 1 +li $a1, 2 +li $a1, 3 +li $a1, 4 +li $a1, 5 diff --git a/tools/asm-processor/tests/include_file_2.s b/tools/asm-processor/tests/include_file_2.s new file mode 100644 index 0000000..4260197 --- /dev/null +++ b/tools/asm-processor/tests/include_file_2.s @@ -0,0 +1,7 @@ +glabel file_2 +li $a2, 0 +li $a2, 1 +li $a2, 2 +li $a2, 3 +li $a2, 4 +li $a2, 5 diff --git a/tools/asm-processor/tests/include_file_3.s b/tools/asm-processor/tests/include_file_3.s new file mode 100644 index 0000000..d0123af --- /dev/null +++ b/tools/asm-processor/tests/include_file_3.s @@ -0,0 +1,7 @@ +glabel file_3 +li $a3, 0 +li $a3, 1 +li $a3, 2 +li $a3, 3 +li $a3, 4 +li $a3, 5 diff --git a/tools/asm-processor/tests/include_file_4.s b/tools/asm-processor/tests/include_file_4.s new file mode 100644 index 0000000..94cb710 --- /dev/null +++ b/tools/asm-processor/tests/include_file_4.s @@ -0,0 +1 @@ +.word 5, 6, 7, 8 diff --git a/tools/asm-processor/tests/label-sameline.objdump b/tools/asm-processor/tests/label-sameline.objdump index 1f1aacf..e69f69c 100644 --- a/tools/asm-processor/tests/label-sameline.objdump +++ b/tools/asm-processor/tests/label-sameline.objdump @@ -8,7 +8,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.rodata]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000004 R_MIPS_32 00000008 R_MIPS_32 blah diff --git a/tools/asm-processor/tests/late_rodata_align.objdump b/tools/asm-processor/tests/late_rodata_align.objdump index 87c05e8..0f7f4c0 100644 --- a/tools/asm-processor/tests/late_rodata_align.objdump +++ b/tools/asm-processor/tests/late_rodata_align.objdump @@ -12,7 +12,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000040 R_MIPS_HI16 .rodata 00000048 R_MIPS_LO16 .rodata 00000090 R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/late_rodata_doubles.objdump b/tools/asm-processor/tests/late_rodata_doubles.objdump index 84d7e67..3545768 100644 --- a/tools/asm-processor/tests/late_rodata_doubles.objdump +++ b/tools/asm-processor/tests/late_rodata_doubles.objdump @@ -13,7 +13,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000040 R_MIPS_HI16 .rodata 00000048 R_MIPS_LO16 .rodata 0000009c R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/late_rodata_doubles_mips1.objdump b/tools/asm-processor/tests/late_rodata_doubles_mips1.objdump index 8fac85f..667ee7d 100644 --- a/tools/asm-processor/tests/late_rodata_doubles_mips1.objdump +++ b/tools/asm-processor/tests/late_rodata_doubles_mips1.objdump @@ -13,7 +13,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000040 R_MIPS_HI16 .rodata 00000044 R_MIPS_LO16 .rodata 00000090 R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/late_rodata_jtbl.objdump b/tools/asm-processor/tests/late_rodata_jtbl.objdump index 44bb6e8..53f664b 100644 --- a/tools/asm-processor/tests/late_rodata_jtbl.objdump +++ b/tools/asm-processor/tests/late_rodata_jtbl.objdump @@ -16,7 +16,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000005c R_MIPS_HI16 .rodata 00000064 R_MIPS_LO16 .rodata 0000018c R_MIPS_HI16 .rodata @@ -26,7 +26,7 @@ OFFSET TYPE VALUE RELOCATION RECORDS FOR [.rodata]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000008c R_MIPS_32 00000090 R_MIPS_32 00000094 R_MIPS_32 diff --git a/tools/asm-processor/tests/late_rodata_jtbl_mips1.objdump b/tools/asm-processor/tests/late_rodata_jtbl_mips1.objdump index d37c8bb..a781a92 100644 --- a/tools/asm-processor/tests/late_rodata_jtbl_mips1.objdump +++ b/tools/asm-processor/tests/late_rodata_jtbl_mips1.objdump @@ -16,7 +16,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000005c R_MIPS_HI16 .rodata 00000060 R_MIPS_LO16 .rodata 00000190 R_MIPS_HI16 .rodata @@ -26,7 +26,7 @@ OFFSET TYPE VALUE RELOCATION RECORDS FOR [.rodata]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000008c R_MIPS_32 00000090 R_MIPS_32 00000094 R_MIPS_32 diff --git a/tools/asm-processor/tests/late_rodata_misaligned_doubles.objdump b/tools/asm-processor/tests/late_rodata_misaligned_doubles.objdump index cdba84d..46cdc0c 100644 --- a/tools/asm-processor/tests/late_rodata_misaligned_doubles.objdump +++ b/tools/asm-processor/tests/late_rodata_misaligned_doubles.objdump @@ -13,7 +13,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000040 R_MIPS_HI16 .rodata 00000048 R_MIPS_LO16 .rodata 0000009c R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/o0.objdump b/tools/asm-processor/tests/o0.objdump index f94ec7a..98a27dc 100644 --- a/tools/asm-processor/tests/o0.objdump +++ b/tools/asm-processor/tests/o0.objdump @@ -13,7 +13,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000030 R_MIPS_HI16 .rodata 00000034 R_MIPS_LO16 .rodata 0000006c R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/o2.objdump b/tools/asm-processor/tests/o2.objdump index 61e3fb0..07b3895 100644 --- a/tools/asm-processor/tests/o2.objdump +++ b/tools/asm-processor/tests/o2.objdump @@ -13,7 +13,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000001c R_MIPS_HI16 .rodata 00000024 R_MIPS_LO16 .rodata 0000003c R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/pascal.objdump b/tools/asm-processor/tests/pascal.objdump index cb3740f..de2db82 100644 --- a/tools/asm-processor/tests/pascal.objdump +++ b/tools/asm-processor/tests/pascal.objdump @@ -9,8 +9,8 @@ SYMBOL TABLE: 00000000 l O .bss 00000000 $dat 00000000 g F .text 0000000c foo 000000d0 g F .text 0000000c bar -0000000c g F .text 00000004 test -00000048 g F .text 00000004 test2 +0000000c g F .text 0000003c test +00000048 g F .text 00000044 test2 0000008c g F .text 00000044 test3 00000000 *UND* 00000000 get 00000000 *UND* 00000000 put diff --git a/tools/asm-processor/tests/static-global.c b/tools/asm-processor/tests/static-global.c index a7f6169..12b1434 100644 --- a/tools/asm-processor/tests/static-global.c +++ b/tools/asm-processor/tests/static-global.c @@ -25,9 +25,11 @@ nop ) static int xtext(int a, int b, int c) { + static int bss2; return 1; } void baz(void) { + { static int bss2; } xtext(bss2, rodata2[0], data2[0]); } diff --git a/tools/asm-processor/tests/static-global.objdump b/tools/asm-processor/tests/static-global.objdump index a519e4f..0b21d2e 100644 --- a/tools/asm-processor/tests/static-global.objdump +++ b/tools/asm-processor/tests/static-global.objdump @@ -15,10 +15,12 @@ SYMBOL TABLE: 00000004 g O .data 00000000 data2 00000004 g O .bss 00000000 bss2 00000030 g F .text 00000000 xtext +00000008 g O .bss 00000000 bss2:1 +0000000c g O .bss 00000000 bss2:2 RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000004c R_MIPS_HI16 .bss 00000064 R_MIPS_LO16 .bss 00000050 R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/static.objdump b/tools/asm-processor/tests/static.objdump index dd62c94..a2f3638 100644 --- a/tools/asm-processor/tests/static.objdump +++ b/tools/asm-processor/tests/static.objdump @@ -18,7 +18,7 @@ SYMBOL TABLE: RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000004c R_MIPS_HI16 .bss 00000064 R_MIPS_LO16 .bss 00000050 R_MIPS_HI16 .rodata diff --git a/tools/asm-processor/tests/test1.objdump b/tools/asm-processor/tests/test1.objdump index b97ae46..af12f90 100644 --- a/tools/asm-processor/tests/test1.objdump +++ b/tools/asm-processor/tests/test1.objdump @@ -12,13 +12,13 @@ SYMBOL TABLE: 0000003c g F .text 0000003c f 00000000 g F .text 0000003c test 0000000c g .rodata 00000000 some_rodata -00000078 g F .text 00000004 g +00000078 g F .text 00000034 g 00000000 *UND* 00000000 D_410100 00000000 *UND* 00000000 func_004000B0 RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000044 R_MIPS_HI16 .rodata 00000054 R_MIPS_LO16 .rodata 00000048 R_MIPS_HI16 some_rodata diff --git a/tools/asm-processor/tests/test2.objdump b/tools/asm-processor/tests/test2.objdump index 276ed9f..968b7d0 100644 --- a/tools/asm-processor/tests/test2.objdump +++ b/tools/asm-processor/tests/test2.objdump @@ -11,12 +11,12 @@ SYMBOL TABLE: 00000010 g O .rodata 00000001 buf3 00000044 g F .text 0000007c func3 000000c4 g F .text 00000000 jumptarget -000000c0 g F .text 0000000c func4 +000000c0 g F .text 0000002c func4 0000004c g .rodata 00000000 rv RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000000 R_MIPS_HI16 .rodata 00000008 R_MIPS_LO16 .rodata 0000001c R_MIPS_HI16 .rodata @@ -30,7 +30,7 @@ OFFSET TYPE VALUE RELOCATION RECORDS FOR [.rodata]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 0000002c R_MIPS_32 .text 00000030 R_MIPS_32 .text 00000034 R_MIPS_32 .text diff --git a/tools/asm-processor/tests/test3.objdump b/tools/asm-processor/tests/test3.objdump index bfa7f6e..e1e96c2 100644 --- a/tools/asm-processor/tests/test3.objdump +++ b/tools/asm-processor/tests/test3.objdump @@ -13,14 +13,14 @@ SYMBOL TABLE: 00000008 g O .rodata 00000001 constBufInit2 0000000c g O .rodata 00000001 constBufInit3 0000003c g F .text 00000054 f -00000000 g F .text 00000004 test -00000090 g F .text 00000004 g +00000000 g F .text 0000003c test +00000090 g F .text 00000034 g 00000000 *UND* 00000000 D_410100 00000000 *UND* 00000000 func_004000B0 RELOCATION RECORDS FOR [.text]: -OFFSET TYPE VALUE +OFFSET TYPE VALUE 00000044 R_MIPS_HI16 .rodata 00000048 R_MIPS_LO16 .rodata 00000050 R_MIPS_HI16 .rodata diff --git a/yamls/jp/header.yaml b/yamls/jp/header.yaml index b971498..501d6d5 100644 --- a/yamls/jp/header.yaml +++ b/yamls/jp/header.yaml @@ -46,6 +46,7 @@ options: use_legacy_include_asm: False create_asm_dependencies: True asm_function_macro: glabel + asm_end_label: endlabel asm_jtbl_label_macro: jlabel asm_data_macro: dlabel include_macro_inc: False |