diff options
| -rw-r--r-- | .github/actions/fix-bss-and-generate-patch/action.yml | 20 | ||||
| -rw-r--r-- | .github/scripts/apply_fix_bss_patches.py | 112 | ||||
| -rwxr-xr-x | .github/scripts/generate_patch.sh | 10 | ||||
| -rw-r--r-- | .github/workflows/ci.yml | 134 | ||||
| -rw-r--r-- | .github/workflows/format.yml | 22 | ||||
| -rw-r--r-- | Jenkinsfile | 110 | ||||
| -rw-r--r-- | README.md | 6 |
7 files changed, 301 insertions, 113 deletions
diff --git a/.github/actions/fix-bss-and-generate-patch/action.yml b/.github/actions/fix-bss-and-generate-patch/action.yml new file mode 100644 index 000000000..726f69304 --- /dev/null +++ b/.github/actions/fix-bss-and-generate-patch/action.yml @@ -0,0 +1,20 @@ +inputs: + version: + required: true + +runs: + using: composite + steps: + - name: Fix BSS + shell: sh + run: .venv/bin/python3 tools/fix_bss.py -v ${{ inputs.version }} + + - name: Generate patch + shell: sh + run: git diff > fix_bss_${{ inputs.version }}.patch + + - name: Upload patch + uses: actions/upload-artifact@v7 + with: + name: fix_bss_${{ inputs.version }}.patch + path: fix_bss_${{ inputs.version }}.patch diff --git a/.github/scripts/apply_fix_bss_patches.py b/.github/scripts/apply_fix_bss_patches.py new file mode 100644 index 000000000..06c79fe52 --- /dev/null +++ b/.github/scripts/apply_fix_bss_patches.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: © 2026 ZeldaRET +# SPDX-License-Identifier: CC0-1.0 + +from pathlib import Path +import re +import subprocess + + +def get_increment_block_numbers(p: Path, version: str): + increment_block_numbers: list[int] = [] + is_in_pragma = False + n_fake_structs = None + for l in p.read_text().splitlines(): + if l.startswith("#pragma increment_block_number"): + is_in_pragma = True + n_fake_structs = 0 + if is_in_pragma: + m = next(re.finditer(rf"{version}:(\d+)", l), None) + if m is not None: + n_fake_structs = int(m.group(1)) + if is_in_pragma and not l.endswith("\\"): + is_in_pragma = False + assert n_fake_structs is not None + increment_block_numbers.append(n_fake_structs) + n_fake_structs = None + return increment_block_numbers + + +# Formats #pragma increment_block_number as a list of lines +def format_pragma(amounts: dict[str, int], max_line_length: int) -> list[str]: + lines = [] + pragma_start = "#pragma increment_block_number " + current_line = pragma_start + '"' + first = True + for version, amount in sorted(amounts.items()): + part = f"{version}:{amount}" + if len(current_line) + len(" ") + len(part) + len('" \\') > max_line_length: + lines.append(current_line + '" ') + current_line = " " * len(pragma_start) + '"' + first = True + if not first: + current_line += " " + current_line += part + first = False + lines.append(current_line + '"\n') + + if len(lines) >= 2: + # add and align vertically all continuation \ characters + n_align = max(map(len, lines[:-1])) + for i in range(len(lines) - 1): + lines[i] = f"{lines[i]:{n_align}}\\\n" + + return lines + + +def set_increment_block_numbers( + p: Path, increment_block_numbers_by_version: dict[str, list[int]] +): + print(p, increment_block_numbers_by_version) + i_pragma = 0 + is_in_pragma = False + pragma_lines = [] + new_lines = [] + for l in p.read_text().splitlines(keepends=True): + if l.startswith("#pragma increment_block_number"): + is_in_pragma = True + if not is_in_pragma: + new_lines.append(l) + if is_in_pragma: + pragma_lines.append(l.removesuffix("\\\n")) + if is_in_pragma and not l.endswith("\\\n"): + is_in_pragma = False + pragma_string = "".join(pragma_lines) + amounts: dict[str, int] = {} + for part in pragma_string.replace('"', "").split()[2:]: + version, amount_str = part.split(":") + amount = int(amount_str) + amounts[version] = amount + for ( + version, + increment_block_numbers, + ) in increment_block_numbers_by_version.items(): + amounts[version] = increment_block_numbers[i_pragma] + i_pragma += 1 + column_limit = 120 # matches .clang-format's ColumnLimit + new_pragma_lines = format_pragma(amounts, column_limit) + new_lines.extend(new_pragma_lines) + p.write_text("".join(new_lines)) + + +increment_block_numbers_by_version_by_file: dict[Path, dict[str, list[int]]] = {} +for p in Path(".").glob("fix_bss_*.patch"): + version = p.name.removeprefix("fix_bss_").removesuffix(".patch") + subprocess.check_call(["git", "apply", str(p)]) + touched_files = subprocess.check_output( + "git diff --name-only".split(), + text=True, + ).splitlines() + for file in touched_files: + file_p = Path(file) + increment_block_numbers = get_increment_block_numbers(file_p, version) + increment_block_numbers_by_version_by_file.setdefault(file_p, {})[ + version + ] = increment_block_numbers + subprocess.check_call("git checkout -- .".split()) + + +for ( + file, + increment_block_numbers_by_version, +) in increment_block_numbers_by_version_by_file.items(): + set_increment_block_numbers(file, increment_block_numbers_by_version) diff --git a/.github/scripts/generate_patch.sh b/.github/scripts/generate_patch.sh new file mode 100755 index 000000000..04cc431b0 --- /dev/null +++ b/.github/scripts/generate_patch.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +PATCH=$(git diff | base64 -w 0) +if [ -n "$PATCH" ]; then + echo 'Fixes were made for your PR. To apply these changes to your working directory, copy and run the following command:' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "echo -n $PATCH | base64 -d | git apply -" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..af0bc96d5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,134 @@ +name: Build + +# Build on every branch push, tag push, and pull request change: +on: + push: + pull_request: + +jobs: + build_repo: + # This is a *private* build container. + container: ghcr.io/zeldaret/mm-build:main + + name: Build repo (${{ matrix.version }}) + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + version: + - n64-jp-1.1 # N64 Japan 1.1 + - n64-us # N64 USA + include: + - version: n64-jp-1.1 + non_matching: 1 + + # By default no version in the matrix contains a non_matching value, + # meaning ${{ matrix.non_matching }} expand to an empty string, unless it + # is explicitly listed on the `include` block. + # Use the value from the matrix if it exists, or fallback to 0 if it doesn't. + env: + NON_MATCHING: ${{ matrix.non_matching || 0 }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: git config safe.directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Install system dependencies + run: | + apt-get install -y git build-essential binutils-mips-linux-gnu curl python3 python3-pip python3-venv libxml2-dev + + - name: Get the dependency + run: ln -s /orig/${{ matrix.version }}/baserom.z64 baseroms/${{ matrix.version }}/baserom.z64 + + # The compiler archives are embedded in the runner image, to avoid downloading them from GitHub (during make setup), which occasionally fails. + - name: Provide compiler archives + run: ln -s /compiler_archives tools/compiler_archives/archives + + - name: venv + run: make -j $(nproc) VERSION=${{ matrix.version }} venv + + - name: Setup + run: make -j $(nproc) VERSION=${{ matrix.version }} setup 2> >(tee tools/warnings_count/warnings_setup_new.txt)" + + - name: Check setup warnings + run: ./tools/warnings_count/compare_warnings.sh setup + + - name: Assets + run: make -j $(nproc) VERSION=${{ matrix.version }} assets 2> >(tee tools/warnings_count/warnings_assets_new.txt)" + + - name: Check assets warnings + run: ./tools/warnings_count/compare_warnings.sh assets + + - name: Disasm + run: make -j $(nproc) VERSION=${{ matrix.version }} disasm 2> >(tee tools/warnings_count/warnings_disasm_new.txt)" + + - name: Check disasm warnings + run: ./tools/warnings_count/compare_warnings.sh disasm + + - name: Build ${{ matrix.version }} + id: build + run: make -j $(nproc) VERSION=${{ matrix.version }} rom 2> >(tee tools/warnings_count/warnings_build_new.txt) + + - name: Check build warnings + run: ./tools/warnings_count/compare_warnings.sh build + + - name: Compress ${{ matrix.version }} + id: build + run: make -j $(nproc) VERSION=${{ matrix.version }} compress 2> >(tee tools/warnings_count/warnings_compress_new.txt) + + - name: Check compress warnings + run: ./tools/warnings_count/compare_warnings.sh compress + + - name: Fix BSS and generate patch + if: failure() && steps.build.outcome == 'failure' + uses: ./.github/actions/fix-bss-and-generate-patch + with: + version: ${{ matrix.version }} + + - name: Show warnings + if: failure() && steps.build.outcome == 'failure' + run: cat tools/warnings_count/warnings_setup_new.txt tools/warnings_count/warnings_assets_new.txt tools/warnings_count/warnings_disasm_new.txt tools/warnings_count/warnings_build_new.txt tools/warnings_count/warnings_compress_new.txt + + - name: Upload map + uses: actions/upload-artifact@v6 + with: + name: mm-${{ matrix.version }}.map + path: build/${{ matrix.version }}/mm-${{ matrix.version }}.map + + # This job does not do anything, its purpose is to be used as a status check in GitHub rules. + all_versions_built: + name: All versions built + needs: [build_repo] + runs-on: ubuntu-latest + # Solution 1 from https://github.com/actions/runner/issues/2566#issuecomment-3053484216 + if: always() + steps: + - run: | + if [ "${{ needs.build_repo.result }}" != "success" ]; then + exit 1 + fi + + merge_bss_fixes: + name: Merge BSS fixes + runs-on: ubuntu-latest + needs: [build_repo] + if: '!cancelled()' # Run even if build_repo fails + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download patches + uses: actions/download-artifact@v8 + with: + pattern: fix_bss_*.patch + merge-multiple: true + + - name: Apply patches + run: python3 .github/scripts/apply_fix_bss_patches.py + + - name: Generate patch + run: .github/scripts/generate_patch.sh diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 000000000..043b9ee5d --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,22 @@ +name: Check format + +# Build on every branch push, tag push, and pull request change: +on: + push: + pull_request: + +jobs: + build: + name: Check format + runs-on: ubuntu-latest + + steps: + - name: Checkout reposistory + uses: actions/checkout@v6 + + - name: Install package requirements + run: | + sudo apt-get install -y python3 clang-format-14 clang-tidy-14 + + - name: Check formatting + run: tools/check_format.sh diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 9c0ae16da..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,110 +0,0 @@ -pipeline { - agent { - label 'mm' - } - - options { - ansiColor('xterm') - } - - stages { - stage('Check formatting') { - steps { - echo 'Checking formatting...' - sh 'bash -c "tools/check_format.sh 2>&1 >(tee tools/check_format.txt)"' - } - } - stage('Check relocs') { - steps { - echo 'Checking relocs on spec...' - sh 'bash -c "tools/reloc_spec_check.sh"' - } - } - stage('Install Python dependencies') { - steps { - sh 'bash -c "make -j venv"' - sh '.venv/bin/python3 -m pip install GitPython' // Progress script from jenkins requires GitPython - } - } - stage('Copy ROM') { - steps { - echo 'Setting up ROM...' - sh 'cp /usr/local/etc/roms/mm.us.rev1.z64 baseroms/n64-us/baserom.z64' - } - } - stage('Setup') { - steps { - sh 'bash -c "make -j setup 2> >(tee tools/warnings_count/warnings_setup_new.txt)"' - } - } - stage('Check setup warnings') { - steps { - sh 'bash -c "./tools/warnings_count/compare_warnings.sh setup"' - } - } - stage('Assets') { - steps { - sh 'bash -c "make -j assets 2> >(tee tools/warnings_count/warnings_assets_new.txt)"' - } - } - stage('Check assets warnings') { - steps { - sh 'bash -c "./tools/warnings_count/compare_warnings.sh assets"' - } - } - stage('Build') { - steps { - sh 'bash -c "make -j rom 2> >(tee tools/warnings_count/warnings_build_new.txt)"' - } - } - stage('Check build warnings') { - steps { - sh 'bash -c "./tools/warnings_count/compare_warnings.sh build"' - } - } - stage('Compress') { - steps { - sh 'bash -c "make -j compress 2> >(tee tools/warnings_count/warnings_compress_new.txt)"' - } - } - stage('Check compress warnings') { - steps { - sh 'bash -c "./tools/warnings_count/compare_warnings.sh compress"' - } - } - stage('Report Progress') { - when { - branch 'main' - } - steps { - sh 'mkdir reports' - sh '.venv/bin/python3 ./tools/progress.py csv >> reports/progress-mm-nonmatching.csv' - sh '.venv/bin/python3 ./tools/progress.py csv -m >> reports/progress-mm-matching.csv' - sh '.venv/bin/python3 ./tools/progress.py shield-json > reports/progress-mm-shield.json' - stash includes: 'reports/*', name: 'reports' - } - } - stage('Update Progress') { - when { - branch 'main' - } - agent{ - label 'zeldaret_website' - } - steps { - unstash 'reports' - sh 'cat reports/progress-mm-nonmatching.csv >> /var/www/zelda64.dev/assets/csv/progress-mm-nonmatching.csv' - sh 'cat reports/progress-mm-matching.csv >> /var/www/zelda64.dev/assets/csv/progress-mm-matching.csv' - sh 'cat reports/progress-mm-shield.json > /var/www/zelda64.dev/assets/csv/progress-mm-shield.json' - } - } - } - post { - failure { - sh 'cat tools/check_format.txt tools/warnings_count/warnings_setup_new.txt tools/warnings_count/warnings_build_new.txt' - } - always { - cleanWs() - } - } -} @@ -1,9 +1,9 @@ # Legend of Zelda: Majora's Mask (US) 1.0 -[![Build Status][jenkins-badge]][jenkins] [![Decompilation Progress][progress-badge]][progress] [![Contributors][contributors-badge]][contributors] [![Discord Channel][discord-badge]][discord] +[![Build Status][gha-badge]][gha] [![Decompilation Progress][progress-badge]][progress] [![Contributors][contributors-badge]][contributors] [![Discord Channel][discord-badge]][discord] -[jenkins]: https://jenkins.deco.mp/job/MM/job/main -[jenkins-badge]: https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fjenkins.deco.mp%2Fjob%2FMM%2Fjob%2Fmain +[gha]: https://github.com/zeldaret/mm/actions/workflows/ci.yml?query=branch%3Amain+event%3Apush +[gha-badge]: https://img.shields.io/github/actions/workflow/status/zeldaret/mm/ci.yml [progress]: https://zelda.deco.mp/games/mm [progress-badge]: https://img.shields.io/endpoint?url=https://zelda.deco.mp/assets/csv/progress-mm-shield.json |