Generate decomp.dev progress reports (#249)HEADmain

* Generate decomp.dev progress reports

- Wire up CI to generate decomp.dev progress reports (using the objdiff report format) and upload them as GHA artifacts, by using `mapfile_parser`.
- This also adds a `decomp.yaml` file, which is used by `mapfile_parser` to generate the objdiff reports.
- The `progress.py` script used to locally visualize the current progress has been updated to parse objdiff reports too.
- `mapfile_parser` 2.13.0 or above is now required.
- Delete the old `upload_frogress.py` script.

Once we have this we can create an af entry in [decomp.dev](https://decomp.dev/).
I'll talk with Encounter to migrate the old frogress progress into decomp.dev too

* Activate venv

6 files changed, 196 insertions, 154 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a0f9eab..b38857b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -86,6 +86,17 @@ jobs:
         if: failure() && steps.build.outcome == 'failure'
         run: cat tools/warnings_count/warnings_setup_new.txt tools/warnings_count/warnings_assets_new.txt tools/warnings_count/warnings_disasm_new.txt tools/warnings_count/warnings_build_new.txt tools/warnings_count/warnings_compress_new.txt
 
+      - name: Generate objdiff report
+        run: |
+          . .venv/bin/activate
+          python3 -m mapfile_parser objdiff_report report.json --version ${{ matrix.version }}
+
+      - name: Upload progress report artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: ${{ matrix.version }}_report
+          path: report.json
+
       - name: Upload map
         uses: actions/upload-artifact@v7
         with:
diff --git a/.gitignore b/.gitignore
index 3deab9a..4e45f51 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,5 @@ ctx.c.m2c
 
 .vscode/*
 !.vscode/c_cpp_properties.json
+
+report.json
diff --git a/decomp.yaml b/decomp.yaml
new file mode 100644
index 0000000..9fb5ec0
--- /dev/null
+++ b/decomp.yaml
@@ -0,0 +1,114 @@
+name: Animal Forest
+repo: https://github.com/zeldaret/af
+platform: n64
+build_system: make
+versions:
+  - name: jp
+    fullname: jp
+    sha1: 3ee1f866eb08c0ce57a30e09f0c8b0a16a760c8f
+    paths:
+      target: "baseroms/jp/baserom-decompressed.z64"
+
+      build_dir: "build"
+      map: "build/animalforest-jp.map"
+      compiled_target: "build/animalforest-jp.map"
+      elf: "build/animalforest-jp.elf"
+
+      expected_dir: "expected/"
+
+      asm: "asm/jp"
+      nonmatchings: "asm/jp/nonmatchings"
+
+      compressed_target: "baseroms/jp/baserom.z64"
+      compressed_compiled_target: "build/animalforest-jp-compressed.z64"
+
+tools:
+  mapfile_parser:
+    progress_report:
+      check_asm_paths: False
+      report_data: True
+      prefixes_to_trim:
+        - build/assets/jp/
+        - build/assets/
+        - build/lib/
+        - build/src/
+        - build/asm/jp/data/
+        - build/asm/jp/
+        - build/asm/data/
+        - build/asm/
+        - build/
+
+      # List of categories. `id`s must be unique, but each path may be
+      # duplicated across categories.
+      categories:
+        - id: boot
+          name: Segment boot
+          paths:
+            - boot/
+        - id: dmadata
+          name: Segment dmadata
+          paths:
+            - dmadata/
+        - id: code
+          name: Segment code
+          paths:
+            - code/
+        - id: overlays
+          name: All Overlays
+          paths:
+            - overlays/
+        - id: objects
+          name: All objects
+          paths:
+            - objects/
+
+        - id: assets
+          name: All assets
+          paths:
+            - assets/
+            - misc/
+            - combi/
+
+        - id: buffers
+          name: Buffers
+          paths:
+            - buffers/
+
+        - id: libraries
+          name: Libraries
+          paths:
+            - boot/libc/
+            - boot/libc64/
+            - boot/libm/
+            - boot/libu64/
+            - libultra/
+
+        - id: overlays_actors
+          name: Overlays actors
+          paths:
+            - overlays/actors/
+
+        - id: overlays_furniture
+          name: Overlays furniture
+          paths:
+            - overlays/furniture/
+
+        - id: overlays_gamestates
+          name: Overlays gamestates
+          paths:
+            - overlays/gamestates/
+
+        - id: overlays_managers
+          name: Overlays managers
+          paths:
+            - overlays/managers/
+
+        - id: overlays_submenu
+          name: Overlays submenu
+          paths:
+            - overlays/submenu/
+
+        - id: overlays_weather
+          name: Overlays weather
+          paths:
+            - overlays/weather/
diff --git a/requirements.txt b/requirements.txt
index 3d902cf..de04d4b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 splat64[mips]==0.24.0
-mapfile-parser>=2.4.0,<3.0.0
+mapfile-parser>=2.13.0,<3.0.0
 colorama
 libyaz0
 
diff --git a/tools/progress.py b/tools/progress.py
index c4e99a4..857563a 100755
--- a/tools/progress.py
+++ b/tools/progress.py
@@ -1,132 +1,87 @@
 #!/usr/bin/env python3
 
-# SPDX-FileCopyrightText: © 2023 ZeldaRET
+# SPDX-FileCopyrightText: © 2023-2025 ZeldaRET
 # SPDX-License-Identifier: MIT
 
 from __future__ import annotations
 
 import argparse
-import mapfile_parser
+import sys
 from pathlib import Path
 
+import decomp_settings
+import mapfile_parser
 
-ASMPATH = Path("asm")
-NONMATCHINGS = "nonmatchings"
-
-
-def getProgressFromMapFile(mapFile: mapfile_parser.MapFile, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2) -> tuple[mapfile_parser.ProgressStats, dict[str, mapfile_parser.ProgressStats]]:
-    totalStats = mapfile_parser.ProgressStats()
-    progressPerFolder: dict[str, mapfile_parser.ProgressStats] = dict()
-
-    for segment in mapFile:
-        for file in segment:
-            if len(file) == 0:
-                continue
-
-            folder = file.filepath.parts[pathIndex]
-
-            if folder in aliases:
-                folder = aliases[folder]
-
-            if folder not in progressPerFolder:
-                progressPerFolder[folder] = mapfile_parser.ProgressStats()
-
-            originalFilePath = Path(*file.filepath.parts[pathIndex:])
-
-            extensionlessFilePath = originalFilePath
-            while extensionlessFilePath.suffix:
-                extensionlessFilePath = extensionlessFilePath.with_suffix("")
-
-            fullAsmFile = asmPath / extensionlessFilePath.with_suffix(".s")
-
-            handwrittenAsmFiles = [Path("boot/libc64/fp.o"), Path("makerom/entry.o")]
-            
-            if originalFilePath in handwrittenAsmFiles:
-                wholeFileIsUndecomped = False
-            else:
-                wholeFileIsUndecomped = fullAsmFile.exists()
-
-
-            for func in file:
-                funcAsmPath = nonmatchings / extensionlessFilePath / f"{func.name}.s"
-
-                symSize = 0
-                if func.size is not None:
-                    symSize = func.size
-
-                if wholeFileIsUndecomped:
-                    totalStats.undecompedSize += symSize
-                    progressPerFolder[folder].undecompedSize += symSize
-                elif funcAsmPath.exists():
-                    totalStats.undecompedSize += symSize
-                    progressPerFolder[folder].undecompedSize += symSize
-                else:
-                    totalStats.decompedSize += symSize
-                    progressPerFolder[folder].decompedSize += symSize
-
-    return totalStats, progressPerFolder
-
-
-def getProgress(mapPath: Path, version: str) -> tuple[mapfile_parser.ProgressStats, dict[str, mapfile_parser.ProgressStats]]:
-    mapFile = mapfile_parser.MapFile()
-    mapFile.readMapFile(mapPath)
-
-    for segment in mapFile:
-        for file in segment:
-            if len(file) == 0:
-                continue
-
-            filepathParts = list(file.filepath.parts)
-            if version in filepathParts:
-                filepathParts.remove(version)
-            file.filepath = Path(*filepathParts)
-
-    nonMatchingsPath = ASMPATH / version / NONMATCHINGS
-
-    return getProgressFromMapFile(mapFile.filterBySectionType(".text"), ASMPATH / version, nonMatchingsPath, aliases={"ultralib": "libultra"})
-
-def getAssetProgress(mapPath: Path, version: str):
-    totalStats = mapfile_parser.ProgressStats()
-    progressPerFolder: dict[str, mapfile_parser.ProgressStats] = dict()
-    progressPerFolder["objects"] = mapfile_parser.ProgressStats()
-    progressPerFolder["unidentified"] = mapfile_parser.ProgressStats()
-
-    mapFile = mapfile_parser.MapFile()
-    mapFile.readMapFile(mapPath)
-    mapFile = mapFile.filterBySectionType(".data")
-
-    for segment in mapFile:
-        for file in segment:
-
-            if len(file) == 0:
-                continue
 
-            if str(file.filepath).startswith("build/src/objects"):
-                totalStats.decompedSize += file.size
-                progressPerFolder["objects"].decompedSize += file.size
-            elif str(file.filepath).startswith("build/assets/" + version + "/objects"):
-                totalStats.undecompedSize += file.size
-                progressPerFolder["objects"].undecompedSize += file.size
-            elif str(file.filepath).startswith("build/assets/" + version):
-                progressPerFolder["unidentified"].undecompedSize += file.size
-                totalStats.undecompedSize += file.size
-    
-    return totalStats, progressPerFolder
+def doThing(
+    version: str,
+    settings: decomp_settings.Config,
+    units: bool,
+    sort: bool,
+    remaining: bool,
+):
+    selectedVersion = settings.get_version_by_name(version)
+    assert selectedVersion is not None
+    mapPath = Path(selectedVersion.paths.map)
+    outputPath = Path(selectedVersion.paths.build_dir) / "report.json"
+
+    specificSettings = mapfile_parser.frontends.objdiff_report.SpecificSettings.fromDecompConfig(settings)
+    assert specificSettings is not None
+    prefixesToTrim = specificSettings.prefixesToTrim
+    pathIndex = specificSettings.pathIndex
+    assert pathIndex is not None
+
+    reportCategories = mapfile_parser.ReportCategories()
+    for cat in specificSettings.categories:
+        reportCategories.push(cat.ide, cat.name, cat.paths)
+
+    if specificSettings.checkAsmPaths:
+        assert selectedVersion.paths.asm is not None
+        asmPath = Path(selectedVersion.paths.asm)
+    else:
+        asmPath = None
+
+    summaryTableConfig = mapfile_parser.frontends.objdiff_report.SummaryTableConfig(
+        doUnits=units,
+        sort=sort,
+        remaining=remaining,
+    )
+
+    exitcode = mapfile_parser.frontends.objdiff_report.doObjdiffReport(
+        mapPath,
+        outputPath,
+        prefixesToTrim,
+        reportCategories,
+        pathIndex=pathIndex,
+        asmPath=asmPath,
+        summaryTableConfig=summaryTableConfig,
+    )
+    sys.exit(exitcode)
 
 def progressMain():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-v", "--version", help="version to process", default="jp")
+    settings = decomp_settings.scan_for_config()
+    versionsChoices = [x.name for x in settings.versions]
 
-    args = parser.parse_args()
+    parser = argparse.ArgumentParser(description="Print the progress for each category in your terminal.")
+    parser.add_argument("-v", "--version", help="version to process", choices=versionsChoices, default=versionsChoices[0])
+    parser.add_argument("-u", "--units", help="Print units instead of categories", action="store_true")
+    parser.add_argument("-s", "--sort", help="Sort by decomped size", action="store_true")
+    parser.add_argument("-r", "--remaining", help="Print an extra column indicating the remaining percentage to match of each entry", action="store_true")
 
-    mapPath = Path("build") / f"animalforest-{args.version}.map"
+    args = parser.parse_args()
+    version: str = args.version
+    units: bool = args.units
+    sort: bool = args.sort
+    remaining: bool = args.remaining
+
+    doThing(
+        version,
+        settings,
+        units,
+        sort,
+        remaining,
+    )
 
-    codeTotalStats, codeProgressPerFolder = getProgress(mapPath, args.version)
-    assetTotalStats, assetProgressPerFolder = getAssetProgress(mapPath, args.version)
-    print("code:")
-    mapfile_parser.progress_stats.printStats(codeTotalStats, codeProgressPerFolder)
-    print("\n\nassets:")
-    mapfile_parser.progress_stats.printStats(assetTotalStats, assetProgressPerFolder)
 
 if __name__ == "__main__":
     progressMain()
diff --git a/tools/upload_frogress.py b/tools/upload_frogress.py
deleted file mode 100755
index 480c8cf..0000000
--- a/tools/upload_frogress.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-# SPDX-FileCopyrightText: © 2023 ZeldaRET
-# SPDX-License-Identifier: MIT
-
-from __future__ import annotations
-
-import argparse
-import mapfile_parser
-from pathlib import Path
-
-import progress
-
-
-BASE_URL = "https://progress.deco.mp"
-PROJECT = "animalforest"
-
-def uploadProgressMain():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("version", help="Version slug")
-    parser.add_argument("--apikey", help="API key")
-
-    args = parser.parse_args()
-
-    version: str = args.version
-    apikey: str = args.apikey
-    mapPath = Path("build") / f"animalforest-{args.version}.map"
-
-    codeTotalStats, codeProgressPerFolder = progress.getProgress(mapPath, version)
-    codeEntries: dict[str, int] = mapfile_parser.frontends.upload_frogress.getFrogressEntriesFromStats(codeTotalStats, codeProgressPerFolder, verbose=True)
-
-    assetTotalStats, assetProgressPerFolder = progress.getAssetProgress(mapPath, version)
-    assetEntries: dict[str, int] = mapfile_parser.frontends.upload_frogress.getFrogressEntriesFromStats(assetTotalStats, assetProgressPerFolder, verbose=True)
-
-    url = mapfile_parser.utils.generateFrogressEndpointUrl(BASE_URL, PROJECT, version)
-    mapfile_parser.frontends.upload_frogress.uploadEntriesToFrogress(codeEntries, "code", url, apikey=apikey, verbose=True)
-    mapfile_parser.frontends.upload_frogress.uploadEntriesToFrogress(assetEntries, "assets", url, apikey=apikey, verbose=True)
-
-if __name__ == '__main__':
-    uploadProgressMain()