Build Sync (#1600)

* Rename outputs

* Makefile target renames

* Add run target

* yeet z64compress

* venv

* baserom_uncompressed -> baserom-decompressed

* input rom name to baserom.z64

* Add BUILD_DIR makefile variable

* Move built roms to build dir

* Move baserom to baseroms folder

* Add version to map file name

* Makefile cleanup

* Rename ldscript to include version

* Multiversion build

* n64-us version name

* Remove venv as dependency of setup

* Readme wording

* extract_baserom.py suggestion

* Readd checksums

* Make .venv work with windows

* missed an endif

* Cleaner windows venv implementation

* Remove duplciate process

* Build process steps

* Move make_options back

* Fix schedule build directory

* Fix schedule includes

* Makefile NON_MATCHING check -> != 0

* OOT 1704 changes

* Small cleanups

* Missed 1 thing

* UNSET -> SYMS

* Update extract_baserom.py

* dmadata.py

* Small cleanup

* dmadata_start

* Format

* dmadata files

* Fix makefile comment

* Jenkins report fix

* extracted dir

* Python dependencies order in readme

1 files changed, 188 insertions, 144 deletions

diff --git a/Makefile b/Makefile
index aacbfdf25..113be97f9 100644
--- a/Makefile
+++ b/Makefile
@@ -5,8 +5,35 @@
 
 MAKEFLAGS += --no-builtin-rules
 
+# Ensure the build fails if a piped command fails
+SHELL = /bin/bash
+.SHELLFLAGS = -o pipefail -c
+
+# OS Detection
+ifeq ($(OS),Windows_NT)
+  DETECTED_OS = windows
+  MAKE = make
+  VENV_BIN_DIR = Scripts
+else
+  UNAME_S := $(shell uname -s)
+  ifeq ($(UNAME_S),Linux)
+    DETECTED_OS = linux
+    MAKE = make
+    VENV_BIN_DIR = bin
+  endif
+  ifeq ($(UNAME_S),Darwin)
+    DETECTED_OS = macos
+    MAKE = gmake
+    VENV_BIN_DIR = bin
+  endif
+endif
+
+
 #### Defaults ####
 
+# Target game version. Currently only the following version is supported:
+#   n64-us   N64 USA (default)
+VERSION ?= n64-us
 # If COMPARE is 1, check the output md5sum after building
 COMPARE ?= 1
 # If NON_MATCHING is 1, define the NON_MATCHING C flag when building
@@ -29,15 +56,20 @@ ASM_PROC_FORCE ?= 0
 N_THREADS ?= $(shell nproc)
 # MIPS toolchain prefix
 MIPS_BINUTILS_PREFIX ?= mips-linux-gnu-
+# Python virtual environment
+VENV ?= .venv
 # Python interpreter
-PYTHON ?= python3
+PYTHON ?= $(VENV)/$(VENV_BIN_DIR)/python3
+# Emulator w/ flags
+N64_EMULATOR ?=
+
 
 #### Setup ####
 
 # Ensure the map file being created using English localization
 export LANG := C
 
-ifeq ($(NON_MATCHING),1)
+ifneq ($(NON_MATCHING),0)
   CFLAGS := -DNON_MATCHING
   CPPFLAGS := -DNON_MATCHING
   COMPARE := 0
@@ -48,25 +80,19 @@ ifneq ($(FULL_DISASM), 0)
   DISASM_FLAGS += --all
 endif
 
-PROJECT_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+PROJECT_DIR   := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+
+BASEROM_DIR   := baseroms/$(VERSION)
+BUILD_DIR     := build/$(VERSION)
+EXTRACTED_DIR := extracted/$(VERSION)
 
-MAKE = make
 CPPFLAGS += -P
 
-ifeq ($(OS),Windows_NT)
-  DETECTED_OS=windows
-else
-  UNAME_S := $(shell uname -s)
-  ifeq ($(UNAME_S),Linux)
-    DETECTED_OS=linux
-  endif
-  ifeq ($(UNAME_S),Darwin)
-    DETECTED_OS=macos
-    MAKE=gmake
-    CPPFLAGS += -xc++
-  endif
+ifeq ($(DETECTED_OS), macos)
+  CPPFLAGS += -xc++
 endif
 
+
 #### Tools ####
 ifneq ($(shell type $(MIPS_BINUTILS_PREFIX)ld >/dev/null 2>/dev/null; echo $$?), 0)
   $(error Unable to find $(MIPS_BINUTILS_PREFIX)ld. Please install or build MIPS binutils, commonly mips-linux-gnu. (or set MIPS_BINUTILS_PREFIX if your MIPS binutils install uses another prefix))
@@ -87,19 +113,20 @@ ifeq ($(ORIG_COMPILER),1)
   CC_OLD    = $(QEMU_IRIX) -L tools/ido5.3_compiler tools/ido5.3_compiler/usr/bin/cc
 endif
 
-AS         := $(MIPS_BINUTILS_PREFIX)as
-LD         := $(MIPS_BINUTILS_PREFIX)ld
-OBJCOPY    := $(MIPS_BINUTILS_PREFIX)objcopy
-OBJDUMP    := $(MIPS_BINUTILS_PREFIX)objdump
-ASM_PROC   := $(PYTHON) tools/asm-processor/build.py
+AS      := $(MIPS_BINUTILS_PREFIX)as
+LD      := $(MIPS_BINUTILS_PREFIX)ld
+NM      := $(MIPS_BINUTILS_PREFIX)nm
+OBJCOPY := $(MIPS_BINUTILS_PREFIX)objcopy
+OBJDUMP := $(MIPS_BINUTILS_PREFIX)objdump
 
+ASM_PROC   := $(PYTHON) tools/asm-processor/build.py
 ASM_PROC_FLAGS := --input-enc=utf-8 --output-enc=euc-jp --convert-statics=global-with-filename
 
 ifneq ($(ASM_PROC_FORCE), 0)
-	ASM_PROC_FLAGS += --force
+  ASM_PROC_FLAGS += --force
 endif
 
-IINC       := -Iinclude -Isrc -Iassets -Ibuild -I.
+IINC := -Iinclude -Isrc -Iassets -I$(BUILD_DIR) -I.
 
 ifeq ($(KEEP_MDEBUG),0)
   RM_MDEBUG = $(OBJCOPY) --remove-section .mdebug $@
@@ -125,11 +152,15 @@ ZAPD          := tools/ZAPD/ZAPD.out
 FADO          := tools/fado/fado.elf
 MAKEYAR       := $(PYTHON) tools/buildtools/makeyar.py
 CHECKSUMMER   := $(PYTHON) tools/buildtools/checksummer.py
-SHIFTJIS_CONV := $(PYTHON) tools/shiftjis_conv.py
+SHIFTJIS_CONV := $(PYTHON) tools/buildtools/shiftjis_conv.py
 SCHC          := $(PYTHON) tools/buildtools/schc.py
 
 SCHC_FLAGS  :=
 
+# Command to replace path variables in the spec file. We can't use the C
+# preprocessor for this because it won't substitute inside string literals.
+SPEC_REPLACE_VARS := sed -e 's|$$(BUILD_DIR)|$(BUILD_DIR)|g'
+
 OPTFLAGS := -O2 -g3
 ASFLAGS := -march=vr4300 -32 -Iinclude
 MIPS_VERSION := -mips2
@@ -156,23 +187,20 @@ else ifneq ($(RUN_CC_CHECK),0)
   CC_CHECK += -m32
 endif
 
-# rom compression flags
-COMPFLAGS := --threads $(N_THREADS)
-ifneq ($(NON_MATCHING),1)
-  COMPFLAGS += --matching
-endif
 
 #### Files ####
 
 # ROM image
-ROMC := mm.us.rev1.rom.z64
-ROM := $(ROMC:.rom.z64=.rom_uncompressed.z64)
-ELF := $(ROM:.z64=.elf)
+ROM      := $(BUILD_DIR)/mm-$(VERSION).z64
+ROMC     := $(ROM:.z64=-compressed.z64)
+ELF      := $(ROM:.z64=.elf)
+MAP      := $(ROM:.z64=.map)
+LDSCRIPT := $(ROM:.z64=.ld)
 # description of ROM segments
 SPEC := spec
 
 # create asm directories
-$(shell mkdir -p asm data)
+$(shell mkdir -p asm data extracted)
 
 SRC_DIRS := $(shell find src -type d)
 ASM_DIRS := $(shell find asm -type d -not -path "asm/non_matchings*") $(shell find data -type d)
@@ -183,117 +211,116 @@ ASSET_BIN_DIRS := $(shell find assets/* -type d -not -path "assets/xml*" -not -p
 ASSET_BIN_DIRS_C_FILES := $(shell find assets/* -type d -not -path "assets/xml*" -not -path "assets/code*" -not -path "assets/overlays*")
 
 ASSET_FILES_BIN := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.bin))
-ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_BIN:.bin=.bin.inc.c),build/$f)
+ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_BIN:.bin=.bin.inc.c),$(BUILD_DIR)/$f)
 
 TEXTURE_FILES_PNG := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.png))
 TEXTURE_FILES_JPG := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.jpg))
-TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG:.png=.inc.c),build/$f) \
-					 $(foreach f,$(TEXTURE_FILES_JPG:.jpg=.jpg.inc.c),build/$f) \
+TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG:.png=.inc.c),$(BUILD_DIR)/$f) \
+                     $(foreach f,$(TEXTURE_FILES_JPG:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f) \
 
 C_FILES       := $(foreach dir,$(SRC_DIRS) $(ASSET_BIN_DIRS_C_FILES),$(wildcard $(dir)/*.c))
-S_FILES       := $(shell grep -F "build/asm" spec | sed 's/.*build\/// ; s/\.o\".*/.s/') \
-                 $(shell grep -F "build/data" spec | sed 's/.*build\/// ; s/\.o\".*/.s/')
+S_FILES       := $(shell grep -F "\$$(BUILD_DIR)/asm" spec | sed 's/.*$$(BUILD_DIR)\/// ; s/\.o\".*/.s/') \
+                 $(shell grep -F "\$$(BUILD_DIR)/data" spec | sed 's/.*$$(BUILD_DIR)\/// ; s/\.o\".*/.s/')
 SCHEDULE_FILES:= $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.schl))
-BASEROM_FILES := $(shell grep -F "build/baserom" spec | sed 's/.*build\/// ; s/\.o\".*//')
-ARCHIVES_O    := $(shell grep -F ".yar.o" spec | sed 's/.*include "// ; s/\.o\".*/.o/')
-O_FILES       := $(foreach f,$(S_FILES:.s=.o),build/$f) \
-                 $(foreach f,$(C_FILES:.c=.o),build/$f) \
-                 $(foreach f,$(BASEROM_FILES),build/$f.o) \
-                 $(ARCHIVES_O)
+BASEROM_FILES := $(shell grep -F "\$$(BUILD_DIR)/baserom" spec | sed 's/.*$$(BUILD_DIR)\/// ; s/\.o\".*//')
+ARCHIVES_O    := $(shell grep -F ".yar.o" spec | sed 's/.*include "// ; s/.*$$(BUILD_DIR)\/// ; s/\.o\".*/.o/')
+O_FILES       := $(foreach f,$(S_FILES:.s=.o),$(BUILD_DIR)/$f) \
+                 $(foreach f,$(C_FILES:.c=.o),$(BUILD_DIR)/$f) \
+                 $(foreach f,$(BASEROM_FILES),$(BUILD_DIR)/$f.o) \
+                 $(foreach f,$(ARCHIVES_O),$(BUILD_DIR)/$f)
 
 SHIFTJIS_C_FILES	:= src/libultra/voice/voicecheckword.c src/audio/voice_external.c src/code/z_message.c src/code/z_message_nes.c
-SHIFTJIS_O_FILES	:= $(foreach f,$(SHIFTJIS_C_FILES:.c=.o),build/$f)
+SHIFTJIS_O_FILES	:= $(foreach f,$(SHIFTJIS_C_FILES:.c=.o),$(BUILD_DIR)/$f)
 
-OVL_RELOC_FILES := $(shell $(CPP) $(CPPFLAGS) $(SPEC) | grep -o '[^"]*_reloc.o' )
+OVL_RELOC_FILES := $(shell $(CPP) $(CPPFLAGS) $(SPEC) | $(SPEC_REPLACE_VARS) | grep -o '[^"]*_reloc.o' )
 
-SCHEDULE_INC_FILES := $(foreach f,$(SCHEDULE_FILES:.schl=.schl.inc),build/$f)
+SCHEDULE_INC_FILES := $(foreach f,$(SCHEDULE_FILES:.schl=.schl.inc),$(BUILD_DIR)/$f)
 
 # Automatic dependency files
 # (Only asm_processor dependencies and reloc dependencies are handled for now)
 DEP_FILES := $(O_FILES:.o=.asmproc.d) $(OVL_RELOC_FILES:.o=.d)
 
 # create build directories
-$(shell mkdir -p build/baserom $(foreach dir,$(SRC_DIRS) $(ASM_DIRS) $(ASSET_BIN_DIRS) $(ASSET_BIN_DIRS_C_FILES),build/$(dir)))
+$(shell mkdir -p $(BUILD_DIR)/baserom $(foreach dir,$(SRC_DIRS) $(ASM_DIRS) $(ASSET_BIN_DIRS) $(ASSET_BIN_DIRS_C_FILES),$(BUILD_DIR)/$(dir)))
 
 # directory flags
-build/src/boot/O2/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/libultra/os/%.o: OPTFLAGS := -O1
+$(BUILD_DIR)/src/libultra/voice/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/libultra/io/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/libultra/libc/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/libultra/gu/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/libultra/rmon/%.o: OPTFLAGS := -O2
 
-build/src/boot/libc/%.o: OPTFLAGS := -O2
-build/src/boot/libm/%.o: OPTFLAGS := -O2
-build/src/boot/libc64/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/boot/O2/%.o: OPTFLAGS := -O2
 
-build/src/libultra/os/%.o: OPTFLAGS := -O1
-build/src/libultra/voice/%.o: OPTFLAGS := -O2
-build/src/libultra/io/%.o: OPTFLAGS := -O2
-build/src/libultra/libc/%.o: OPTFLAGS := -O2
-build/src/libultra/gu/%.o: OPTFLAGS := -O2
-build/src/libultra/rmon/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/boot/libc/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/boot/libm/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/boot/libc64/%.o: OPTFLAGS := -O2
 
-build/src/audio/%.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/audio/%.o: OPTFLAGS := -O2
 
-build/assets/%.o: OPTFLAGS := -O1
-build/assets/%.o: ASM_PROC_FLAGS := 
+$(BUILD_DIR)/assets/%.o: OPTFLAGS := -O1
+$(BUILD_DIR)/assets/%.o: ASM_PROC_FLAGS := 
 
 # file flags
-build/src/boot/fault.o: CFLAGS += -trapuv
-build/src/boot/fault_drawer.o: CFLAGS += -trapuv
+$(BUILD_DIR)/src/libultra/libc/ll.o: OPTFLAGS := -O1
+$(BUILD_DIR)/src/libultra/libc/ll.o: MIPS_VERSION := -mips3 -32
+$(BUILD_DIR)/src/libultra/libc/llcvt.o: OPTFLAGS := -O1
+$(BUILD_DIR)/src/libultra/libc/llcvt.o: MIPS_VERSION := -mips3 -32
 
-build/src/code/jpegutils.o: OPTFLAGS := -O2
-build/src/code/jpegdecoder.o: OPTFLAGS := -O2
-build/src/code/jpegutils.o: CC := $(CC_OLD)
-build/src/code/jpegdecoder.o: CC := $(CC_OLD)
+$(BUILD_DIR)/src/boot/fault.o: CFLAGS += -trapuv
+$(BUILD_DIR)/src/boot/fault_drawer.o: CFLAGS += -trapuv
 
-build/src/code/osFlash.o: OPTFLAGS := -g
-build/src/code/osFlash.o: MIPS_VERSION := -mips1
-build/src/code/osFlash.o: CC := $(CC_OLD)
+$(BUILD_DIR)/src/code/jpegdecoder.o: CC := $(CC_OLD)
+$(BUILD_DIR)/src/code/jpegdecoder.o: OPTFLAGS := -O2
+$(BUILD_DIR)/src/code/jpegutils.o: CC := $(CC_OLD)
+$(BUILD_DIR)/src/code/jpegutils.o: OPTFLAGS := -O2
 
-build/src/libultra/libc/ll.o: OPTFLAGS := -O1
-build/src/libultra/libc/ll.o: MIPS_VERSION := -mips3 -32
-build/src/libultra/libc/llcvt.o: OPTFLAGS := -O1
-build/src/libultra/libc/llcvt.o: MIPS_VERSION := -mips3 -32
+$(BUILD_DIR)/src/code/osFlash.o: CC := $(CC_OLD)
+$(BUILD_DIR)/src/code/osFlash.o: OPTFLAGS := -g
+$(BUILD_DIR)/src/code/osFlash.o: MIPS_VERSION := -mips1
 
 # cc & asm-processor
-build/src/boot/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
-build/src/boot/O2/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
+$(BUILD_DIR)/src/libultra/%.o: CC := $(CC_OLD)
+
+$(BUILD_DIR)/src/boot/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
+$(BUILD_DIR)/src/boot/O2/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
 
-build/src/libultra/%.o: CC := $(CC_OLD)
+$(BUILD_DIR)/src/code/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
+$(BUILD_DIR)/src/audio/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
 
-build/src/code/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
-build/src/audio/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
+$(BUILD_DIR)/src/overlays/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
 
-build/src/overlays/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
+$(BUILD_DIR)/assets/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
 
-build/assets/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
 
 $(SHIFTJIS_O_FILES): CC_CHECK += -Wno-multichar -Wno-type-limits -Wno-overflow
 
 #### Main Targets ###
 
-uncompressed: $(ROM)
-ifeq ($(COMPARE),1)
+rom: $(ROM)
+ifneq ($(COMPARE),0)
 	@md5sum $(ROM)
-	@md5sum -c checksum_uncompressed.md5
+	@md5sum -c $(BASEROM_DIR)/checksum.md5
 endif
 
-compressed: $(ROMC)
-ifeq ($(COMPARE),1)
+compress: $(ROMC)
+ifneq ($(COMPARE),0)
 	@md5sum $(ROMC)
-	@md5sum -c checksum.md5
+	@md5sum -c $(BASEROM_DIR)/checksum-compressed.md5
 endif
 
-.PHONY: all uncompressed compressed clean assetclean distclean assets disasm init setup
-.DEFAULT_GOAL := uncompressed
-all: uncompressed compressed
 
 $(ROM): $(ELF)
 	$(OBJCOPY) --gap-fill=0x00 -O binary $< $@
 	$(CHECKSUMMER) $@
 
-$(ROMC): $(ROM)
-	$(PYTHON) tools/z64compress_wrapper.py $(COMPFLAGS) $(ROM) $@ $(ELF) build/$(SPEC)
+$(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt
+	$(PYTHON) tools/buildtools/compress.py --in $(ROM) --out $@ --dma-start `tools/buildtools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
+	$(PYTHON) -m ipl3checksum sum --cic 6105 --update $@
 
-$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) build/ldscript.txt build/undefined_syms.txt
-	$(LD) -T build/undefined_syms.txt -T build/ldscript.txt --no-check-sections --accept-unknown-input-arch --emit-relocs -Map build/mm.map -o $@
+$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/undefined_syms.txt
+	$(LD) -T $(LDSCRIPT) -T $(BUILD_DIR)/undefined_syms.txt --no-check-sections --accept-unknown-input-arch --emit-relocs -Map $(MAP) -o $@
 
 ## Order-only prerequisites 
 # These ensure e.g. the O_FILES are built before the OVL_RELOC_FILES.
@@ -310,127 +337,144 @@ $(O_FILES): | schedule_inc_files
 
 .PHONY: o_files asset_files schedule_inc_files
 
+
 #### Main commands ####
 
 ## Cleaning ##
 clean:
-	$(RM) -rf $(ROMC) $(ROM) $(ELF) build
+	$(RM) -r $(BUILD_DIR)
 
 assetclean:
-	$(RM) -rf $(ASSET_BIN_DIRS)
-	$(RM) -rf assets/text/*.h
-	$(RM) -rf build/assets
-	$(RM) -rf .extracted-assets.json
+	$(RM) -r $(ASSET_BIN_DIRS)
+	$(RM) -r $(BUILD_DIR)/assets
+	$(RM) -r assets/text/*.h
+	$(RM) -r .extracted-assets.json
 
 distclean: assetclean clean
-	$(RM) -rf asm baserom data
+	$(RM) -r asm data extracted
 	$(MAKE) -C tools clean
 
+venv:
+	test -d $(VENV) || python3 -m venv $(VENV)
+	$(PYTHON) -m pip install -U pip
+	$(PYTHON) -m pip install -U -r requirements.txt
+
 ## Extraction step
 setup:
 	$(MAKE) -C tools
-	$(PYTHON) tools/fixbaserom.py
-	$(PYTHON) tools/extract_baserom.py
-	$(PYTHON) tools/decompress_yars.py
+	$(PYTHON) tools/buildtools/decompress_baserom.py $(VERSION)
+	$(PYTHON) tools/buildtools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 -o $(EXTRACTED_DIR)/baserom --dmadata-start `cat $(BASEROM_DIR)/dmadata_start.txt` --dmadata-names $(BASEROM_DIR)/dmadata_names.txt
+	$(PYTHON) tools/buildtools/extract_yars.py $(VERSION)
 
 assets:
 	$(PYTHON) extract_assets.py -j $(N_THREADS) -Z Wno-hardcoded-pointer
 
 ## Assembly generation
 disasm:
-	$(RM) -rf asm data
+	$(RM) -r asm data
 	$(PYTHON) tools/disasm/disasm.py -j $(N_THREADS) $(DISASM_FLAGS)
 
-diff-init: uncompressed
-	$(RM) -rf expected/
+diff-init: rom
+	$(RM) -r expected/
 	mkdir -p expected/
 	cp -r build expected/build
 
-init:
-	$(MAKE) distclean
+init: distclean
+	$(MAKE) venv
 	$(MAKE) setup
 	$(MAKE) assets
 	$(MAKE) disasm
 	$(MAKE) all
 	$(MAKE) diff-init
 
+run: $(ROM)
+ifeq ($(N64_EMULATOR),)
+	$(error Emulator path not set. Set N64_EMULATOR in the Makefile, .make_options, or define it as an environment variable)
+endif
+	$(N64_EMULATOR) $<
+
+.PHONY: all rom compress clean assetclean distclean assets disasm init venv setup run
+.DEFAULT_GOAL := rom
+all: rom compress
+
+
 #### Various Recipes ####
 
-build/undefined_syms.txt: undefined_syms.txt
-	$(CPP) $(CPPFLAGS) $< > build/undefined_syms.txt
+$(BUILD_DIR)/undefined_syms.txt: undefined_syms.txt
+	$(CPP) $(CPPFLAGS) $< > $(BUILD_DIR)/undefined_syms.txt
 
-build/$(SPEC): $(SPEC)
-	$(CPP) $(CPPFLAGS) $< > $@
+$(BUILD_DIR)/$(SPEC): $(SPEC)
+	$(CPP) $(CPPFLAGS) $< | $(SPEC_REPLACE_VARS) > $@
 
-build/ldscript.txt: build/$(SPEC)
+$(LDSCRIPT): $(BUILD_DIR)/$(SPEC)
 	$(MKLDSCRIPT) $< $@
 
-build/dmadata_table_spec.h: build/$(SPEC)
-	$(MKDMADATA) $< $@
+$(BUILD_DIR)/dmadata_table_spec.h $(BUILD_DIR)/compress_ranges.txt: $(BUILD_DIR)/$(SPEC)
+	$(MKDMADATA) $< $(BUILD_DIR)/dmadata_table_spec.h $(BUILD_DIR)/compress_ranges.txt
 
 # Dependencies for files that may include the dmadata header automatically generated from the spec file
-build/src/boot/z_std_dma.o: build/dmadata_table_spec.h
-build/src/dmadata/dmadata.o: build/dmadata_table_spec.h
+$(BUILD_DIR)/src/boot/z_std_dma.o: $(BUILD_DIR)/dmadata_table_spec.h
+$(BUILD_DIR)/src/dmadata/dmadata.o: $(BUILD_DIR)/dmadata_table_spec.h
 
-build/asm/%.o: asm/%.s
+$(BUILD_DIR)/asm/%.o: asm/%.s
 	$(AS) $(ASFLAGS) $< -o $@
 
-build/assets/%.o: assets/%.c
+$(BUILD_DIR)/assets/%.o: assets/%.c
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
 	$(OBJCOPY_BIN)
 	$(RM_MDEBUG)
 
-build/%.yar.o: build/%.o
+$(BUILD_DIR)/%.yar.o: $(BUILD_DIR)/%.o
 	$(MAKEYAR) $< $(@:.yar.o=.yar.bin) $(@:.yar.o=.symbols.o)
 	$(OBJCOPY) -I binary -O elf32-big $(@:.yar.o=.yar.bin) $@
 
-build/baserom/%.o: baserom/%
+$(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/%
 	$(OBJCOPY) -I binary -O elf32-big $< $@
 
-build/data/%.o: data/%.s
+$(BUILD_DIR)/data/%.o: data/%.s
 	$(AS) $(ASFLAGS) $< -o $@
 
-build/assets/text/message_data.enc.h: assets/text/message_data.h
+$(BUILD_DIR)/assets/text/message_data.enc.h: assets/text/message_data.h
 	python3 tools/msg/nes/msgencNES.py -o $@ $<
 
-build/assets/text/staff_message_data.enc.h: assets/text/staff_message_data.h
+$(BUILD_DIR)/assets/text/staff_message_data.enc.h: assets/text/staff_message_data.h
 	python3 tools/msg/staff/msgencStaff.py -o $@ $<
 
-build/assets/text/message_data_static.o: build/assets/text/message_data.enc.h
-build/assets/text/staff_message_data_static.o: build/assets/text/staff_message_data.enc.h
-build/src/code/z_message.o: build/assets/text/message_data.enc.h build/assets/text/staff_message_data.enc.h
+$(BUILD_DIR)/assets/text/message_data_static.o: $(BUILD_DIR)/assets/text/message_data.enc.h
+$(BUILD_DIR)/assets/text/staff_message_data_static.o: $(BUILD_DIR)/assets/text/staff_message_data.enc.h
+$(BUILD_DIR)/src/code/z_message.o: $(BUILD_DIR)/assets/text/message_data.enc.h $(BUILD_DIR)/assets/text/staff_message_data.enc.h
+
+$(SHIFTJIS_O_FILES): $(BUILD_DIR)/src/%.o: src/%.c
+	$(SHIFTJIS_CONV) -o $(@:.o=.enc.c) $<
+	$(CC_CHECK) $(@:.o=.enc.c)
+	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $(@:.o=.enc.c)
+	$(OBJDUMP_CMD)
+	$(RM_MDEBUG)
 
-build/src/overlays/%.o: src/overlays/%.c
+$(BUILD_DIR)/src/overlays/%.o: src/overlays/%.c
 	$(CC_CHECK) $<
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
-	@$(OBJDUMP) -d $@ > $(@:.o=.s)
+	$(OBJDUMP_CMD)
 	$(RM_MDEBUG)
 
-build/src/overlays/%_reloc.o: build/$(SPEC)
+$(BUILD_DIR)/src/overlays/%_reloc.o: $(BUILD_DIR)/$(SPEC)
 	$(FADO) $$(tools/buildtools/reloc_prereq $< $(notdir $*)) -n $(notdir $*) -o $(@:.o=.s) -M $(@:.o=.d)
 	$(AS) $(ASFLAGS) $(@:.o=.s) -o $@
 
-build/src/%.o: src/%.c
+$(BUILD_DIR)/src/%.o: src/%.c
 	$(CC_CHECK) $<
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
 	$(OBJDUMP_CMD)
 	$(RM_MDEBUG)
 
-$(SHIFTJIS_O_FILES): build/src/%.o: src/%.c
-	$(SHIFTJIS_CONV) -o $(@:.o=.enc.c) $<
-	$(CC_CHECK) $(@:.o=.enc.c)
-	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $(@:.o=.enc.c)
-	$(OBJDUMP_CMD)
-	$(RM_MDEBUG)
-
-build/src/libultra/libc/ll.o: src/libultra/libc/ll.c
+$(BUILD_DIR)/src/libultra/libc/ll.o: src/libultra/libc/ll.c
 	$(CC_CHECK) $<
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
 	$(PYTHON) tools/set_o32abi_bit.py $@
 	$(OBJDUMP_CMD)
 	$(RM_MDEBUG)
 
-build/src/libultra/libc/llcvt.o: src/libultra/libc/llcvt.c
+$(BUILD_DIR)/src/libultra/libc/llcvt.o: src/libultra/libc/llcvt.c
 	$(CC_CHECK) $<
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
 	$(PYTHON) tools/set_o32abi_bit.py $@
@@ -439,16 +483,16 @@ build/src/libultra/libc/llcvt.o: src/libultra/libc/llcvt.c
 
 # Build C files from assets
 
-build/%.inc.c: %.png
+$(BUILD_DIR)/%.inc.c: %.png
 	$(ZAPD) btex -eh -tt $(subst .,,$(suffix $*)) -i $< -o $@
 
-build/assets/%.bin.inc.c: assets/%.bin
+$(BUILD_DIR)/assets/%.bin.inc.c: assets/%.bin
 	$(ZAPD) bblb -eh -i $< -o $@
 
-build/assets/%.jpg.inc.c: assets/%.jpg
+$(BUILD_DIR)/assets/%.jpg.inc.c: assets/%.jpg
 	$(ZAPD) bren -eh -i $< -o $@
 
-build/%.schl.inc: %.schl
+$(BUILD_DIR)/%.schl.inc: %.schl
 	$(SCHC) $(SCHC_FLAGS) -o $@ $<
 
 -include $(DEP_FILES)