Partial linking of spec segments (#2661)

* Partial linking of overlay segments, relax linker script alignment

* Partial linking of all spec segments

* Fix update, remove _RomPos from linker script

* iQue version working pending COM-plugin update

* Add plf map file resolution to sym_info.py, local symbol merging is broken

* git subrepo pull tools/com-plugin

subrepo:
  subdir:   "tools/com-plugin"
  merged:   "c4f3ba845"
upstream:
  origin:   "git@github.com:Thar0/com-plugin.git"
  branch:   "main"
  commit:   "c4f3ba845"
git-subrepo:
  version:  "0.4.6"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "110b9eb"

* Make tools compatible with partial linking

Co-authored-by: Anghelo Carvajal <angheloalf95@gmail.com>

* Remove unused files

* Fix some makefile bits

* mkspecrules cleanup

* Comment on the makerom linker layout in mkldscript

* Revert linker padding strategy back to pad_text spec directives

* Comment on objcopy elf -> rom step

* Adjust tool descriptions

* Fix compressed builds

* rm reloc_prereq, no longer used

* Makefile merge fix

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

---------

Co-authored-by: Anghelo Carvajal <angheloalf95@gmail.com>
Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

1 files changed, 72 insertions, 40 deletions

diff --git a/Makefile b/Makefile
index 08fc532a4..40a44ceed 100644
--- a/Makefile
+++ b/Makefile
@@ -324,6 +324,8 @@ else
   ICONV := iconv
 endif
 
+LD_OFORMAT := $(shell $(LD) --print-output-format)
+
 INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I. -I$(EXTRACTED_DIR)
 
 # Check code syntax with host compiler
@@ -332,15 +334,15 @@ CHECK_WARNINGS += -Werror=implicit-int -Werror=implicit-function-declaration -We
 
 # The `cpp` command behaves differently on macOS (it behaves as if
 # `-traditional-cpp` was passed) so we use `gcc -E` instead.
-CPP        := gcc -E
-MKLDSCRIPT := tools/mkldscript
-MKDMADATA  := tools/mkdmadata
-ELF2ROM    := tools/elf2rom
-BIN2C      := tools/bin2c
-FADO       := tools/fado/fado.elf
-PYTHON     ?= $(VENV)/bin/python3
+CPP            := gcc -E
+MKLDSCRIPT     := tools/mkldscript
+MKSPECRULES    := tools/mkspecrules
+MKDMADATA      := tools/mkdmadata
+BIN2C          := tools/bin2c
+FADO           := tools/fado/fado.elf
+PYTHON         ?= $(VENV)/bin/python3
 BUILD_FROM_PNG := tools/assets/build_from_png/build_from_png
-BUILD_JFIF := tools/assets/build_jfif/build_jfif
+BUILD_JFIF     := tools/assets/build_jfif/build_jfif
 
 # Command to replace $(BUILD_DIR) in some files with the build path.
 # We can't use the C preprocessor for this because it won't substitute inside string literals.
@@ -510,16 +512,6 @@ ASSET_FILES_BIN_COMMITTED := $(foreach dir,$(ASSET_BIN_DIRS_COMMITTED),$(wildcar
 ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_BIN_EXTRACTED:.bin=.bin.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \
                    $(foreach f,$(ASSET_FILES_BIN_COMMITTED:.bin=.bin.inc.c),$(BUILD_DIR)/$f)
 
-# Find all .o files included in the spec
-SPEC_O_FILES := $(shell $(CPP) $(CPPFLAGS) -I. $(SPEC) | $(BUILD_DIR_REPLACE) | sed -n -E 's/^[ \t]*include[ \t]*"([a-zA-Z0-9/_.-]+\.o)"/\1/p')
-
-# Split out reloc files
-O_FILES := $(filter-out %_reloc.o,$(SPEC_O_FILES))
-OVL_RELOC_FILES := $(filter %_reloc.o,$(SPEC_O_FILES))
-
-# Automatic dependency files
-DEP_FILES := $(O_FILES:.o=.d) $(O_FILES:.o=.asmproc.d) $(OVL_RELOC_FILES:.o=.d) $(BUILD_DIR)/spec.d
-
 TEXTURE_FILES_PNG_EXTRACTED := $(foreach dir,$(ASSET_BIN_DIRS_EXTRACTED),$(wildcard $(dir)/*.png))
 TEXTURE_FILES_PNG_COMMITTED := $(foreach dir,$(ASSET_BIN_DIRS_COMMITTED),$(wildcard $(dir)/*.png))
 TEXTURE_FILES_JPG_EXTRACTED := $(foreach dir,$(ASSET_BIN_DIRS_EXTRACTED),$(wildcard $(dir)/*.jpg))
@@ -529,10 +521,13 @@ TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG_EXTRACTED:.png=.inc.c),$(f:
                      $(foreach f,$(TEXTURE_FILES_JPG_EXTRACTED:.jpg=.jpg.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \
                      $(foreach f,$(TEXTURE_FILES_JPG_COMMITTED:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f)
 
+SEGMENTS_DIR := $(BUILD_DIR)/segments
+
 # create build directories
 $(shell mkdir -p $(BUILD_DIR)/baserom \
                  $(BUILD_DIR)/assets/text \
-                 $(BUILD_DIR)/linker_scripts)
+                 $(BUILD_DIR)/linker_scripts \
+				 $(SEGMENTS_DIR))
 $(shell mkdir -p $(foreach dir, \
                       $(SRC_DIRS) \
                       $(UNDECOMPILED_DATA_DIRS) \
@@ -552,6 +547,40 @@ $(shell mkdir -p $(foreach dir, \
                     $(dir:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)))
 endif
 
+COM_PLUGIN := tools/com-plugin/common-plugin.so
+COM_PLUGIN_FLAGS =
+ifeq ($(PLATFORM),IQUE)
+  ifeq ($(NON_MATCHING),0)
+    $(SEGMENTS_DIR)/boot.plf: $(BASEROM_DIR)/bss-order-boot.txt
+    $(SEGMENTS_DIR)/boot.plf: COM_PLUGIN_FLAGS += -plugin $(COM_PLUGIN) -plugin-opt order=$(BASEROM_DIR)/bss-order-boot.txt -plugin-opt min_align=0x10
+
+    $(SEGMENTS_DIR)/code.plf: $(BASEROM_DIR)/bss-order-code.txt
+    $(SEGMENTS_DIR)/code.plf: COM_PLUGIN_FLAGS += -plugin $(COM_PLUGIN) -plugin-opt order=$(BASEROM_DIR)/bss-order-code.txt -plugin-opt min_align=0x10
+  endif
+endif
+
+# Generate and include segment makefile rules for combining .o files into single .plf files for an entire spec segment.
+# Overlay relocations will be generated from these if the spec segment has the OVERLAY flag.
+# If this makefile doesn't exist or if the spec has been modified since make was last ran it will use the rule
+# later on in the file to regenerate this file before including it. The test against MAKECMDGOALS ensures this
+# doesn't happen if we're not running a task that needs these partially linked files; this is especially important
+# for setup since the rule to generate the segment makefile rules requires setup to have ran first.
+SEG_LDFLAGS = -r $(COM_PLUGIN_FLAGS) -T $(@:.plf=.ld) -Map $(@:.plf=.map)
+SEG_VERBOSE = @
+ifeq ($(MAKECMDGOALS),$(filter-out clean assetclean distclean setup,$(MAKECMDGOALS)))
+include $(SEGMENTS_DIR)/Makefile
+else
+SEGMENT_FILES :=
+OVL_SEGMENT_FILES :=
+endif
+OVL_RELOC_FILES := $(OVL_SEGMENT_FILES:.plf=.reloc.o)
+
+O_FILES := $(shell $(CPP) $(CPPFLAGS) -I. $(SPEC) | $(BUILD_DIR_REPLACE) | sed -n -E 's/^[ \t]*include[ \t]*"([a-zA-Z0-9/_.-]+\.o)"/\1/p')
+MAKEROM_O_FILES := $(BUILD_DIR)/src/makerom/rom_header.o $(BUILD_DIR)/src/makerom/ipl3.o $(BUILD_DIR)/src/makerom/entry.o
+
+# Automatic dependency files
+DEP_FILES := $(O_FILES:.o=.d) $(O_FILES:.o=.asmproc.d) $(OVL_RELOC_FILES:.o=.d) $(BUILD_DIR)/spec.d $(MAKEROM_O_FILES:.o=.d)
+
 $(BUILD_DIR)/src/boot/build.o: CPP_DEFINES += -DBUILD_CREATOR="\"$(BUILD_CREATOR)\"" -DBUILD_DATE="\"$(BUILD_DATE)\"" -DBUILD_TIME="\"$(BUILD_TIME)\""
 
 $(BUILD_DIR)/src/audio/internal/seqplayer.o: CPP_DEFINES += -DMML_VERSION=MML_VERSION_OOT
@@ -760,7 +789,7 @@ else
 $(BUILD_DIR)/assets/%.o: CFLAGS += -fno-zero-initialized-in-bss -fno-toplevel-reorder
 $(BUILD_DIR)/src/%.o: CFLAGS += -fexec-charset=euc-jp
 $(BUILD_DIR)/src/libultra/libc/ll.o: OPTFLAGS := -Ofast
-$(BUILD_DIR)/src/overlays/%.o: CFLAGS += -fno-merge-constants -mno-explicit-relocs -mno-split-addresses
+$(BUILD_DIR)/src/overlays/%.o: CFLAGS += -mno-explicit-relocs -mno-split-addresses
 endif
 
 #### Main Targets ###
@@ -839,23 +868,18 @@ else
 endif
 
 $(ROM): $(ELF)
-	$(ELF2ROM) -cic $(CIC) $< $@
+# Here we extract the value of the _RomSize symbol to know to what size the ROM should be padded to
+	$(OBJCOPY) --pad-to 0x$$($(OBJDUMP) -t $< | grep _RomSize | cut -d ' ' -f 1) -O binary $< $@
+	$(PYTHON) -m ipl3checksum sum --cic $(CIC) --update $@
 
 $(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt
 	$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dmadata-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS) $(COMPRESS_ARGS)
 	$(PYTHON) -m ipl3checksum sum --cic $(CIC) --update $@
 
-COM_PLUGIN := tools/com-plugin/common-plugin.so
+LDFLAGS := -T $(LDSCRIPT) -T $(BUILD_DIR)/linker_scripts/makerom.ld -T $(BUILD_DIR)/undefined_syms.txt --emit-relocs -Map $(MAP)
 
-LDFLAGS := -T $(LDSCRIPT) -T $(BUILD_DIR)/linker_scripts/makerom.ld -T $(BUILD_DIR)/undefined_syms.txt --no-check-sections --accept-unknown-input-arch --emit-relocs -Map $(MAP)
-ifeq ($(PLATFORM),IQUE)
-  ifeq ($(NON_MATCHING),0)
-    LDFLAGS += -plugin $(COM_PLUGIN) -plugin-opt order=$(BASEROM_DIR)/bss-order.txt
-    $(ELF): $(BASEROM_DIR)/bss-order.txt
-  endif
-endif
-
-$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/linker_scripts/makerom.ld $(BUILD_DIR)/undefined_syms.txt \
+$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(SEGMENT_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(MAKEROM_O_FILES) \
+        $(BUILD_DIR)/linker_scripts/makerom.ld $(BUILD_DIR)/undefined_syms.txt \
         $(SAMPLEBANK_O_FILES) $(SOUNDFONT_O_FILES) $(SEQUENCE_O_FILES) \
         $(BUILD_DIR)/assets/audio/sequence_font_table.o $(BUILD_DIR)/assets/audio/audiobank_padding.o
 	$(LD) $(LDFLAGS) -o $@
@@ -879,13 +903,26 @@ $(BUILD_DIR)/spec: $(SPEC) $(SPEC_INCLUDES)
 	$(CPP) $(CPPFLAGS) -MD -MP -MF $@.d -MT $@ -I. $< | $(BUILD_DIR_REPLACE) > $@
 
 $(LDSCRIPT): $(BUILD_DIR)/spec
-	$(MKLDSCRIPT) $< $@
+	$(MKLDSCRIPT) $< $@ $(BUILD_DIR)/src/makerom $(SEGMENTS_DIR)
+
+# Generates a makefile containing rules for building .plf files
+# from overlay .o files for every overlay defined in the spec.
+$(SEGMENTS_DIR)/Makefile: $(BUILD_DIR)/spec
+	$(MKSPECRULES) $< $(SEGMENTS_DIR) $@
+
+# Generates relocations for each overlay after partial linking so that the final
+# link step cannot later insert padding between individual overlay files after
+# relocations have already been calculated.
+$(SEGMENTS_DIR)/%.reloc.o: $(SEGMENTS_DIR)/%.plf
+	$(FADO) $< -n $(notdir $*) -o $(@:.o=.s)
+	$(POSTPROCESS_OBJ) $(@:.o=.s)
+	$(AS) $(ASFLAGS) $(@:.o=.s) -o $@
 
 $(BUILD_DIR)/undefined_syms.txt: undefined_syms.txt
 	$(CPP) $(CPPFLAGS) $< > $@
 
 $(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/%
-	$(OBJCOPY) -I binary -O elf32-big $< $@
+	$(OBJCOPY) -I binary -O $(LD_OFORMAT) $< $@
 
 $(BUILD_DIR)/data/%.o: data/%.s
 	$(CPP) $(CPPFLAGS) -MD -MP -MF $(@:.o=.d) -MT $@ -Iinclude $< | $(AS) $(ASFLAGS) -o $@
@@ -940,7 +977,7 @@ endif
 	$(OBJDUMP_CMD)
 
 $(BUILD_DIR)/src/makerom/ipl3.o: $(EXTRACTED_DIR)/incbin/ipl3
-	$(OBJCOPY) -I binary -O elf32-big --rename-section .data=.text $< $@
+	$(OBJCOPY) -I binary -O $(LD_OFORMAT) --rename-section .data=.text $< $@
 
 $(BUILD_DIR)/src/%.o: src/%.s
 ifeq ($(COMPILER),ido)
@@ -988,15 +1025,10 @@ $(BUILD_DIR)/src/audio/game/session_init.o: src/audio/game/session_init.c $(BUIL
 
 ifeq ($(PLATFORM),IQUE)
 ifneq ($(NON_MATCHING),1)
-$(BUILD_DIR)/src/overlays/misc/ovl_kaleido_scope/ovl_kaleido_scope_reloc.o: POSTPROCESS_OBJ := $(PYTHON) tools/patch_ique_kaleido_reloc.py
+$(BUILD_DIR)/segments/ovl_kaleido_scope.reloc.o: POSTPROCESS_OBJ := $(PYTHON) tools/patch_ique_kaleido_reloc.py
 endif
 endif
 
-$(BUILD_DIR)/src/overlays/%_reloc.o: $(BUILD_DIR)/spec
-	$(FADO) $$(tools/reloc_prereq $< $(notdir $*)) -n $(notdir $*) -o $(@:.o=.s) -M $(@:.o=.d)
-	$(POSTPROCESS_OBJ) $(@:.o=.s)
-	$(AS) $(ASFLAGS) $(@:.o=.s) -o $@
-
 # Assets from assets/
 
 $(BUILD_DIR)/assets/%.inc.c: assets/%.png