New assets system (#2481)

* wip: New assets system tm

Builds gc-eu-mq-dbg OK from clean after
1) make setup
2) python3 -m tools.assets.extract -j
3) replace 0x80A8E610 with sShadowTex in extracted/gc-eu-mq-dbg/assets/overlays/ovl_En_Jsjutan/sShadowMaterialDL.inc.c
4) make various symbols in extracted data like sTex static

* use variables from config.yml for gMtxClear and sShadowTex addresses

* Write source with static for overlays using `HACK_IS_STATIC_ON` hack

* gc-eu-mq-dbg OK from clean with `make setup && make`

* implement more skeleton-related types, cleanups, fixups

* fix extracted data to no longer produce compilation warnings

* implement more of RoomShapeImage types

* yeet XmlPath from ExternalFile usage

* Implement PlayerAnimationDataResource (link_animetion data)

* fix csdis CS_TIME extra arg

* dmadata file names no longer hardcoded for gc-eu-mq-dbg

* ntsc-1.0 OK

* xml fixes

* slightly improve standard output

* rm extract_assets.py

* generate and use Limb enums (TODO: check Skin skels and implement for Curve skels)

* handle dependencies between xmls

* introduce RawPointers xml attribute to ignore specific pointers and keep them raw

* add tools/extract_assets.sh

* fixups

* only extract if xmls changed or if -f (force) is used

* fixups, gc-eu OK

* all versions OK

* check attributes of xml resources elements

* Implement legacy skelanime resources

* fix ASSET_FILES_BIN_EXTRACTED/COMMITTED: look for .u8.bin specifically instead of just .bin

* implement JFIFResource

* fix png/jpg wildcards: look specifically for .u64.png .u32.png .u64.jpg

* Makefile: Add rules to build .png, .bin and .jpg in assets/ too

* start writing actual docs

* extract sTransCircleDL and sTransWipeDL

* misc cleanup/fixes, pygfxd 1.0.3

* refactor CDataExt.set_write callback args to use a dataclass

* Move {} to in-source

* misc

* more progress on spec

* fix missing braces in n64dd_error_textures.c

* finish xml spec doc

* assets xmls fixes

* some cleanup, use `gNameTex_WIDTH/HEIGHT` macros in dlists

* handle hackmode_syotes_room, fix compile

* C build_from_png

* rm tools/assets/bin2c

* rm ZAPD

* format

* remove rule to generate dmadata_table.py

* CC0 license (and some import cleanup)

* dont try to build zapd (rmd)

* simplify palettes with single user (ci images with a non-shared palette)

* add docs on how images are handled

* bss

* allow -j N

* fix n64texconv python bindings memory management

* move -j at the end of calling extraction script

* with -j, update last_extracts.json as each job completes rather than only if all complete

* make interrupting less jank by making child processes ignore sigint

* use enum names in `SCENE_CMD_SKYBOX_SETTINGS`

* `multiprocessing.get_context("fork")`

* import rich, except ImportError s

* fix optional rich usage

* .bss

* .bss

* .bss

* assets extraction: -j -> -j$(N_THREADS)

* .bss

* change LIMB_NONE/MAX defaults to be FILE_OFFSET instead of SKELNAME

* 0XHEX -> 0xHEX

* fix bss

* Proper includes for assets

mostly proper, some includes like dlists resources always causing a sys_matrix.h include (when not every dlist references gIdentityMtx) could be done better

* rm z64.h

* rm z64.h take two

* bss

* Make .u64 suffix for pngs optional

* fixup: rm .u64 suffix from n64dd image paths

* Remove elemtype suffixes from .bin and .jpg files

* Update images.md

* some build_from_png cleanup, more error handling, comments

* Handle skybox textures

Introduce "sub-format" suffix for pngs, with sub-formats split_lo and split_hi being used for skybox textures

* fixup for older python

* improve collision output some

* fully use SURFACETYPE[01] macros in writing extracted surface types

* use WATERBOX_PROPERTIES in extracted waterboxes

* some SceneCommandsResource cleanup

* format EnvLightSettingsList output

7 files changed, 12 insertions, 65 deletions

diff --git a/src/code/z_actor.c b/src/code/z_actor.c
index 721087332..83105d23e 100644
--- a/src/code/z_actor.c
+++ b/src/code/z_actor.c
@@ -35,7 +35,7 @@
 #include "assets/objects/gameplay_dangeon_keep/gameplay_dangeon_keep.h"
 #include "assets/objects/object_bdoor/object_bdoor.h"
 
-#pragma increment_block_number "gc-eu:128 gc-eu-mq:128 gc-jp:0 gc-jp-ce:0 gc-jp-mq:0 gc-us:0 gc-us-mq:0 ntsc-1.0:128" \
+#pragma increment_block_number "gc-eu:128 gc-eu-mq:128 gc-jp:0 gc-jp-ce:0 gc-jp-mq:0 gc-us:0 gc-us-mq:0 ntsc-1.0:0" \
                                "ntsc-1.1:0 ntsc-1.2:0 pal-1.0:0 pal-1.1:0"
 
 CollisionPoly* sCurCeilingPoly;
diff --git a/src/code/z_eff_blure.c b/src/code/z_eff_blure.c
index 614cfa5b5..6e2023b67 100644
--- a/src/code/z_eff_blure.c
+++ b/src/code/z_eff_blure.c
@@ -9,7 +9,6 @@
 #include "z64effect.h"
 #include "z64skin_matrix.h"
 
-#include "z64.h" // required for gameplay keep, the header doesnt include any external dependencies
 #include "assets/objects/gameplay_keep/gameplay_keep.h"
 
 void EffectBlure_AddVertex(EffectBlure* this, Vec3f* p1, Vec3f* p2) {
diff --git a/src/code/z_fbdemo_circle.c b/src/code/z_fbdemo_circle.c
index 499867d57..d8aa81df0 100644
--- a/src/code/z_fbdemo_circle.c
+++ b/src/code/z_fbdemo_circle.c
@@ -3,6 +3,7 @@
 #include "color.h"
 #include "gfx.h"
 #include "sfx.h"
+#include "z64transition.h"
 
 typedef enum TransitionCircleDirection {
     /* 0 */ TRANS_CIRCLE_DIR_IN,
@@ -14,34 +15,7 @@ Gfx sTransCircleEmptyDL[] = {
     gsSPEndDisplayList(),
 };
 
-#include "assets/code/fbdemo_circle/z_fbdemo_circle.c"
-
-Gfx sTransCircleDL[] = {
-    gsDPPipeSync(),
-    gsSPClearGeometryMode(G_ZBUFFER | G_SHADE | G_CULL_BOTH | G_FOG | G_LIGHTING | G_TEXTURE_GEN |
-                          G_TEXTURE_GEN_LINEAR | G_LOD | G_SHADING_SMOOTH),
-    gsSPSetGeometryMode(G_SHADE | G_SHADING_SMOOTH),
-    gsDPSetOtherMode(G_AD_DISABLE | G_CD_MAGICSQ | G_CK_NONE | G_TC_FILT | G_TF_BILERP | G_TT_NONE | G_TL_TILE |
-                         G_TD_CLAMP | G_TP_PERSP | G_CYC_1CYCLE | G_PM_NPRIMITIVE,
-                     G_AC_NONE | G_ZS_PIXEL | G_RM_XLU_SURF | G_RM_XLU_SURF2),
-    gsDPSetCombineMode(G_CC_BLENDPEDECALA, G_CC_BLENDPEDECALA),
-    gsSPTexture(0xFFFF, 0xFFFF, 0, G_TX_RENDERTILE, G_ON),
-    gsDPLoadTextureBlock(0x08000000, G_IM_FMT_I, G_IM_SIZ_8b, 16, 64, 0, G_TX_NOMIRROR | G_TX_WRAP,
-                         G_TX_NOMIRROR | G_TX_CLAMP, 4, 6, G_TX_NOLOD, G_TX_NOLOD),
-    gsSPDisplayList(0x09000000),
-    gsSPVertex(sTransCircleVtx, 32, 0),
-    gsSP2Triangles(0, 1, 2, 0, 1, 3, 4, 0),
-    gsSP2Triangles(3, 5, 6, 0, 5, 7, 8, 0),
-    gsSP2Triangles(7, 9, 10, 0, 9, 11, 12, 0),
-    gsSP2Triangles(11, 13, 14, 0, 13, 15, 16, 0),
-    gsSP2Triangles(15, 17, 18, 0, 17, 19, 20, 0),
-    gsSP2Triangles(19, 21, 22, 0, 21, 23, 24, 0),
-    gsSP2Triangles(23, 25, 26, 0, 25, 27, 28, 0),
-    gsSP1Triangle(27, 29, 30, 0),
-    gsSPVertex(&sTransCircleVtx[31], 3, 0),
-    gsSP1Triangle(0, 1, 2, 0),
-    gsSPEndDisplayList(),
-};
+#include "assets/code/fbdemo_circle/code.c"
 
 void TransitionCircle_Start(void* thisx) {
     TransitionCircle* this = (TransitionCircle*)thisx;
diff --git a/src/code/z_fbdemo_triforce.c b/src/code/z_fbdemo_triforce.c
index d844644a2..6624a4373 100644
--- a/src/code/z_fbdemo_triforce.c
+++ b/src/code/z_fbdemo_triforce.c
@@ -1,8 +1,10 @@
 #include "transition_triforce.h"
 
 #include "printf.h"
+#include "z64math.h"
+#include "z64transition_instances.h"
 
-#include "assets/code/fbdemo_triforce/z_fbdemo_triforce.c"
+#include "assets/code/fbdemo_triforce/code.c"
 
 void TransitionTriforce_Start(void* thisx) {
     TransitionTriforce* this = (TransitionTriforce*)thisx;
diff --git a/src/code/z_fbdemo_wipe1.c b/src/code/z_fbdemo_wipe1.c
index d876b0b73..25b45e7e2 100644
--- a/src/code/z_fbdemo_wipe1.c
+++ b/src/code/z_fbdemo_wipe1.c
@@ -9,35 +9,7 @@ typedef enum TransitionWipeDirection {
     /* 1 */ TRANS_WIPE_DIR_OUT
 } TransitionWipeDirection;
 
-#include "assets/code/fbdemo_wipe1/z_fbdemo_wipe1.c"
-
-Gfx sTransWipeDL[] = {
-    gsDPPipeSync(),
-    gsSPClearGeometryMode(G_ZBUFFER | G_SHADE | G_CULL_BOTH | G_FOG | G_LIGHTING | G_TEXTURE_GEN |
-                          G_TEXTURE_GEN_LINEAR | G_LOD | G_SHADING_SMOOTH),
-    gsSPSetGeometryMode(G_ZBUFFER | G_SHADE | G_SHADING_SMOOTH),
-    gsDPSetOtherMode(G_AD_DISABLE | G_CD_MAGICSQ | G_CK_NONE | G_TC_FILT | G_TF_BILERP | G_TT_NONE | G_TL_TILE |
-                         G_TD_CLAMP | G_TP_PERSP | G_CYC_2CYCLE | G_PM_1PRIMITIVE,
-                     G_AC_NONE | G_ZS_PRIM | G_RM_PASS | G_RM_AA_ZB_TEX_EDGE2),
-    gsDPSetCombineLERP(TEXEL1, TEXEL0, PRIM_LOD_FRAC, TEXEL0, TEXEL1, TEXEL0, PRIM_LOD_FRAC, TEXEL0, COMBINED, 0,
-                       PRIMITIVE, 0, COMBINED, 0, PRIMITIVE, 0),
-    gsDPSetPrimDepth(0, 0),
-    gsDPLoadTextureBlock_4b(sTransWipeTex, G_IM_FMT_I, 64, 64, 0, G_TX_NOMIRROR | G_TX_WRAP, G_TX_MIRROR | G_TX_WRAP, 6,
-                            6, 11, G_TX_NOLOD),
-    gsDPLoadMultiBlock_4b(sTransWipeTex, 0x0100, 1, G_IM_FMT_I, 64, 64, 0, G_TX_NOMIRROR | G_TX_WRAP,
-                          G_TX_MIRROR | G_TX_WRAP, 6, 6, 11, 1),
-    gsDPSetTextureLUT(G_TT_NONE),
-    gsSPTexture(0xFFFF, 0xFFFF, 0, G_TX_RENDERTILE, G_ON),
-    gsSPDisplayList(0x08000000),
-    gsSPVertex(sTransWipeVtx, 25, 0),
-    gsSP2Triangles(0, 1, 2, 0, 1, 3, 4, 0),
-    gsSP2Triangles(5, 6, 7, 0, 6, 8, 9, 0),
-    gsSP2Triangles(8, 10, 11, 0, 10, 12, 13, 0),
-    gsSP2Triangles(12, 14, 15, 0, 14, 16, 17, 0),
-    gsSP2Triangles(16, 18, 19, 0, 18, 20, 21, 0),
-    gsSP2Triangles(20, 22, 23, 0, 22, 0, 24, 0),
-    gsSPEndDisplayList(),
-};
+#include "assets/code/fbdemo_wipe1/code.c"
 
 // unused.
 Gfx sTransWipeSyncDL[] = {
diff --git a/src/code/z_kankyo.c b/src/code/z_kankyo.c
index 88dcd5e59..c78fc5de8 100644
--- a/src/code/z_kankyo.c
+++ b/src/code/z_kankyo.c
@@ -1,5 +1,5 @@
-#pragma increment_block_number "gc-eu:192 gc-eu-mq:192 gc-jp:192 gc-jp-ce:192 gc-jp-mq:192 gc-us:192 gc-us-mq:192" \
-                               "ique-cn:192 ntsc-1.0:192 ntsc-1.1:192 ntsc-1.2:192 pal-1.0:192 pal-1.1:192"
+#pragma increment_block_number "gc-eu:192 gc-eu-mq:192 gc-jp:128 gc-jp-ce:128 gc-jp-mq:128 gc-us:128 gc-us-mq:128" \
+                               "ique-cn:128 ntsc-1.0:192 ntsc-1.1:192 ntsc-1.2:192 pal-1.0:192 pal-1.1:192"
 
 #include "libc64/qrand.h"
 #include "libu64/gfxprint.h"
@@ -238,8 +238,8 @@ s16 sLightningFlashAlpha;
 s16 sSunDepthTestX;
 s16 sSunDepthTestY;
 
-#pragma increment_block_number "gc-eu:192 gc-eu-mq:192 gc-jp:192 gc-jp-ce:192 gc-jp-mq:192 gc-us:192 gc-us-mq:192" \
-                               "ique-cn:160 ntsc-1.0:192 ntsc-1.1:192 ntsc-1.2:192 pal-1.0:192 pal-1.1:192"
+#pragma increment_block_number "gc-eu:192 gc-eu-mq:192 gc-jp:128 gc-jp-ce:128 gc-jp-mq:128 gc-us:128 gc-us-mq:128" \
+                               "ique-cn:128 ntsc-1.0:128 ntsc-1.1:128 ntsc-1.2:128 pal-1.0:192 pal-1.1:192"
 
 LightNode* sNGameOverLightNode;
 LightInfo sNGameOverLightInfo;
diff --git a/src/code/z_message.c b/src/code/z_message.c
index 046d3ff88..1d6fee156 100644
--- a/src/code/z_message.c
+++ b/src/code/z_message.c
@@ -27,7 +27,7 @@
 #include "assets/textures/parameter_static/parameter_static.h"
 
 #pragma increment_block_number "gc-eu:0 gc-eu-mq:0 gc-jp:0 gc-jp-ce:0 gc-jp-mq:0 gc-us:0 gc-us-mq:0 ntsc-1.0:32" \
-                               "ntsc-1.1:32 ntsc-1.2:32 pal-1.0:128 pal-1.1:128"
+                               "ntsc-1.1:32 ntsc-1.2:32 pal-1.0:0 pal-1.1:0"
 
 #if !PLATFORM_IQUE
 #define MSG_BUF_DECODED (msgCtx->msgBufDecoded)