From b2d5551ece8e5a2004f76414db01cf4f1a3d6a3d Mon Sep 17 00:00:00 2001 From: James Dansereau <106631954+jkdansereau@users.noreply.github.com> Date: Sun, 13 Sep 2026 23:17:29 -0400 Subject: [PATCH] Drop-in ROM: first-run sidecar conversion, no Python (issue #6) (#84) * Drop-in ROM: first-run sidecar conversion, no Python (issue #6, Part A + interim) The release bundle now turns a dropped-in NTSC-U ROM into a playable game with zero tooling: when data/pcmodels-* / data/pccg-* are missing at boot, the engine spawns the frozen prepare-assets converter shipped in the bundle (prepare-assets/ge007-convert[.exe], PyInstaller --onefile), waits for it, and re-checks before continuing (port/src/romconvert.c, called from romdataInit after ROM validation). Failure aborts boot with a message instead of the later D179-class crash. - prepare-assets.py: runs in-process via runpy when frozen (no user Python to exec); NTSC-U-only gate with a clear message for PAL/JP ROMs; d88's sibling import needs an explicit sys.path entry (run_path does not set one). Verified: frozen exe output byte-identical to the unfrozen path and to the shipped sidecars. - CI (linux + windows): PyInstaller freeze step before bundling (ensurepip / --break-system-packages for MSYS2's pip-less python). - bundle scripts: copy ge007-convert into prepare-assets/ (hard-fail if not frozen), size guard 60 -> 120 MB. - Release notes: US-only drop-in instructions, no Python required. - D258: PAL/JP filelist naming defects documented; multi-region deferred to the Part B follow-up. * CI: install pip via pacman on Windows (MSYS2 ensurepip is disabled); regen findings index for D258 --- .github/release-notes.md | 19 +- .github/workflows/ci.yml | 37 +++- .gitignore | 4 + docs/dev/findings-index.csv | 1 + docs/dev/findings.md | 1 + port/include/romconvert.h | 24 +++ port/src/romconvert.c | 203 ++++++++++++++++++ port/src/romdata.c | 16 +- tools_pc/bundle-linux.sh | 18 +- tools_pc/bundle-win.sh | 17 +- .../dist/prepare-assets/prepare-assets.py | 59 ++++- 11 files changed, 370 insertions(+), 29 deletions(-) create mode 100644 port/include/romconvert.h create mode 100644 port/src/romconvert.c diff --git a/.github/release-notes.md b/.github/release-notes.md index 718a07e9..8997542d 100644 --- a/.github/release-notes.md +++ b/.github/release-notes.md @@ -93,21 +93,20 @@ needs to be installed first. ### Running it You supply your own **GoldenEye 007 N64 ROM** (`.z64`, big-endian) that you -legally own — NTSC-U (US), PAL (EU) or NTSC-J (JP); the asset step detects the -region from the SHA-1. US is the best-tested. +legally own — the **NTSC-U (US)** release is what this build supports (PAL / JP +are on the roadmap). No Python, no toolchain, nothing to install. 1. Unpack the archive. 2. Make a `data/` folder next to the executable and put the ROM in it, named - per region (`ge007.ntsc-final.z64` / `ge007.pal-final.z64` / - `ge007.jpn-final.z64`). -3. Run the one-time asset step (needs Python 3.8+): - `python3 prepare-assets/prepare-assets.py` — it reads your ROM and writes - the two `data/pc*-/` folders the engine needs. Standard library - only; a few seconds. -4. Run the executable **from that folder**. + `ge007.ntsc-final.z64`. +3. Run the executable **from that folder**. The first run takes a few extra + seconds: it detects your ROM, generates the two derived asset folders + (`data/pcmodels-ntsc-final/`, `data/pccg-ntsc-final/`) once, and saves them + for every future run. (The generator is `prepare-assets/ge007-convert` + inside the bundle; you can also run it manually — it prints what it's doing.) **Steam Deck:** sideload the unpacked folder (USB or a file manager), do steps -2–4, then add the executable to Games → *Add Game* as a non-Steam game. +2–3, then add the executable to Games → *Add Game* as a non-Steam game. **In-game settings on the Deck:** the options overlay is fully gamepad-driven — it opens with **Select**, the D-pad or left stick (up/down) moves between diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04ec9b07..772b1883 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,7 +181,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential gcc-14 g++-14 cmake python3 \ - libsdl2-dev zlib1g-dev libgl1-mesa-dev ccache patchelf + python3-pip libsdl2-dev zlib1g-dev libgl1-mesa-dev ccache patchelf - name: Restore ccache uses: actions/cache@v4 @@ -195,6 +195,22 @@ jobs: - name: Regenerate ROM-asset symbol file (US) run: python3 scripts/gen_romassets.py u + # Drop-in ROM support: freeze the asset-prep tool into a single + # ge007-convert binary the engine spawns on first run when the + # pcmodels/pccg sidecars are missing (port/src/romconvert.c). Pure- + # stdlib scripts, so PyInstaller --onefile is clean; lands in + # tools_pc/dist/prepare-assets/ where bundle-linux.sh picks it up. + - name: Freeze asset converter (PyInstaller) + run: | + python3 -c "import PyInstaller" 2>/dev/null \ + || python3 -m pip install --quiet --break-system-packages pyinstaller + python3 -m PyInstaller --onefile --noconfirm --clean \ + --name ge007-convert \ + --distpath tools_pc/dist/prepare-assets \ + --workpath build-pyinst --specpath build-pyinst \ + tools_pc/dist/prepare-assets/prepare-assets.py + ls -la tools_pc/dist/prepare-assets/ge007-convert + - name: Configure (ntsc-final) run: cmake -S . -B build-linux -DROMID=ntsc-final -DCMAKE_C_COMPILER=gcc-14 -DCMAKE_CXX_COMPILER=g++-14 @@ -297,6 +313,25 @@ jobs: - name: Regenerate ROM-asset symbol file (US) run: python scripts/gen_romassets.py u + # Drop-in ROM support: freeze the asset-prep tool into a single + # ge007-convert.exe the engine spawns on first run when the pcmodels/ + # pccg sidecars are missing (port/src/romconvert.c). Lands in + # tools_pc/dist/prepare-assets/ where bundle-win.sh picks it up. + - name: Freeze asset converter (PyInstaller) + run: | + # MSYS2's mingw python ships without pip and has ensurepip + # disabled — install pip from pacman, then PyInstaller. + python -c "import PyInstaller" 2>/dev/null || { + pacman -S --noconfirm mingw-w64-x86_64-python-pip + python -m pip install --quiet --break-system-packages pyinstaller + } + python -m PyInstaller --onefile --noconfirm --clean \ + --name ge007-convert \ + --distpath tools_pc/dist/prepare-assets \ + --workpath build-pyinst --specpath build-pyinst \ + tools_pc/dist/prepare-assets/prepare-assets.py + ls -la tools_pc/dist/prepare-assets/ge007-convert.exe + - name: Build (ntsc-final) run: ./build-pc.sh ntsc-final diff --git a/.gitignore b/.gitignore index 3b832aa7..93e3676a 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,7 @@ nohup.out gdb.txt gdb-hang.txt + +# frozen asset converter (built by CI via PyInstaller; local freezes land here) +tools_pc/dist/prepare-assets/ge007-convert* +build-pyinst/ diff --git a/docs/dev/findings-index.csv b/docs/dev/findings-index.csv index 461b7bc3..09884f1b 100644 --- a/docs/dev/findings-index.csv +++ b/docs/dev/findings-index.csv @@ -139,3 +139,4 @@ D254,FIXED,Linux crash handler destroyed its own logs:,FIXED — field-verified. D255,OPEN,"Facility firefight SIGSEGV on Steam Deck (v0.2.0-pre CI bundle, D253 fix confirmed working —","OPEN — case identified (NULL model on an active OBJ record; destroy-path ordering), reproducible on Deck in combat (Bunker + Frigate); fix pending rule-#2 sign-off." D256,FIXED,Saves (and config) silently failed to persist on Linux whenever the game was not launched from the directory containing data/ —,FIXED (path half) — save-flow detail still owed from the user. D257,FIXED,"New Game.AllUnlocked option (F10, default ON for this cut):",FIXED — verified by user playtest; PR open. +D258,OPEN,PAL/JP asset conversion is broken at the source-data level —,"OPEN — repair path verified, not applied (out of v0.2.0 scope)." diff --git a/docs/dev/findings.md b/docs/dev/findings.md index 5c3728b3..948403b3 100644 --- a/docs/dev/findings.md +++ b/docs/dev/findings.md @@ -535,6 +535,7 @@ covers D24–D69; the log continues in §H (D32 procedure, D70–D121). | D255 | **Facility firefight SIGSEGV on Steam Deck (v0.2.0-pre CI bundle, D253 fix confirmed working — user got past the opening crouch tile).** Crash during gunplay; abort-unwind backtrace points at `sub_GAME_7F09BAC4` (`vtxstore.c:158`) ← `sub_GAME_7F09BBBC` (`:220`) ← `lvlManageMpGame` (`lv.c:1302`) ← `bossMainloop` (`boss.c:567`). Disassembling the exact CI binary at the fault PC (`0x200db135`) gives `mov 0x14(%rax),%edi; mov 0x10(%rdi),%rax`: the function walks the active PropRecord list keeping `type == 1` — which is **PROP_TYPE_OBJ, not CHR** (the decompiler cast to the union's first member and misnamed the field `ChrRecord.chrflags`; it is actually `ObjectRecord.model` at `0x14`) — then dereferences `model->obj`. Two port-relevant wrinkles: (a) the load is **32-bit zero-extended** because the decomp types the field through the `CHRFLAG` enum, so a model pointer ≥ 4 GB would truncate to garbage; (b) the loop skips records whose `model->obj` is NULL but *dereferences* `model` unconditionally, so an active object with a NULL/freed `model` faults — plausible in a firefight where destructibles (crates etc.) die. Not reproduced on WSL after equivalent play. **Register capture (D254 format, 2026-09-13):** two more Deck crashes — Bunker mid-firefight, then Frigate with ~3 NPCs alive; log: `SIGNAL: 11 / FAULT ADDR: 0x10 / Rax=0x701e5f54 Rdi=(nil) / PC 0x200db135`. `Rdi` = the record's model value = **exactly NULL** → case (b) confirmed: an active PROP_TYPE_OBJ record with a NULL `model`, dereferenced by the unconditional `mov 0x10(%rdi),%rax`; truncation (a) and stale/garbage pointers are ruled out. Faulting PC matches the recorded disassembly (`sub_GAME_7F09BAC4` = `vtxstore_fix_refs`). Grep-verified: nothing in game code ever writes NULL into `ObjectRecord.model` — `clear_model_obj()` nulls `model->obj` only, and `objInit`/spawn paths only activate a record once model ≠ NULL — so the NULL comes from either a fresh/uninitialized record slot being activated or a destroy-path ordering race (model freed/nulled while the record is still active this tick). **Fix constraint:** any guard lands in game code (`vtxstore.c`) → needs rule-#2 sign-off; `port/` cannot intercept this dereference. Next: (1) decision on whether a NULL-model skip in `vtxstore_fix_refs` qualifies as an ABI-class exception (semantics-preserving defensive guard, PD analogue check); (2) if yes, add it plus a `#ifdef PORT` trace at the deactivation/model-free sites to name the path that leaves the record model-less. | OPEN — case identified (NULL model on an active OBJ record; destroy-path ordering), reproducible on Deck in combat (Bunker + Frigate); fix pending rule-#2 sign-off. | | D256 | **Saves (and config) silently failed to persist on Linux whenever the game was not launched from the directory containing `data/` — reported as "saves do not work at all" on Steam Deck.** `romdataInit` finds the ROM in `$S/` (CWD `data/`) *or* `$E/` (exe dir) *or* `./`, but `sysResolvePath`'s POSIX `$S` branch (used by `ge007.eep` and `ge007.ini`) only ever produced a CWD-relative path — its "fallback" was `./data/`, identical to the primary. Launch from any other CWD (Steam non-steam-game shortcut, terminal in `~`): ROM loads via the exe-dir candidate, game runs, but every EEPROM/config write hits `fopen(..., "wb")` on a nonexistent `./data/` and fails with a stderr-only "cannot write" — saves never persist, and **every launch also starts with default settings** (F10 changes don't stick), which presents as "draw distance/LOD is low" on levels like Dam where the user had raised `Video.DrawDistance` on another machine. The Windows branch already did CWD-else-exedir; the POSIX branch now mirrors it (`readlink("/proc/self/exe")`) and creates the directory best-effort so first-run writes don't fail. **Verified on WSL:** with exe+`data/` in `/tmp/fb2` and CWD=`/`, config/eeprom/ROM all resolve into `/tmp/fb2/data/` and a 25 s run completes clean; the pre-fix binary in the same layout fails to find the ROM at all. **Caveat:** if the user's Deck launch *was* from the bundle dir (CWD had `data/`), this fix does not explain their save failure — need the save-flow detail (does the memory-card screen appear? slot confirm? "saved" message?) before closing the report. Side note found while testing: a `data/` without the generated sidecars (`pccg-*/`, `pcmodels-*/` from `tools_pc/d69_emit.py`/`d43_emit.py`) starts fine but faults in `load_bg_file` (`bg.c:865`) on level load — the reserve functions warn "stage loads will fault" and return 0; expected degraded mode, not a new bug. | FIXED (path half) — save-flow detail still owed from the user. | | D257 | **New `Game.AllUnlocked` option (F10, default ON for this cut): skip N64 progression entirely — all 21 solo levels selectable at every difficulty, 007 mode available, and the cheat menu fully populated. Port-layer only; no game-code edits.** Three mechanisms, all in `port/`: (1) `main.c` seeds the two live `LEFTOVERDEBUG` RAM flags after `configLoad()` — `debug_enable_all_levels_flag` (all levels) and `debug_007_unlock_flag` (007 mode; both in `src/game/debugmenu_handler.c`, same class as the existing `GE_UNLOCK_ALL` getenv hook in the getter); (2) the eep shim (`libultra.c geEepromPatchAllCheats`) patches the save block at read time — special-casing `fileValidateSaves`'s exact pattern (block 4, 5×96 bytes) — setting all 20 progression-gated cheat-unlock bits per slot (`unlocked_cheats_1/2/3`; the cheat menu row on mode-select only renders when ≥1 is set, `front.c interface_menu06_modesel`) **and** filling every *empty* stage/difficulty completion-time field with the max time 0x3FF (existing player times preserved); both are then CRC'd per slot with the game's own `fileGenerateCRC` or `fileValidateSaves` would wipe the slots. (3) `optionsoverlay.c` row so users can toggle OFF for faithful progression. **Why the completion-time half is required (found by user repro):** when ANY single-player cheat is active, `g_AppendCheatSinglePlayer` makes mission-select (`get_highest_unlocked_difficulty_for_level`, front.c) accept only levels with status `STAGESTATUS_COMPLETED (3)`, not merely `UNLOCKED (1)` — AllUnlocked's debug-flag path returns 1, so enabling a cheat locked every level ("could only do Silo"). Reading the save as a fully-completed campaign satisfies both branches. Time bit-math mirrors `fileGet/SetSaveStageDifficultyTime` verbatim (10-bit fields, difficulties 0–2; 007 times are virtual). **Pitfall hit during implementation:** the first mirror of `save_data` used `times[80]` instead of `(SP_LEVEL_MAX-1)*4 = 76`, making `sizeof` 100≠96 so the block-read size check silently never matched — now enforced by a compile-time `ge_save_slot_size_check`. **Semantics:** unlocked cheats/completions persist to `ge007.eep` on any subsequent save write (self-consistent; mirrors N64 where completing a level permanently unlocks its cheat). No *active* cheats are enabled (invincibility/all-guns stay OFF); weapons remain per-mission pickups. **Verified:** fresh eep — all 5 slots validate `stored==calc` under `GE_SAVELOG=1`; user playtest: all levels selectable at each difficulty, 007 mode present, cheat menu populated, cheats toggle and work (time-modifying cheats untested by user), mission select intact with all cheats ON. | FIXED — verified by user playtest; PR open. | +| D258 | **PAL/JP asset conversion is broken at the source-data level — `scripts/filelist.e.csv` and `filelist.j.csv` carry naming defects, so `gen_romassets.py e/j` (and the d43/d69/d88 emitters' `find_row`) cannot resolve PAL/JP builds. v0.2.0's drop-in ROM feature therefore ships NTSC-U only.** Discovered while scoping issue #6 (drop-in ROM). **PAL:** 31 of the 512 model rows in `filelist.e.csv` have truncated/renamed basenames (an extraction artifact — e.g. `Pbridge_console` ×6 where the resource table says `Pbridge_console1aZ…3bZ`, `Pchrgrenadelauncher` for `PchrgrenadelaunchZ`). **JP:** one font row is mislabeled — `fontBankGothic_fontchartable.bin` @3044276 is named `fontZurichBold_fontchartable.bin` (a duplicate label). Both were proven without a PAL/JP ROM: (a) greedy two-pointer sequence alignment of the 512-row `file_resource_table.inc.c` model order against each CSV's model rows gives an exact 1:1 match (481 by-name + 31 positional for PAL), offsets strictly non-decreasing, and zero size anomalies vs US (all within [0.5×, 2×]); (b) JP font row proven by the contiguity invariant — Gothic kerning @3043600+676 = 3044276 exactly hits the mislabeled row, whose size (8716) equals the US Gothic chartable, while Zurich's own chain is contiguous one row later. **Consequence:** no PAL or JP PC exe has ever been buildable from this tree (latent — only NTSC was ever exercised); the CSV `file` column is a label only (emitters look up by basename, never read the path), so the repair is a pure data fix: rename the 31 PAL model rows + 1 JP font row to the exact table symbols. **Scope decision (v0.2.0):** ship drop-in ROM support for NTSC-U only — `prepare-assets.py`/the frozen `ge007-convert` gate non-US ROMs with a clear message after SHA-1 region detection; the CSV repair + multi-region matrix + per-region exes are deferred to the Part B follow-up. | OPEN — repair path verified, not applied (out of v0.2.0 scope). | Phase 2 replaced the Phase-1 demo loop with the real `mainproc()` on real OS threads, compiled GE's real `src/sched.c`, and brought in PD's fast3d software diff --git a/port/include/romconvert.h b/port/include/romconvert.h new file mode 100644 index 00000000..ee755658 --- /dev/null +++ b/port/include/romconvert.h @@ -0,0 +1,24 @@ +#ifndef PORT_ROMCONVERT_H +#define PORT_ROMCONVERT_H + +/* + * First-run sidecar conversion — see port/src/romconvert.c. Called from + * romdataInit() once a valid ROM image is in hand; spawns the bundled frozen + * converter (ge007-convert) if the pcmodels/pccg sidecars are missing. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* romImg: mapped/read ROM image (country byte at 0x3E selects the region); + * romRelPath: the "$S/..." / "$E/..." / "./..." path the ROM was opened from. + * Returns 1 when both sidecars are present, -1 when they could not be + * produced (caller should abort boot). */ +int romConvertEnsureSidecars(const unsigned char *romImg, const char *romRelPath); + +#ifdef __cplusplus +} +#endif + +#endif /* PORT_ROMCONVERT_H */ diff --git a/port/src/romconvert.c b/port/src/romconvert.c new file mode 100644 index 00000000..72b7fd17 --- /dev/null +++ b/port/src/romconvert.c @@ -0,0 +1,203 @@ +/* + * romconvert.c — first-run sidecar conversion ("drop in your ROM and play"). + * + * The engine needs two ROM-derived sidecar trees that are NOT shipped with a + * release (they are copyrighted game data derived from the user's own ROM): + * + * $S/pcmodels-/ (pcmodels.bin + manifest.csv) — port/src/pcmodels.c + * $S/pccg-/ (pccg.bin + manifest.csv) — port/src/pccg.c + * + * The release bundle ships a frozen copy of the asset-prep tool + * (prepare-assets.py, PyInstaller --onefile) as + * /prepare-assets/ge007-convert[.exe]. When the sidecars are missing + * at boot, romdataInit() calls romConvertEnsureSidecars(), which spawns that + * binary with `--rom --out `, waits for it to finish, and + * re-checks. The conversion is deterministic and takes a few seconds; after + * the first run the sidecars exist and this path is a no-op (two fsExists). + * + * Returns 1 when both sidecars are present (already or after converting), + * -1 when they could not be produced (caller should abort boot with a + * message rather than crash later in modelPromoteNodeOffsetsToPointers, D179). + */ + +#include +#include + +/* D38: / resolve to the decomp's N64 stubs via the include + * path; declare the host functions this file uses. */ +extern void *malloc(unsigned long long size); +extern int snprintf(char *str, size_t maxsize, const char *format, ...); + +#include "platform.h" + +#if defined(PLATFORM_WINDOWS) + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + #include +#else + #include + #include +#endif + +#include "system.h" +#include "fs.h" + +/* Same country→region map as pcmodels.c / pccg.c (finding: plan Q3). */ +static const char *rcRegionForCountry(unsigned char country) +{ + switch (country) { + case 'E': return "ntsc-final"; + case 'P': return "pal-final"; + case 'J': return "jpn-final"; + default: return NULL; + } +} + +/* sysResolvePath() returns a static buffer — copy before the next call. */ +static int rcExistsResolved(const char *rel) +{ + const char *p = sysResolvePath(rel); + return fsExists(p); +} + +static int rcSidecarsPresent(const char *region) +{ + char rel[256]; + snprintf(rel, sizeof(rel), "$S/pcmodels-%s/pcmodels.bin", region); + if (!rcExistsResolved(rel)) + return 0; + snprintf(rel, sizeof(rel), "$S/pccg-%s/pccg.bin", region); + return rcExistsResolved(rel); +} + +#if defined(PLATFORM_WINDOWS) +/* Spawn `exePath --rom "" --out ""` and wait for it. + * Returns the exit code, or -1 if it could not be started. */ +static int rcRunConverter(const char *exePath, const char *rom, const char *out) +{ + char cmd[4096]; + int n = snprintf(cmd, sizeof(cmd), "\"%s\" --rom \"%s\" --out \"%s\"", + exePath, rom, out); + if (n < 0 || (size_t)n >= sizeof(cmd)) + return -1; /* paths too long to build a command line */ + + int wlen = MultiByteToWideChar(CP_UTF8, 0, cmd, -1, NULL, 0); + if (wlen <= 0) + return -1; + WCHAR *wcmd = (WCHAR *)malloc((size_t)wlen * sizeof(WCHAR)); + if (!wcmd) + return -1; + MultiByteToWideChar(CP_UTF8, 0, cmd, -1, wcmd, wlen); + + STARTUPINFOW si; + PROCESS_INFORMATION pi; + int rc = -1; + memset(&si, 0, sizeof(si)); + si.cb = sizeof(si); + memset(&pi, 0, sizeof(pi)); + if (CreateProcessW(NULL, wcmd, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD code = 0; + GetExitCodeProcess(pi.hProcess, &code); + rc = (int)code; + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + } + free(wcmd); + return rc; +} +#else +static int rcRunConverter(const char *exePath, const char *rom, const char *out) +{ + pid_t pid = fork(); + if (pid < 0) + return -1; + if (pid == 0) { + /* Child: stdio is inherited (console apps see the converter's + * progress lines; GUI builds simply discard them). */ + execv(exePath, (char *const[]){ exePath, "--rom", (char *)rom, + "--out", (char *)out, NULL }); + _exit(127); /* execv failed */ + } + int status = 0; + while (waitpid(pid, &status, 0) < 0) + ; /* EINTR */ + if (WIFEXITED(status)) + return WEXITSTATUS(status); + return -1; +} +#endif + +int romConvertEnsureSidecars(const unsigned char *romImg, const char *romRelPath) +{ + const char *region = rcRegionForCountry(romImg[0x3E]); + if (!region) + return 1; /* unknown country — romHeaderValid() rejects it anyway */ + + if (rcSidecarsPresent(region)) + return 1; + + /* Resolve the ROM path we were given ("$S/...", "$E/..." or "./...") to + * an absolute form for the child's argv. */ + char romPath[1024]; + snprintf(romPath, sizeof(romPath), "%s", sysResolvePath(romRelPath)); + + const char *exedir = sysGetExeDir(); + char outDir[1024]; + snprintf(outDir, sizeof(outDir), "%s", exedir); + + /* Converter candidates: bundled under /prepare-assets/ (the + * release layout), then the CWD-relative form (running from a source + * tree or an unpacked bundle in the CWD). */ +#if defined(PLATFORM_WINDOWS) + static const char *candRel[4] = { + "$E/prepare-assets/ge007-convert.exe", + "./prepare-assets/ge007-convert.exe", + "$E/ge007-convert.exe", + "./ge007-convert.exe", + }; +#else + static const char *candRel[4] = { + "$E/prepare-assets/ge007-convert", + "./prepare-assets/ge007-convert", + "$E/ge007-convert", + "./ge007-convert", + }; +#endif + char convPath[1024] = ""; + for (int i = 0; i < 4; i++) { + if (rcExistsResolved(candRel[i])) { + /* sysResolvePath() returns a static buffer — snapshot it. */ + snprintf(convPath, sizeof(convPath), "%s", sysResolvePath(candRel[i])); + break; + } + } + + if (!convPath[0]) { + sysLogPrintf(LOG_ERROR, + "romconvert: sidecars for '%s' are missing and no ge007-convert " + "binary was found (looked in /prepare-assets/). Put your " + "ROM at data/ge007.%s.z64 and re-run from the bundle folder, or " + "run prepare-assets manually.", region, region); + return -1; + } + + sysLogPrintf(LOG_INFO, + "romconvert: sidecars missing — running %s (one-time, a few seconds)…", + convPath); + int code = rcRunConverter(convPath, romPath, outDir); + if (code != 0) + sysLogPrintf(LOG_ERROR, + "romconvert: ge007-convert exited with code %d — see its output " + "above (a non-retail or byte-swapped ROM will fail here)", code); + + if (!rcSidecarsPresent(region)) { + sysLogPrintf(LOG_ERROR, + "romconvert: data/pcmodels-%s/ and data/pccg-%s/ are still " + "missing — cannot continue", region, region); + return -1; + } + sysLogPrintf(LOG_INFO, "romconvert: sidecars generated — continuing boot"); + return 1; +} diff --git a/port/src/romdata.c b/port/src/romdata.c index 010607d1..d5012af6 100644 --- a/port/src/romdata.c +++ b/port/src/romdata.c @@ -44,6 +44,7 @@ extern int snprintf(char *str, size_t maxsize, const char *format, ...); #include "romdata.h" #include "pcmodels.h" #include "pccg.h" +#include "romconvert.h" /* D50: struct font/fontchar layouts for the font-segment re-layout. Order * matters (see pcmodels.c): ultra64.h must finish before bondtypes.h. */ @@ -315,11 +316,11 @@ int romdataInit(void) static const char *locs[3] = { "$S/", "$E/", "./" }; int li; FSFile *f = NULL; + char romRelPath[1024] = ""; /* locs[li] + tok of the ROM we opened */ for (li = 0; li < 3 && !f; li++) { - char pathbuf[1024]; - snprintf(pathbuf, sizeof(pathbuf), "%s%s", locs[li], tok); - f = fsOpen(sysResolvePath(pathbuf), "rb"); + snprintf(romRelPath, sizeof(romRelPath), "%s%s", locs[li], tok); + f = fsOpen(sysResolvePath(romRelPath), "rb"); } if (!f) continue; @@ -340,6 +341,15 @@ int romdataInit(void) continue; } + /* Drop-in ROM support: if the ROM-derived sidecars are missing, run + * the bundled frozen converter (ge007-convert) once to generate them. + * Without them the game crashes later in + * modelPromoteNodeOffsetsToPointers (D179), so abort boot cleanly. */ + if (!romConvertEnsureSidecars(img, romRelPath)) { + free(img); + return -1; + } + /* D50: reserve room for the PC model sidecar image right after the * ROM ([CART_BASE+romSize, ...)); it is loaded below and patched into * file_resource_table after obInit. */ diff --git a/tools_pc/bundle-linux.sh b/tools_pc/bundle-linux.sh index 1856b222..342c8d73 100755 --- a/tools_pc/bundle-linux.sh +++ b/tools_pc/bundle-linux.sh @@ -108,15 +108,29 @@ nmh="$(find "$PREP/vendor/assets" -iname 'modelfileheader.inc.c' | wc -l)" echo " + prepare-assets/ (emit scripts + $nmh model headers)" [ "$nmh" -gt 400 ] || { echo "error: prepare-assets vendored only $nmh model headers — expected ~512" >&2; exit 1; } +# Frozen first-run converter (drop-in ROM support): the engine spawns this +# when data/pcmodels-* / data/pccg-* are missing (port/src/romconvert.c). +# CI builds it with PyInstaller before bundling; a local bundle needs the +# same step: python3 -m PyInstaller --onefile --name ge007-convert \ +# --distpath tools_pc/dist/prepare-assets \ +# tools_pc/dist/prepare-assets/prepare-assets.py +CV="$REPO_ROOT/tools_pc/dist/prepare-assets/ge007-convert" +[ -f "$CV" ] || { echo "error: $CV not found — freeze it with PyInstaller first (see above)" >&2; exit 1; } +cp "$CV" "$PREP/" +chmod 755 "$PREP/ge007-convert" +echo " + prepare-assets/ge007-convert ($(du -h "$CV" | cut -f1))" + # --- guard: no ROM / game data snuck in ------------------------------ if find "$OUT" -type f \( -iname '*.z64' -o -iname '*.n64' -o -iname '*.v64' \) | grep -q .; then echo "error: bundle contains a ROM image — aborting" >&2 exit 1 fi +# 120 MB: engine + SDL2 are a few MB; the frozen ge007-convert (PyInstaller +# --onefile CPython) adds ~15 MB. Anything far beyond that is game data. BYTES="$(du -sb "$OUT" | cut -f1)" -LIMIT=$((60 * 1024 * 1024)) +LIMIT=$((120 * 1024 * 1024)) if [ "$BYTES" -gt "$LIMIT" ]; then - echo "error: bundle is $BYTES bytes (> 60 MB) — likely contains game data, aborting" >&2 + echo "error: bundle is $BYTES bytes (> 120 MB) — likely contains game data, aborting" >&2 exit 1 fi diff --git a/tools_pc/bundle-win.sh b/tools_pc/bundle-win.sh index 5ef4c433..7a6efa5a 100755 --- a/tools_pc/bundle-win.sh +++ b/tools_pc/bundle-win.sh @@ -106,15 +106,28 @@ nmh="$(find "$PREP/vendor/assets" -iname 'modelfileheader.inc.c' | wc -l)" echo " + prepare-assets/ (emit scripts + $nmh model headers)" [ "$nmh" -gt 400 ] || { echo "error: prepare-assets vendored only $nmh model headers — expected ~512" >&2; exit 1; } +# Frozen first-run converter (drop-in ROM support): the engine spawns this +# when data/pcmodels-* / data/pccg-* are missing (port/src/romconvert.c). +# CI builds it with PyInstaller before bundling; a local bundle needs the +# same step: python -m PyInstaller --onefile --name ge007-convert \ +# --distpath tools_pc/dist/prepare-assets \ +# tools_pc/dist/prepare-assets/prepare-assets.py +CV="$REPO_ROOT/tools_pc/dist/prepare-assets/ge007-convert.exe" +[ -f "$CV" ] || { echo "error: $CV not found — freeze it with PyInstaller first (see above)" >&2; exit 1; } +cp "$CV" "$PREP/" +echo " + prepare-assets/ge007-convert.exe ($(du -h "$CV" | cut -f1))" + # --- guard: no ROM / game data snuck in -------------------------------- if find "$OUT" -type f \( -iname '*.z64' -o -iname '*.n64' -o -iname '*.v64' \) | grep -q .; then echo "error: bundle contains a ROM image — aborting" >&2 exit 1 fi +# 120 MB: engine + DLLs are a few MB; the frozen ge007-convert (PyInstaller +# --onefile CPython) adds ~15 MB. Anything far beyond that is game data. BYTES="$(du -sb "$OUT" | cut -f1)" -LIMIT=$((60 * 1024 * 1024)) +LIMIT=$((120 * 1024 * 1024)) if [ "$BYTES" -gt "$LIMIT" ]; then - echo "error: bundle is $BYTES bytes (> 60 MB) — likely contains game data, aborting" >&2 + echo "error: bundle is $BYTES bytes (> 120 MB) — likely contains game data, aborting" >&2 exit 1 fi diff --git a/tools_pc/dist/prepare-assets/prepare-assets.py b/tools_pc/dist/prepare-assets/prepare-assets.py index 912fc7ab..5ae176e1 100644 --- a/tools_pc/dist/prepare-assets/prepare-assets.py +++ b/tools_pc/dist/prepare-assets/prepare-assets.py @@ -25,12 +25,21 @@ contains this "prepare-assets" folder (i.e. next to ge007.x86_64.exe). import argparse import hashlib +import os +import runpy import shutil import subprocess import sys from pathlib import Path -HERE = Path(__file__).resolve().parent +# When frozen (PyInstaller --onefile, shipped as ge007-convert in the release +# bundle) __file__ is not the install location — the exe lives inside the +# prepare-assets/ folder next to vendor/, so use the executable's directory. +FROZEN = getattr(sys, "frozen", False) +if FROZEN: + HERE = Path(sys.executable).resolve().parent +else: + HERE = Path(__file__).resolve().parent VENDOR = HERE / "vendor" # SHA-1 of each supported retail ROM (big-endian .z64) -> region id the @@ -91,6 +100,34 @@ def find_rom(explicit, bundle_root): return cand[0] +def run_emit(script, argv_tail, cwd): + """Run one emit script with CWD=cwd. Unfrozen: a subprocess (as before). + Frozen: in-process via runpy — there is no user Python to exec. The + scripts are pure-stdlib; d88 also imports its sibling d88_propdefs, so + this directory must be on sys.path for the duration (run_path does NOT + do that — only a real `python script.py` launch does).""" + if not FROZEN: + r = subprocess.run([sys.executable, str(HERE / script), *argv_tail], cwd=cwd) + return r.returncode + # Frozen sys.executable carries Windows backslashes on MSYS2; the + # import machinery wants a plain directory string. + here = str(HERE).replace("\\", "/") + old_argv, old_cwd = sys.argv, os.getcwd() + sys.argv = [here + "/" + script, *argv_tail] + sys.path.insert(0, here) + os.chdir(cwd) + rc = 0 + try: + runpy.run_path(here + "/" + script, run_name="__main__") + except SystemExit as e: + rc = e.code if isinstance(e.code, int) else (1 if e.code else 0) + finally: + sys.argv = old_argv + sys.path.pop(0) + os.chdir(old_cwd) + return rc + + def main(): ap = argparse.ArgumentParser(description="Generate the GoldenEye PC port's ROM-derived asset dirs.") ap.add_argument("--rom", help="path to your .z64 ROM (default: auto-detect near the bundle)") @@ -114,8 +151,12 @@ def main(): region = ROM_SHA1.get(digest) if region is None: die(f"ROM SHA-1 {digest} is not a recognised retail GoldenEye 007 image.\n" - " Supported: NTSC-U / PAL / NTSC-J, big-endian .z64.\n" + " Supported: NTSC-U (US), big-endian .z64.\n" " A byte-swapped (.n64/.v64) or overdumped ROM will not work -- convert to .z64 first.") + if region != "ntsc-final": + die(f"this release only supports the NTSC-U (US) ROM; your ROM is the " + f"{region} version.\n" + " PAL / JP support is on the roadmap but not in this build.") print(f"prepare-assets: SHA-1 {digest} -> region '{region}'") # The emit scripts assume CWD is a tree with scripts/ assets/ data/ . @@ -133,11 +174,10 @@ def main(): shutil.rmtree(p) for script, extra, produces in EMIT_STEPS: - argv = [sys.executable, str(HERE / script), region, *extra] - print(f"prepare-assets: running {' '.join(argv[1:])} ...") - r = subprocess.run(argv, cwd=work) - if r.returncode != 0: - die(f"{script} exited {r.returncode}") + print(f"prepare-assets: running {script} {region} {' '.join(extra)} ...") + rc = run_emit(script, (region, *extra), work) + if rc != 0: + die(f"{script} exited {rc}") want = work / "data" / produces.format(region=region) if not want.is_file(): die(f"{script} did not produce {want}") @@ -158,11 +198,8 @@ def main(): if not args.keep_temp and staged_rom.exists() and staged_rom.resolve() != rom.resolve(): staged_rom.unlink() - print("\nprepare-assets: done. You can now run ge007.x86_64.exe from the bundle folder.") + print("\nprepare-assets: done. You can now run the game from the bundle folder.") print(f" (Your ROM must also be at {dest_data / f'ge007.{region}.z64'} for the game itself.)") - if region != "ntsc-final": - print(" NOTE: only the NTSC-U ROM is validated in this alpha; " - f"'{region}' is experimental.") if __name__ == "__main__":