diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f03fb7a..d663553c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,6 +113,12 @@ jobs: - name: Byte-compile Python tooling run: python3 -m compileall -q scripts tools_pc + # Generated dev-docs stay in sync with the tree (speed-ups plan Step 4). + - name: Check generated dev indexes are current + run: | + python3 tools_pc/gen_findings_index.py --check + python3 tools_pc/gen_env_probes.py --check + # US only. gen_romassets.py resolves every symbol for the US region from # the committed CSVs with no ROM present; PAL/JP each need a handful of # region-specific prop/font offsets that only come from that region's ROM @@ -141,6 +147,12 @@ jobs: env: CC: gcc-14 CXX: g++-14 + # CMakeLists.txt opt-in enables CMAKE_{C,CXX}_COMPILER_LAUNCHER=ccache + # when ccache is on PATH (b4de1c68, speed-ups plan Step 5). Game + # sources are frozen (1:1 with the decomp), so after the first run only + # touched port/ files miss cache -- near-total hit rate. + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_MAXSIZE: 500M steps: - uses: actions/checkout@v7 with: @@ -150,7 +162,16 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential gcc-14 g++-14 cmake python3 \ - libsdl2-dev zlib1g-dev libgl1-mesa-dev + libsdl2-dev zlib1g-dev libgl1-mesa-dev ccache + + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-linux-${{ hashFiles('CMakeLists.txt') }}-${{ github.sha }} + restore-keys: | + ccache-linux-${{ hashFiles('CMakeLists.txt') }}- + ccache-linux- - name: Regenerate ROM-asset symbol file (US) run: python3 scripts/gen_romassets.py u @@ -162,6 +183,9 @@ jobs: - name: Build (ntsc-final) run: cmake --build build-linux -j"$(nproc)" + - name: ccache stats + run: ccache -s + - name: Show binary run: | ls -la build-linux/ge007* || true @@ -204,6 +228,13 @@ jobs: defaults: run: shell: msys2 {0} + env: + # Same opt-in ccache CMakeLists.txt already wires up (Step 5). Note: + # this cache path is read/written by the JS actions/cache step outside + # the msys2 shell too, so it stays a plain workspace-relative path + # rather than an msys /c/... form. + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_MAXSIZE: 500M steps: - uses: actions/checkout@v7 with: @@ -219,15 +250,28 @@ jobs: mingw-w64-x86_64-SDL2 mingw-w64-x86_64-zlib mingw-w64-x86_64-python + mingw-w64-x86_64-ccache zip unzip + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-windows-${{ hashFiles('CMakeLists.txt') }}-${{ github.sha }} + restore-keys: | + ccache-windows-${{ hashFiles('CMakeLists.txt') }}- + ccache-windows- + - name: Regenerate ROM-asset symbol file (US) run: python scripts/gen_romassets.py u - name: Build (ntsc-final) run: ./build-pc.sh ntsc-final + - name: ccache stats + run: ccache -s + # Package the distributable bundle on every run so the release path is # always exercised. On a tag build VERSION is the tag; otherwise it is a # throwaway CI string. bundle-win.sh hard-fails if a ROM image or an diff --git a/CLAUDE.md b/CLAUDE.md index b2816cee..249a88db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,6 +54,9 @@ A subagent brief is INVALID and must not be sent unless it contains all of: write-up of a half-solved bug is a deliverable, not a failure. - **KNOWN-GOOD / RULED OUT**, **CONSTRAINTS** (no game-logic changes; ABI/layout/format only; `#ifdef PORT`; documented in §F), **REPORT** shape. +- **VERIFY** — the exact `tools_pc/verify.sh` / probe invocation that proves + the task done, or an explicit "runtime verification impossible here; a human + must run X". - Instruction to read `docs/porting-notes.md` first and append any generalisable quirk to it. @@ -85,9 +88,16 @@ still owns judgment, planning, and the verification gate. - **Backend reality:** ~1 inference slot, slow per token — dispatch **sequentially**, `max_tokens` ~4–8K, treat as batch work. Needs the LiteLLM proxy (`litellm/run.ps1`) + Unsloth Studio both up. -- **Windows quirk:** the worker's shell is `cmd.exe`, not bash — its agent - prompt tells it to use `read_file`/`findstr`, not `cat`/`grep`. Always - spot-check the returned artifact against the real files before integrating; - the 27B will confidently fabricate if a read fails. +- **Corrected 2026-09-04:** the "worker's shell is `cmd.exe`, no grep/cat" + claim that used to live here was WRONG — verified directly against the + harness's own `_run_bash` (`C:\Users\james\tools\claude-code-delegate-local`): + `grep`, `sed`, `cat`, `ls`, `head`, pipes, and `grep -rn` all work fine + (Git-for-Windows/MSYS2 toolchain on PATH). The stale claim, baked into 4 of + 5 local agent `.md` files, caused a real dispatch (a multi-file + cross-referencing investigation) to burn its whole turn budget + full-`read_file`-ing files instead of grepping them, hit the turn limit, and + returned nothing. Agent files fixed. Always still spot-check the returned + artifact against the real files before integrating — the 27B will + confidently fabricate if a read fails, independent of this. - Fits the **dispatch preflight** rules above (FILES / BUDGET / ON EXPIRY / CONSTRAINTS / REPORT) — write the `task` string to that shape. diff --git a/docs/dev-process.md b/docs/dev-process.md index 79293c5c..35664b35 100644 --- a/docs/dev-process.md +++ b/docs/dev-process.md @@ -101,6 +101,7 @@ KNOWN-GOOD / RULED OUT: . PRE-FLIGHT ATTACHED: . BUDGET: run cycles / ~M min>. On expiry: revert probes, write up with confidence. CONSTRAINTS: no game-logic changes; ABI/layout/format only; #ifdef PORT; documented in the finding log. +VERIFY: . REPORT: (a) root cause + file:line evidence (b) fix diff, or why not (c) probes left in tree (d) confidence. Append any generalisable quirk to docs/porting-notes.md. diff --git a/docs/dev/GE-ENV-PROBES.md b/docs/dev/GE-ENV-PROBES.md index 9c645317..ff5edcbb 100644 --- a/docs/dev/GE-ENV-PROBES.md +++ b/docs/dev/GE-ENV-PROBES.md @@ -1,7 +1,9 @@ # GE_* environment probes — consolidated list -Every `getenv("GE_…")` in the tree, from a grep of `src/` + `port/` on -2026-08-31 (master `31d036da`). All are **env-gated**: unset = zero behavior +Every `getenv("GE_…")` in the tree. The table prose is hand-curated; +**`tools_pc/gen_env_probes.py`** re-greps the live sites and reports drift +(NEW / GONE vars + a fresh file:line map) — run it before trusting the +File:line cells. Last reconciled 2026-09-04 (M-51). All are **env-gated**: unset = zero behavior change, so they are safe to leave in a release build (they only cost a `getenv` on the guarded path). "Dead" below means *the finding it was built for is closed* — the probe is strip-candidate scaffolding, not that it does @@ -22,6 +24,10 @@ Two classes: | `GE_INPUTLOG` | `port/src/input.c:329`, `port/src/config.c:53` (`configGetInputLog`) | Logs computed pad state (buttons + stick) each poll when non-zero. Also `[Debug] InputLog` ini. Also traces D165 `menuptr est/tgt/eff/stick`. | **live** | | `GE_STARTMENU` (+ `GE_STARTMENU_PAGE`, `GE_STARTMENU_DIFF`) | `src/game/lv.c:398-402` (`#ifdef PORT`) | Boot straight into a front-end menu id (13=MISSION_COMPLETE, 10=BRIEFING, 7=MISSION_SELECT, 12=MISSION_FAILED, 6=MODE_SELECT); `_PAGE`=folder row (def 1=Dam), `_DIFF`=0..3. Crash-test a screen fast. | **live** | | `GE_UNLOCK_ALL` | `src/game/debugmenu_handler.c:1098` (`get_debug_enable_all_levels_flag`, `#ifdef PORT`) | Mission-select shows every solo level (playtest jump-to-any-level aid). Cached in a `static int c`. | **live** | +| `GE_OPTIONSOVERLAY` | `port/src/optionsoverlay.c:231` | Force-enable the F10 port-layer options overlay (D184 / PR #10) regardless of ini. `atoi != 0`. | **live** | +| `GE_TEXPITCH` | `port/fast3d/gfx_pc.cpp:924` (`gfx_tex_pitch_fix`, cached) | `=0` disables the D183 defensive de-stride in `import_texture` (default on). RC2/RC3 texture-shear A/B knob. | **live** (RC3 not finished) | +| `GE_TEXRAW` | `port/fast3d/gfx_pc.cpp:1079` | With `GE_TEXDUMP`, also dumps the raw pre-import source bytes (`texdump/rNNN_f…_s…_WxH.bin`) so a decode bug can be told from a source-data bug offline (D183). | **live** (RC3 not finished) | +| `GE_WRAPFIX` | `port/fast3d/gfx_pc.cpp:3166` (`gfx_set_wrap_fix`) | RC3 test override for the `Video.WrapFix` D74/D167 wrap-block path (`atoi != 0` wins over the ini). | **live** (RC3 not finished) | | `GE_SAVELOG` | `src/game/file.c:46`, `src/game/file2.c:16` (`SAVELOG` macro), `src/boss.c:749`, `src/game/objective_status.c:297,343` | Traces the campaign save/unlock chain (objective completion → `end_of_mission_briefing` → `fileWriteSave` → EEPROM). Used to confirm D157. `g_savelogObjOnce` gates the per-criterion dump to the `bossReturnTitleStage` call. | **live** (strip once a few more level boundaries are playtested) | | `GE_D160` | `src/boss.c:744`, `src/game/bondview2.c:791`, `src/aicommands.def:10055,10181,10236,10242,10247` (all `#ifdef PORT`) | Dam exit-cutscene (D148/D160) diagnostic: trace points in `bossReturnTitleStage`, `bondviewSetCameraMode`, and AI cmds `EndLevel`/`exit_level`/`CameraLookAtBondFromPad`/`CameraSwitch`. **Investigation in progress** — user owes a `GE_D160=1` Dam run. | **live** | | `GE_DTEX` | `port/fast3d/gfx_pc.cpp:1001` | Per-`import_texture` param log (dims / line / size / lod / gen_mipmaps). Used to root-cause RC2 and D159. Kept as an inert diagnostic. | **live** (RC2/RC3 not finished) | @@ -33,7 +39,10 @@ Two classes: | Env var | File:line | Finding / what it logged | Status | |---|---|---|---| | `GE_D51` | `port/src/libultra.c:496`, `src/game/model.c:125,244,544` | msgQ 32-slot overflow watch / ModelSlot layout | dead (D51 closed) | -| `GE_D54` | — (all blocks stripped M-32, commit `49ce620a`) | music seq-table ABI / endianness (`ALMidiHdr`) | dead (D54 closed) — fully removed | +| `GE_D54` | — (all blocks stripped M-32, commit `49ce620a`) | music seq-table ABI / endianness (`ALMidiHdr`) | dead (D54 closed) — fully removed (kept here as a tombstone; the generator flags it GONE) | +| `GE_D154` | `src/game/bg.c:3418,3447,3632` (capped 64 calls) | bg room-GDL call trace (room / gdlidx / vtxoff / op / raw hdr words) | dead (D154 closed) | +| `GE_D176` | `src/game/bgfog.c:459,471`, `src/game/sky.c:325,381` (all `#ifdef PORT`) | D176 sky/fog env-match + sky-vert trace (Path B, PR #18) | **live** (D176(a)/(b) OPEN) | +| `GE_D178` | `src/game/front.c:6586,6594` (`#ifdef PORT`) | briefing-data u16 decode trace (blank-objectives, D143/D178) | dead (D178 closed) | | `GE_D56` | `src/game/model.c:227,830` | watch `Model` raw-offset reads | dead (D56 closed) | | `GE_D60` | `port/src/libultra.c:521,735` | gfx frame msgQ delivery trace | dead (D60 closed) | | `GE_D61` | `port/src/libultra.c:795` (`s_d61opened`) | opens a one-shot log file | dead (D61 closed) | @@ -62,5 +71,9 @@ Two classes: - `[Debug]` ini keys (`Debug.FrameDump`, `Debug.InputLog`) mirror `GE_PCDUMP` / `GE_INPUTLOG` as a fallback (env var wins) — M-26, `port/src/config.c`. -- Not GE_-prefixed but same category: `GE_DETERM` is **designed, not - implemented** (fixed-tick determinism mode, §F D117). +- Not GE_-prefixed but same category: `GE_DETERM` is **implemented but + experimental — not yet achieving its determinism goal** (fixed-tick + mode, §F D117, M-52 four-pass writeup); `GE_DETERM_TRACE` (M-52 4th + pass) adds instrumented clock-event logging when used alongside it, for + diffing two runs to localize remaining divergence. Both env-gated, + default off, zero cost when unset. diff --git a/docs/dev/findings-index.csv b/docs/dev/findings-index.csv new file mode 100644 index 00000000..5197501c --- /dev/null +++ b/docs/dev/findings-index.csv @@ -0,0 +1,80 @@ +# GENERATED by tools_pc/gen_findings_index.py -- do not hand-edit. +# Grep this to pick which findings.md entries to read; it is not authoritative. +label,bucket,one_liner,status_raw +D24-D30,RESOLVED,green threads / kernel / crash handler,resolved +D31-D42,RESOLVED,"boot ABI: ANIM_DATA, music seq-table, libaudio banks, Globalimagetable rebase, GL context, rsp toggle",resolved +D43-D50,RESOLVED,"model-file loading → offline sidecar ""Plan B"" (d43_emit.py); D50.6 texCopyGdls full-slot copy",resolved +D51-D58,RESOLVED,"font pixeldata, osGetCount tick rate, model RW pools, cseq BE header, synth slots, RLE menu bg, watch raw offsets, gun-barrel DL",resolved +D59-D68,RESOLVED,"intro render: blood-RLE clobber, DMA validate, OSMesgQueue, HEADS/BODIES sentinels, romCopy width, image_entry, Globalimagetable BE→LE",resolved +D70-D74,RESOLVED,"intro-logo pixels: C-array bswap, UV path, sinf/cosf DVAL(), texture-import truncation",resolved +D75 · D77,OPEN,front-end 3D model transforms · audio,"OPEN (parked). M-32 triage (see §F ""D75 ADDENDUM""):" +D76 · D164,PROPOSED,disclaimer/legal screen only draws line 1 —,fix proposed (not applied — +D159,FIXED,"front-end wallet-Bond photo ""interlaced""/combed (RC1 / D149) —",FIXED (#ifdef PORT no-op the swap in image.c) +D161,FIXED,Depot (-level_30) ceiling = bright-blue speckle + radial rays (B2).,FIXED (port/fast3d/gfx_pc.cpp) +D165 · D166,FIXED,"input-layer polish (M-31, port-only, port/src/input.c):",FIXED +D180 · D181,PARTIAL,"native-PC input pass (QoL run, qol/native-pc-input-menu):","LANDED, config-gated, feel-checks owed" +D186,PARTIAL,"Video.FpsCap throttles the sim, not just presentation (M-39 diag, M-42 clamp).","CLAMP LANDED (port/fast3d/gfx_sdl2.cpp, port/src/video.c); real fix owed" +D187,FIXED,"HUD ammo-type/weapon icon vanished at -O2 (M-46 root-cause, M-47 fix + verify).","FIXED (src/game/gunfire.c, #ifdef AVOID_UB); PR #19" +D188,FIXED,Linux -O2 boot crash:,"FIXED (src/libultrare/libc/xprintf.c, src/sprintf.c, #ifdef PORT); PR #22" +D189,FIXED,Linux -fstack-protector aborts on the decomp's latent stack-buffer overruns (M-48) —,"FIXED (CMakeLists.txt, src/game/stan.c #ifdef PORT); PR #23" +D190,OPEN,"Linux -level_45 SIGSEGV at load in add_ptr_to_objective (objective.c:64) (M-48, open).",OPEN — not investigated (M-48) +D191,OPEN,Windows -O2 release-bundle crash:,OPEN — not investigated (M-49). +D193,OPEN,NPC AI locomotion is too slow —,"OPEN — root-cause hypothesis (timing family), not investigated (M-49)." +D194,OPEN,Mouse input needs another tuning pass —,"OPEN — root-caused, not fixed (M-49)" +D196,OPEN,"OS mouse cursor stays visible after closing the F10 options overlay in click-to-lock mode (M-49, user playtest).","OPEN — root-caused, 1-line fix identified, not applied (M-49)" +D192,OPEN,Front-end mission/level-select grid pointer still can't reach the outer cells —,"OPEN — root-caused, not fixed (M-49)" +D152+,FIXED,D152 addendum (M-31):,"FIXED (src/snd.c, port/src/libultra.c, #ifdef PORT) —" +D168,FIXED,GE_PCDUMP / F12 PPM captures were vertically flipped —,FIXED (port/fast3d/gfx_opengl.cpp); D114 / D116 reclassified as capture-orientation artifacts (below). +D169,FIXED,Front-end mouse pointer can't reach the outer cells of the mission/level-select grid —,"FIXED (M-33, port/src/input.c, port-only, no #ifdef PORT)." +D178,FIXED,Pre-mission briefing screen:,FIXED (M-36) — high confidence +D175,?,In-game stutter / brief hang during normal play (user QA report):,"Observed, not investigated" +D176,OPEN,"Surface exterior renders wrong (-level_36), two independent defects.","(a) root-caused, fix owed / (b) open" +D177,FIXED,Ladders non-functional —,FIXED (M-36) — high confidence; interactive climb test owed +D174,?,"""No blood effect"" (user QA report).",Observed; first step = BUNKER1 firefight with regenerated sidecars +D183,OPEN,"M-36 Family A (""texture line/pitch shear"") is DISPROVEN for the Surface repro (-level_36).",Hypothesis DISPROVEN + diagnostics shipped; D176(b)/D182(2) still OPEN +D173,?,Third-person Bond model spawns too high (user QA report):,"Observed, not investigated" +D179,FIXED,Packaged build crashes after the logos —,"Diagnosed, not fixed —" +D180,PARTIAL,"Native-PC input pass (M-XX QoL run, qol/native-pc-input-menu, port-only).","LANDED, port-only, no #ifdef PORT." +D181,PARTIAL,Game.ScreenShakeIntensity —,"LANDED behind config, default = original." +D184,PARTIAL,F10 port-layer options overlay —,"LANDED, port-only, no #ifdef PORT in src/." +D85,OPEN,room primary/secondary DL → texLoadFromGdl garbage,"OPEN (safety-netted; widen/pool fixes landed, geometry now renders)" +D86 · D87,RESOLVED,modelInitRwData truncated ptr · attract-demo BE ramromfilestructure,resolved +D88.4,RESOLVED,propDefs polymorphic record stream not byteswapped → setupDoor crash,resolved (d88_propdefs.py); layout audit M-20 → D132 +D132,PROPOSED,D88 propDefs layout audit (M-20):,proposed fix (not applied) — +D157,FIXED,"Campaign never unlocks the next level (M-30, user bug —","FIXED (src/bondtypes.h, #ifdef PORT —" +D160,?,Dam level-end cutscene (Bond off the dam / bungee) never plays —,"DIAGNOSTIC SHIPPED, not root-caused (c95713f5, #ifdef PORT + GE_D160=1):" +D89-D92,RESOLVED,"stage-load→frame: stan zero-fill overrun, portal address trunc, chr/AI spawn ptrs",resolved +D93-D102,RESOLVED,struct-pun / hardcoded-size pass:,resolved (D95 pool bump partial) +D103-D107,RESOLVED,"BUNKER1 viewport height, depth-buffer clear (G_CLEAR_DEPTH_EXT), portal near-plane, LOD mip tile",resolved +D108-D112,RESOLVED,skeletal models: d43_emit.py put_f32 byte-reversal bug,resolved +D113-D116,FIXED,portal BFS ok · matrix chain ok · player raw-offset audit + gunfire.c THROW* · HUD text X-mirror,"D114/D116 CLOSED — NOT A BUG (M-33, see D168):" +D117,OPEN,frame nondeterminism root-caused; GE_DETERM deferred; framediff.py structural.,"OPEN (mode attempted, not achieved)" +D118,RESOLVED,SDL input layer (port/src/input.c); M-24 mouse-look rework:,resolved (D118a residual) +D150,FIXED,watch OBJECTIVES / BRIEF page crash:,FIXED (src/str.c #ifdef PORT); interactive re-verify pending +D137,FIXED,right-mouse (aim-sight) crash:,FIXED (src/game/gunfire.c #ifdef PORT — +D119 · D120,FIXED,guard-attack weapons_held[]->chr pun crash · blood-stain PointUsage[] chain hang,D119 fixed · D120 FIXED (M-30) +D120,FIXED,Blood-decal PointUsage[] chain —,FIXED (tools_pc/d43_emit.py). +D121,RESOLVED,WS1 frictionless per-level boot:,resolved +D122,FIXED,per-level prop/item model-load crash (Dam/Facility/Runway modelLoad/modelInitRwData):,converter fixed; residual chr/fast3d crashes on those levels are separate +D123,FIXED,crash class C1: chrIsNotDeadOrShot NULL deref on 6 levels (Dam/Runway/Frigate/Statue/Streets/Cradle).,"converter fixed (OBJ_ID_WORDS); C1 cleared on all 6, residual crashes are fast3d (C2)" +D125,RESOLVED,crash classes C3+C6 (Aztec/Bunker2/Surface2):,resolved (M-16b) — line 374 → 8 NULs; all 21 sidecars regen'd; Aztec -level_28 now PASSES (was C3 CRASH). +D126,RESOLVED,"crash classes C3r/C4/C6 (Bunker2 -level_27 door7F054FB4 propobj.c:13523, Depot -level_30 prop.c:902, Surface2 -level_43 loadobjectmodel.c:393):",resolved — d88_propdefs.py PROPDEF_PC_BYTES[30/32/33/35]=24 + typed handler; loadobjectmodel.c sizepropdef PORT returns 6. +D124,FIXED,crash class C2: fast3d bad texture pointer.,Jungle fixed (port/src/gimgfixup.c); Facility → D130 +D131,RESOLVED,crash class C2m (Jungle -level_37):,"resolved — seg_addr() restores the module high word for any fallthrough w1 in [0x40000000, 0x70000000) (module fixed-based at 0x140000000, no ASLR; DRAM/KSEG0/segmented/phys all handled earlier)." +D133,?,intro render triage (M-20):,"not a regression — parked (D75/D76, GRAPHICS-BACKLOG.md)" +D134,FIXED,"Frame-2 boot hang / sweep ""NO-FRAMES"" flakiness —",FIXED (port/src/libultra.c) +D135,FIXED,"Firefight crash — bgTestHitOnObj (propobj.c:8446) is an unported N64 GBI parser (M-22, WS6 playtest).","FIXED (src/game/propobj.c, #ifdef PORT —" +D136,?,"Stage-unload crash — objFree obj->prop reads float garbage (M-22, WS6 Facility playthrough).",ROOT-CAUSED → D139 (it was NOT a layout bug — +D139,FIXED,Stage-unload crash root cause + fix (M-23).,"FIXED (src/game/cleanup_objects.c, #ifdef PORT —" +D140,FIXED,Pause-menu crash (M-23 open → M-27 fixed).,"FIXED (src/game/bondview.h, src/game/bondview2.c; #ifdef PORT)." +D141,FIXED,"Watch-menu item-model crash (M-27), exposed by D140.","FIXED (src/game/gunfire.c, #ifdef PORT)." +D152,OPEN,"Mission-failed / objective-failed → permanent black screen (M-28, user bug report —","MITIGATED (port/src/libultra.c, M-28, #ifdef PORT/port-only)." +D154,FIXED,"bg.c room-geometry ray/hit-test GBI parser port (M-28, WRITTEN / UNVERIFIED / UNCOMMITTED).",M-30 re-audit + fixes (still playtest-gated for final verify). +D156,?,"Facility outro-cutscene ""hang"" —","GUARDED (src/game/model.c, #ifdef PORT):" +D155,FIXED,"Facility outro-cutscene end → ""hang"" = unclamped deltaFrames spiral (M-29, user bug report).","FIXED (src/game/frametiming.c, #ifdef PORT —" +D153,?,"-level_09 frame ~900–1400 0xc0000005 = D117/D134 load flakiness, NOT a bug (M-28).",NOT A BUG — closed. Run anchors on a quiet machine. +D151,FIXED,Watch data all blank —,"FIXED (src/bondtypes.h, #ifdef PORT —" +D130,RESOLVED,"crash class C2 (Facility -level_34, Runway -level_35):",resolved — stage all 6 N64 fields in a local f[6] before writing any. +D164,PROPOSED,"Disclaimer / legal screen only draws the first line (D76 root cause, M-31).","FIX PROPOSED, NOT APPLIED (front.c is owned by another agent this session)." +D165,FIXED,"Front-end menu cursor doesn't feel like a mouse pointer (M-31, user bug B5 follow-up).","FIXED (port/src/input.c; Input.MenuPointerMode, Input.HipfirePitchSpeed added)." +D166,FIXED,Hipfire mouse pitch is digital while yaw is analog —,FIXED (port/src/input.c). diff --git a/docs/dev/findings.md b/docs/dev/findings.md index 9681faf0..47f148dc 100644 --- a/docs/dev/findings.md +++ b/docs/dev/findings.md @@ -424,7 +424,7 @@ covers D24–D69; the log continues in §H (D32 procedure, D70–D121). | D103–D107 | BUNKER1 viewport height, depth-buffer clear (`G_CLEAR_DEPTH_EXT`), portal near-plane, LOD mip tile | resolved | | D108–D112 | skeletal models: `d43_emit.py put_f32` byte-reversal bug | resolved | | D113–D116 | portal BFS ok · matrix chain ok · player raw-offset audit + `gunfire.c` THROW* · HUD text X-mirror | **D114/D116 CLOSED — NOT A BUG (M-33, see D168): the "X-mirror" was an upside-down `GE_PCDUMP` capture misread as mirrored. PPM writer fixed; the game renders correctly on hardware.** | -| D117 | frame nondeterminism root-caused; `GE_DETERM` deferred; `framediff.py` structural | resolved (mode deferred) | +| D117 | frame nondeterminism root-caused; `GE_DETERM` deferred; `framediff.py` structural. **M-52: `GE_DETERM=1` attempted — boot deadlock fixed, determinism NOT achieved (load-phase fallback still real-time-dependent). See addendum below.** | OPEN (mode attempted, not achieved) | | D118 | SDL input layer (`port/src/input.c`); M-24 mouse-look rework: mode-aware map (aim mode = analog stick, hipfire = digital C-pitch), `config.c` INI now real. D118b/c FIXED; D118a residual (hipfire pitch only) | resolved (D118a residual) | | D150 | watch OBJECTIVES / BRIEF page crash: `strcat(buf, langGet(id))` with `langGet` → NULL (unloaded bank) → NULL deref; `str.c` str* builtins elide a plain param NULL check, so guard via asm-laundered `GE_IS_NULL` | FIXED (`src/str.c` `#ifdef PORT`); interactive re-verify pending | | D137 | right-mouse (aim-sight) crash: `gunDrawSight` (`gunfire.c:6235`) `s32 sp54` holds a `Gfx*` the whole function (`sp54 = *gdl`; `texSelect(&sp54,…)`/`display_image_at_position(&sp54,…)` take `Gfx**`). As N64 `s32` it is 4 B, so `*(Gfx**)&sp54` reads 4 B of the adjacent stack float as the pointer high word → `_g = 0x03e4fca0_70081220` → wild DL write in `texSetRenderMode` (`othermodemicrocode.c:177`) the moment the crosshair raises. §A. | FIXED (`src/game/gunfire.c` `#ifdef PORT` — `sp54` is `Gfx *`) | @@ -3675,6 +3675,195 @@ Golden set at `tools_pc/golden/frame_0002{00,320,440}.png` is the D115 baseline — since it is a nondeterministic capture, only structural mode is meaningful against it today. +**D117 — M-52 addendum: `GE_DETERM=1` implemented per the design above, +tested live (Windows console). Boot deadlock FIXED as predicted; full +determinism NOT achieved. Confidence: HIGH on both findings (live-tested, +not argued).** + +Implemented exactly the 5-step design (`port/src/libultra.c`, env-gated, +default off, zero change to the non-`GE_DETERM` path): `osGetCount()` +returns a virtual `g_determTicks` seeded to the region quantum; a +`g_determFrameReady` flag is set at the `osSpTaskStartGo` graphics-task call +site (i.e. by the real "a frame was presented" event, not `videoEndFrame` +directly — same effect, one call site, no `video.c` change needed); +`portTickThread` gets a `GE_DETERM` branch that advances the clock + posts +one retrace only when that flag is set, polling every 200 µs instead of +pacing to the real 16.7/20 ms tick interval. + +*Step 1(b) risk, confirmed exactly as written.* Without a load-phase +fallback, `-level_09 GE_DETERM=1` hangs before the first frame: symbolicated +the live hang (not guessed) — `shedThread` blocked in `osRecvMesg` +(`sched.c:256`, `__scMain`) and `mainThread` in `joyCheckStatusThreadSafe` +(`joy.c:166`), both waiting on the scheduler's interruptQ (the same queue +`portPostVIEvent` posts into, per the D134 comment), which never receives a +retrace because no frame has rendered yet to set `g_determFrameReady` — +exactly the chicken-and-egg the design called out ("boss.c:442 hangs the +same way"). + +*Fallback added, per the design's own suggestion — fixes the hang, breaks +determinism.* Added: when `g_determFrameReady` is false AND +`g_viRetraceMQ->validCount == 0` (nothing left to consume from a prior +tick), advance the clock and post a retrace anyway, so a boot-time waiter +isn't stuck forever. This **does** fix the hang — `-level_09 GE_DETERM=1` +now boots and renders past frame 3000+ instead of hanging. It does **not** +achieve determinism: two independent runs, same build, same command, +`framediff.py --exact` at the golden frame numbers (200/320/440): + +``` +FAIL frame_000200: 276998/307200 px over tol (90.169%, maxchan=218) +FAIL frame_000320: 85131/307200 px over tol (27.712%, maxchan=102) +FAIL frame_000440: 47936/307200 px over tol (15.604%, maxchan=102) +``` + +That's *worse* at frame 200 than the port's existing wall-clock +nondeterminism without any `GE_DETERM` mode at all (32–69 % per the +original D117 measurement above) — the fallback didn't just fail to fix +determinism, it introduced a new, bigger source of it. Root cause (analysis, +not yet re-tested): the fallback fires whenever the 200 µs poll happens to +observe an empty queue, which during the load phase depends on real +OS-scheduling latency between the polling thread and whatever the load path +is doing — so the *number* of extra ticks consumed before the first real +frame renders varies run-to-run, exactly the same class of divergence D117 +already documents, just relocated from "wall-clock frame pacing" to +"wall-clock load-phase tick count." User-observed side effect, consistent +with this: with the fallback active the game visibly runs at a different +apparent frame rate and sped up — because the tick loop is no longer paced +to real 60/50 Hz at all once it starts free-running on empty-queue polls, +it advances (and lets the sim advance) as fast as the poll thread schedules, +not at any fixed rate. + +**D117 — M-52 2nd attempt (same session): moved generation from a polled +fallback to a synchronous one. Root-caused precisely why that still isn't +enough. Confidence: HIGH (live-tested, mechanism traced through sched.c).** + +Traced the actual consumer chain instead of guessing: `sched.c` `__scMain` +is the **sole** consumer of `g_viRetraceMQ` (`sc->interruptQ`), and every +retrace it processes drives task execution (`osSpTaskStartGo`), `joyPoll()` +(which is what unblocks `joyCheckStatusThreadSafe` too — the other M-52 +1st-attempt hang site, for free), and the forward to `mainThread`, all +**synchronously on that one thread** before it asks for the next message. +So replaced the poll-driven tick-thread fallback with generation **inside +`osRecvMesg()` itself**: when `GE_DETERM` is on and the sole consumer of +`g_viRetraceMQ` finds it empty, synthesize the message inline (advance +`g_determTicks`, enqueue) instead of blocking — driven by call sequencing, +not by an independent thread's wall-clock poll. + +Result: still boots (no regression on the deadlock fix), and the divergence +profile changed but did not go to ~0%: two runs, `framediff.py --exact`: + +``` +FAIL frame_000200: 101582/307200 px over tol (33.067%, maxchan=150) +FAIL frame_000320: 116844/307200 px over tol (38.035%, maxchan=161) +FAIL frame_000440: 43617/307200 px over tol (14.198%, maxchan=117) +``` + +Better than the 1st attempt's 90/28/16% at frame 200, but still nowhere +near the ~0% acceptance target. **Root cause, found by diffing the two +boot logs line-for-line** (not guessed): `boss.c`'s boot sequence has a +genuine, bounded, wall-clock real-time wait — +`for (i = 0; i != MAXCONTROLLERS; i++) { osSetTimer(100ms); osRecvMesg(BLOCK); }` +(`src/boss.c:188-190`, controller detection) — untouched by `GE_DETERM` by +design (`osGetTime()`/real timers are explicitly out of scope). During +that ~400 ms real window, **nothing is yet consuming `__scMain`'s forwarded +retraces** (`mainThread` hasn't reached its own retrace-consuming loop +yet), so the new synchronous-generation code has nothing throttling it — +`__scMain` free-spins through `osRecvMesg` as fast as the OS schedules that +thread, and *how many* virtual ticks accumulate during that fixed 400 ms of +real time depends on real thread-scheduling throughput. Same nondeterminism +class as both prior attempts, relocated a third time: generation-on-ask is +only deterministic when something else deterministically paces the asking, +and right now nothing does during any window where a client isn't actively +mid-frame. + +**Disposition:** kept in the tree, env-gated (`GE_DETERM` unset by default, +zero behavior change confirmed via `verify.sh bunker1` before/after across +both attempts — worst cell 12.1–12.5 throughout, within existing noise). +The single-consumer synchronous-generation mechanism in `osRecvMesg()` is +the right shape and should stay — it's strictly better than polling and +fixed a real deadlock class cleanly. What's still missing: a genuine +**rate limit** on `__scMain`'s synthetic-retrace generation, decoupled from +real time — e.g. only synthesize the next one once the client has actually +finished processing the previous forward (a real request/response coupling +to `mainThread`'s progress, not just "the queue looked empty"), with a +separate, explicitly-bounded allowance for pre-first-frame boot code that +legitimately needs to service its own `osSetTimer` waits without the +scheduler racing ahead. That is a genuine design task, not a quick +follow-up — leaving D117 OPEN. `porting-notes.md` not updated (this +addendum's lesson is already exactly what §D117 above says, restated at +one more layer: any unthrottled or wall-clock-throttled generator racing +against a genuinely real-time-bound consumer reintroduces the same +divergence class, no matter how many layers deep you push it). + +**D117 — M-52 4th pass: instrumented tracing (`GE_DETERM_TRACE=1`) precisely +localizes what's left — and it is NOT the virtual clock. Confidence: HIGH, +directly measured, not inferred.** + +Rather than keep auditing code paths by inspection (two dispatches -- one to +the local delegate, one done directly -- had just spent a round ruling out +the level-load path and the audio-thread shared-globals angle with nothing +to show for it), added a temporary trace: every `g_determTicks` +advance/cap event and the `first_task_run` transition get a monotonic +sequence number + timestamp-free log line (`port/src/libultra.c`, gated by +`GE_DETERM_TRACE=1`, zero cost when unset). Ran `-level_09` twice, diffed +the two trace logs directly instead of guessing. + +**Result: the tick mechanism is now proven fully deterministic.** Both runs +are byte-identical (same `ticks=` value at every logged step) all the way +through the `first_task_run` transition — the only difference is *which +spin iteration* it happens on (run 1: spin 4020, run 2: spin 4086 — 66 more +capped no-op spins in run 2, real-OS-scheduling-dependent as expected), but +the **virtual clock value at that moment is identical in both** (1551750 = +`2 × g_determQuantum`). Re-aligning the two logs by that spin-count offset +shows the entire subsequent tick progression is byte-for-byte identical. +The M-52 3rd-attempt fix (cap pre-task ticks to a fixed constant) works +exactly as designed — this is not where the remaining divergence comes +from. + +**The actual mechanism, found by reading `sched.c` with the trace numbers +in hand:** every synthesized retrace — capped ones included — is a real +`VIDEO_MSG` that `__scMain` processes via `__scHandleRetrace()` +(`src/sched.c`), which unconditionally does `sc->frameCount++` +(`sched.c:319`) and then, per registered client, gates forwarding on +`(*((s32*)client + 2) == 0) || ((sc->frameCount & 1) == 0)` +(`sched.c:334`) — i.e. some clients only receive every *other* frame's +retrace by `frameCount` parity. The audio client is registered as exactly +this kind of half-rate client: `osScAddClient(&os_scheduler, +&g_AudioClient[0], &g_AudioManager.frameMessageQueue, 1)` (`src/audi.c:437`, +the trailing `1` is the half-rate flag). Since `frameCount` increments on +*every* capped spin — not gated by the virtual clock at all — and the total +spin count before the first real task varies run-to-run by a genuinely +real-time-scheduling-dependent amount, **`frameCount`'s parity at the +moment real gameplay begins is not guaranteed to match between runs**, even +though the clock value is now provably identical. This run pair happened to +land on a same-parity gap (66, even) — a different pair easily could not. +If it lands on opposite parities, the audio client's very first retrace +forward shifts by one frame's phase between runs, and audio-task scheduling +(which shares the same `osSpTaskStartGo`/tick-gating machinery) diverges +from there. + +**Why no 5th live-coded attempt this session:** the natural next lever — +don't call `__scHandleRetrace` (and thus don't increment `frameCount`) for +a purely-capped "keep `__scMain`/`joyPoll` alive" spin — reopens the exact +chicken-and-egg the 1st attempt hit: `joyCheckStatusThreadSafe`'s handshake +and `__scMain`'s own unblocking need *something* processed during that +window, and `sc->frameCount` is decomp-internal state with no port-visible +accessor to correct after the fact. A real fix needs `frameCount`'s parity +(or the half-rate audio client's phase) to be made deterministic +independent of the real-time-dependent capped-spin count — a genuine +4th-layer design task, not a variant of any of the first three attempts. +Stopping here deliberately rather than rushing a live guess under momentum, +per the same discipline the earlier attempts were written up under. + +**Disposition:** `GE_DETERM_TRACE` tracing kept in the tree alongside +`GE_DETERM`, same env-gating, zero cost when unset — it's the tool that +should be reused for the next attempt rather than re-instrumented from +scratch. D117 stays OPEN. porting-notes.md not updated yet (the +generalisable lesson here — "a per-frame parity/half-rate dispatch gate +fed by an unconditionally-incrementing counter inherits whatever +nondeterminism feeds that counter, even after the counter's *primary* +consumer is made deterministic" — is specific enough to this scheduler +that it's better captured here than genericized prematurely). + **D118 (session M-9) — SDL input layer implemented (Phase 3).** `port/src/input.c` was an unimplemented stub; input only worked via `libultra.c`'s `contSnapshotFromKeyboard()` (Z/Space=A, X=B, LCtrl=Z, diff --git a/port/src/libultra.c b/port/src/libultra.c index d4fe542f..872718ed 100644 --- a/port/src/libultra.c +++ b/port/src/libultra.c @@ -86,8 +86,46 @@ OSTime osGetTime(void) return (OSTime)sysGetMicroseconds(); } +/* GE_DETERM=1 — fixed-tick deterministic mode (speed-ups plan Step 7 / R1, + * design in docs/dev/findings.md §D117). Test-only, default OFF: replaces + * the wall-clock osGetCount() below with a virtual clock advanced exactly + * once per virtual VI retrace (generated synchronously in osRecvMesg() — + * see the GE_DETERM block there — when __scMain, the sole consumer of + * g_viRetraceMQ, asks for the next message; NOT paced by an independent + * thread) instead of real elapsed time, so two runs of the same build take + * the same simulation path. Never the shipping timing model — the default + * (GE_DETERM unset) path below is byte-for-byte what it always was. */ +static int g_determEnabled = 0; +static u32 g_determQuantum = 775875; /* NTSC; portKernelInit sets 931050 PAL */ +static u32 g_determTicks = 0; +/* M-52 3rd attempt: __scMain is the sole consumer of g_viRetraceMQ, but + * before the first real gfx/audio task ever runs (src/boss.c's bounded + * osSetTimer(100ms)-based controller-detection loop -- genuinely real-time, + * out of scope by design), NOTHING is consuming its forwards, so unbounded + * per-ask generation free-spins for that whole real-time window and however + * many ticks accumulate is real-scheduling-dependent (confirmed: 2nd + * attempt's 33/38/14% divergence). waitForNextFrame() (frametiming.c) is a + * tight busy-spin on osGetCount(), not a queue wait -- so FREEZING the + * clock during that window (0 extra ticks) hangs it forever, and letting it + * free-run is exactly the bug. The fix: cap the clock to exactly ONE extra + * quantum beyond the seed before the first task runs -- fixed, identical + * every run, and >=1 quantum is already enough to clear + * waitForNextFrame()'s first-call threshold (see the arithmetic in + * frametiming.c), so it can't hang either. Set true by osSpTaskStartGo. */ +static int g_determTaskEverRun = 0; +static int g_determPreTaskTicksGranted = 0; +/* GE_DETERM_TRACE=1 (with GE_DETERM=1): log every virtual-clock advance with a + * monotonic sequence number, for diffing two runs' logs to find the exact + * first point they diverge -- see docs/dev/findings.md §D117 M-52. Test-only, + * both env-gated off by default; zero cost when unset. */ +static int g_determTraceEnabled = 0; +static u32 g_determTraceSeq = 0; + u32 osGetCount(void) { + if (g_determEnabled) { + return g_determTicks; + } /* The N64 RSP core counter advances at ~OS_CPU_COUNTER (~46.5 MHz), NOT * microseconds. GE's pacing assumes this rate: waitForNextFrame() in * frametiming.c waits for 775,875 - 387,937 ticks per NTSC frame (931,050 @@ -234,6 +272,25 @@ static void *portTickThread(void *arg) { (void)arg; for (;;) { + /* GE_DETERM (§D117, M-52 2nd attempt): retrace generation lives in + * osRecvMesg() now (see there) -- synchronous with the sole + * consumer asking for the next message, not paced by this thread. + * The M-52 FIRST attempt polled g_viRetraceMQ's empty/non-empty + * state from here on a real-time cadence; that poll's own + * OS-scheduling latency varied run-to-run and leaked directly into + * the tick count (measured: 90%/28%/16% frame divergence between + * two runs, worse than the port's baseline wall-clock + * nondeterminism this mode exists to remove). This thread still + * only does what osGetTime()-based real-time bookkeeping needs + * (timers, hang-heartbeat) -- deliberately NOT touching + * g_viRetraceMQ at all in this mode. */ + if (g_determEnabled) { + portServiceTimers(); + portHeartbeatCheck(); + sysSleep(g_tickIntervalUs); + continue; + } + uint64_t now = sysGetMicroseconds(); if (!g_nextTickUs) g_nextTickUs = now + g_tickIntervalUs; @@ -272,8 +329,25 @@ void portKernelInit(void) if (osTvType == OS_TV_MPAL || osTvType == OS_TV_PAL) { g_tickIntervalUs = 1000000 / 50; /* PAL: 50 frames/s -> 20ms */ + g_determQuantum = 931050; } else { g_tickIntervalUs = 1000000 / 60; /* NTSC: 60 frames/s -> ~16.7ms */ + g_determQuantum = 775875; + } + + /* GE_DETERM=1: seed so the first waitForNextFrame() computes exactly + * (quantum + 387937) / quantum == 1 tick, per the §D117 design. */ + g_determEnabled = getenv("GE_DETERM") != NULL; + if (g_determEnabled) { + g_determTicks = g_determQuantum; + g_determTraceEnabled = getenv("GE_DETERM_TRACE") != NULL; + sysLogPrintf(LOG_NOTE, + "GE_DETERM=1: fixed-tick deterministic mode (test-only, " + "quantum=%u) -- osGetCount() is now frame-locked, not wall-clock", + g_determQuantum); + if (g_determTraceEnabled) { + sysLogPrintf(LOG_NOTE, "DETERMTRACE seq=0 ticks=%u event=seed", g_determTicks); + } } pthread_t th; @@ -577,6 +651,42 @@ s32 osRecvMesg(OSMesgQueue *mq, OSMesg *msg, s32 flag) pthread_mutex_unlock(&pq->lock); return -1; } + /* GE_DETERM (§D117, M-52 2nd attempt): __scMain (sched.c) is the + * SOLE consumer of g_viRetraceMQ, and everything the rest of the + * boot/frame chain waits on (task execution via osSpTaskStartGo, + * joyPoll() -- which is also what unblocks joyCheckStatusThreadSafe + * -- and the forward to mainThread's gfxFrameMsgQ) happens + * synchronously inside __scMain's handling of THAT message, on this + * same thread, before it comes back to ask for the next one. So + * "the consumer is asking and none exists" is itself the correct, + * purely call-sequenced trigger for the next virtual retrace -- + * driven by program order, not by an independent thread polling on + * a wall-clock cadence (the M-52 first attempt's mistake: that + * poll's timing varied run-to-run and leaked into the tick count). + * Enqueue inline (not via osEnqueueMesg -- would re-lock pq->lock, + * which we already hold). */ + if (g_determEnabled && mq == g_viRetraceMQ) { + /* Still synthesize the message every time (keeps __scMain/ + * joyPoll unblocked -- no deadlock), but only advance the clock + * up to a fixed cap before the first real task runs. See the + * g_determTaskEverRun comment above. */ + if (g_determTaskEverRun || g_determPreTaskTicksGranted < 1) { + g_determTicks += g_determQuantum; + if (!g_determTaskEverRun) ++g_determPreTaskTicksGranted; + if (g_determTraceEnabled) { + sysLogPrintf(LOG_NOTE, + "DETERMTRACE seq=%u ticks=%u event=advance taskEverRun=%d", + ++g_determTraceSeq, g_determTicks, g_determTaskEverRun); + } + } else if (g_determTraceEnabled) { + sysLogPrintf(LOG_NOTE, + "DETERMTRACE seq=%u ticks=%u event=capped taskEverRun=%d", + ++g_determTraceSeq, g_determTicks, g_determTaskEverRun); + } + mq->msg[(mq->first + mq->validCount) % mq->msgCount] = g_viRetraceMsg; + ++mq->validCount; + break; + } /* Log each queue's first blocking call site (the return address is * the caller); symbolicate with addr2line to find where in game code * this wait happens. */ @@ -1118,6 +1228,12 @@ void osSpTaskLoad(OSTask *t) { (void)t; /* nothing to load on the host */ } void osSpTaskStartGo(OSTask *t) { + if (g_determTraceEnabled && !g_determTaskEverRun) { + sysLogPrintf(LOG_NOTE, + "DETERMTRACE seq=%u ticks=%u event=first_task_run tasktype=%d", + ++g_determTraceSeq, g_determTicks, (int)t->t.type); + } + g_determTaskEverRun = 1; /* GE_DETERM: §D117 M-52 3rd attempt */ if (t->t.type == M_AUDTASK) { /* Phase 3: execute the audio ucode against t->t.data_ptr (an Acmd * list). For now the task simply completes; libaudio's amMain still diff --git a/tools_pc/README.md b/tools_pc/README.md new file mode 100644 index 00000000..b348de09 --- /dev/null +++ b/tools_pc/README.md @@ -0,0 +1,75 @@ +# `tools_pc/` — PC-port dev & test scripts + +Index for the ~60 scripts here (speed-ups plan Step 4 / N2). None of these are +part of the build; they are converters, verification tooling, and +investigation artifacts. + +**Marker:** +- **`living`** — kept and re-run; part of the verification ritual, the release + flow, or a converter the build depends on (via the sidecars it emits). +- **`artifact`** — added to chase one finding. Safe to move to + `tools_pc/archive//` once that finding is closed and the technique is + captured in `docs/dev/findings.md` / `docs/porting-notes.md`. + +## Living — converters the sidecars depend on + +| Script | Role | +|---|---| +| `d43_emit.py` | Offline N64→PC model-file converter → RZ sidecar + `manifest.csv` (D50 / Plan B). Regen after any model-format change. | +| `d43_convert.py` | Reference single-file model converter + full-512 layout/pointer validator (D43). | +| `d69_emit.py` | Offline converter for stage `bg/*.seg` + `Tbg_*_stanZ` → concatenated sidecar (D69/D78–D82). | +| `d88_emit.py` | Offline converter for per-level `Usetup*Z` stage-setup files → appended to the `pccg.bin` sidecar (D88). | +| `d88_propdefs.py` | The `propDefs` polymorphic-record stream N64→PC converter used by `d88_emit.py` (D88.4). | +| `d69_emit.py` / `d88_emit.py` / `d43_emit.py` | Run all three (+ `d88_propdefs`) to rebuild `data/` sidecars from the ROM. | + +## Living — verification & release + +| Script | Role | +|---|---| +| `verify.sh` | Speed-ups plan Step 2. One command → one verdict: build-if-stale, headless `GE_PCDUMP`/`GE_INPUTSCRIPT` capture, crash-detect + symbolication (via `crash_brief.py`), `pixcount`, `framediff` vs golden, `--json`. `verify.sh `, `verify.sh sweep [subset]`, `verify.sh parity --against `. | +| `crash_brief.py` | Speed-ups plan Step 9 / R5. `ge007.crash.log` → symbolicated frames + a dev-process.md-shaped dispatch-ready brief: matches porting-notes.md sections / findings-index.csv entries by resolved function/file, flags known parked-crash signatures (Cuba credits, D188 va_list, D189 stack overrun) so agents stop re-investigating them. Called automatically by `verify.sh` on CRASH; also runnable standalone. | +| `framediff.py` | Visual regression: candidate `GE_PCDUMP` frames vs `tools_pc/golden/`, per-region divergence. Structural/tolerant by default; `--exact` after `GE_DETERM`. | +| `pixcount.py` | Count non-black pixels in a PPM dump — "did the scene render anything" as a number. | +| `level_sweep.sh` | Bare `-level_XX` boot of all 21 solo levels → PASS / NO-FRAMES / CRASH. Predates `verify.sh sweep`; kept as the battle-tested full-21-level runner until `verify.sh sweep` has done an equivalent soak. | +| `playtest.sh` | Launch a level for `docs/dev/LEVEL-PLAYTEST.md` human validation (WS6). | +| `debug.ps1` / `repro_gdb.sh` / `attach_animgen.sh` | Launch (or attach to) the game under gdb so a crash always leaves a backtrace. | +| `bundle-win.sh` / `bundle-linux.sh` | Package a built tree as a distributable archive (exe + licenses + `prepare-assets/`). | +| `romverify.c` | One-shot `.z64` integrity check against the repo's ground truths. | +| `dump_objectives.py` | Dump per-level objectives + win/fail criteria from the ROM (playtest validation; Step 10 / N8 acceptance oracle). | +| `disasm.py` | Minimal MIPS disassembler for the BE ROM (RAM-addr → file offset). | +| `ppm2bmp.py` | PPM → 24-bit BMP, no deps — eyeball `GE_PCDUMP` frames without PIL. | +| `gen_findings_index.py` | Regenerate `docs/dev/findings-index.csv` (grep-before-you-read aid for the 200 KB finding log). `--check` in CI-style use. | +| `gen_env_probes.py` | Drift check for `docs/dev/GE-ENV-PROBES.md` — re-greps live `getenv("GE_*")` sites, reports NEW/GONE. | + +## Living — compiler-verified layout probes (kept: re-run when structs change) + +| Script | Role | +|---|---| +| `d43_layoutprobe.c` | Prints `sizeof`/`offsetof` for every struct `d43_emit.py` must reproduce, built with the port toolchain. | +| `d88_layoutprobe.c` | Same, for the `PROPDEF_*` record structs `d88_emit.py` converts. | +| `mtxtest.c` | Standalone numerical test of the fast3d matrix pipeline conventions (GE swapped-perspective, packing, MUL order). | + +## Investigation artifacts — D43 model-file format (finding closed; archive candidates) + +`d43_chainbound.py`, `d43_cover.py`, `d43_decode.py`, `d43_fullwalk.py`, +`d43_gdldump.py`, `d43_gdlhist.py`, `d43_gdlorder.py`, `d43_gdlseq.py`, +`d43_invariants.py`, `d43_layout.py`, `d43_layout1.py`, `d43_lutscan.py`, +`d43_pointusage.py`, `d43_seg5.py`, `d43_seg5ops.py`, `d43_seg5vtx.py`, +`d43_seg5vtx2.py`, `d43_sizes.py`, `d43_sizes2.py`, `d43_tree_dump.py`, +`d43_vtxfmt.py`, `d43_walk.py` — layout/pointer/GDL-order analysis passes that +derived the D43 converter spec. `d43_sizes.py` is still `exec()`'d by a couple +of the others. + +## Investigation artifacts — other closed/parked findings + +| Script | Finding | +|---|---| +| `d125_check.py` | D125 — is the emitted `propDefs` blob byte-identical to `d88_propdefs.convert_stream()`? | +| `d88_propdef_scan.py` | D88.4 — histogram of `PROPDEF_*` record types across shipped levels. | +| `d51_gdb.py` / `dump_animgen.cmd` | D51 texture-tile crash capture / Facility-outro-hang anim-state dump (gdb scripts). | + +## Other + +- `dist/` — staged bundle README template (`README.md.in`) + tokens. +- `golden/` — per-level golden frames for `framediff.py` (Step 3 fills this out). +- `__pycache__/` — ignored. diff --git a/tools_pc/crash_brief.py b/tools_pc/crash_brief.py new file mode 100644 index 00000000..ac2a2dc2 --- /dev/null +++ b/tools_pc/crash_brief.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""crash_brief.py — crash log -> pre-filled investigation brief (speed-ups +plan Step 9 / R5). + +Symbolicates every faulting PC in a ge007.crash.log with addr2line, then +greps docs/porting-notes.md and docs/dev/findings-index.csv (Step 4/R8) for +sections that already cover that function/file, and checks a small known +parked-crash signature list (Cuba credits, D188 SysV va_list, D189 +stack-protector) so agents stop re-investigating crashes that are already +understood-but-not-worth-fixing. Prints a dev-process.md-shaped brief ready +to hand to a subagent, filling in what's derivable and leaving the rest as +placeholders. + +Usage: + crash_brief.py [--crash-log PATH] [--exe PATH] [--platform win|linux] + [--level NAME] [--json] + +Defaults: --crash-log ge007.crash.log (repo root), --exe autodetected from +--platform (build-pc/ge007.x86_64.exe or build-linux/ge007.x86_64). +""" +import argparse +import json +import os +import re +import shutil +import subprocess +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +_ADDR2LINE = None + + +def find_addr2line(): + """shutil.which() alone is not enough here: this repo's PATH convention + for the win toolchain is the MSYS2 POSIX form (/c/msys64/mingw64/bin), + which a native Windows Python (`python`, as opposed to an MSYS-flavored + `python3`) cannot resolve -- os.path.isfile("/c/...") is not a valid + Windows path, so the PATH search silently finds nothing and every + address comes back unresolved. Fall back to the documented install + location before giving up.""" + global _ADDR2LINE + if _ADDR2LINE: + return _ADDR2LINE + p = shutil.which("addr2line") + if not p: + for c in (r"C:\msys64\mingw64\bin\addr2line.exe", + r"C:\msys64\usr\bin\addr2line.exe", + "/usr/bin/addr2line"): + if os.path.exists(c): + p = c + break + _ADDR2LINE = p or "addr2line" + return _ADDR2LINE + +# -------------------------------------------------------------------------- +# Known parked-crash signatures. Each entry: (regex over the raw crash-log +# text or a resolved symbol, label, one-line note). Matching one of these +# means "this is understood, don't re-derive it — see the label". +KNOWN_SIGNATURES = [ + (re.compile(r"credits", re.I), "D129/D76", + "Cuba end-credits crash -- parked, not a fresh bug."), + (re.compile(r"_Printf|_Putfld|va_list", re.I), "D188 (porting-notes D7)", + "va_list-by-value ABI misuse on x86-64 SysV -- thread a va_list* instead."), + (re.compile(r"tileStack|pointbuf|stack.protector|stack.smash", re.I), + "D189 (porting-notes D8)", + "latent fixed-size stack-buffer over/under-run, fatal only under " + "-fstack-protector-strong (Linux/Ubuntu default)."), +] + +# Known parked crashes by resolved symbol -- a regex over the crash-log text +# alone misses this class: the tell (a local var like tileStack[39]) lives +# inside the function body, not its symbol name, so the KNOWN_SIGNATURES +# text-regex never fires even though the crash *is* the named finding. +FUNC_LABELS = { + "sub_GAME_7F0B1DDC": ("D189 (porting-notes D8)", + "stan.c tileStack[39] overrun -- the exact D189 repro function."), + "add_ptr_to_objective": ("D190", + "objective.c:64 propDef-stride family, Linux -level_45 SIGSEGV at load."), +} + +BACKTRACE_LINE = re.compile(r"^#\d+:\s*(0x[0-9a-fA-F]+)") +PC_LINE = re.compile(r"^PC:\s*(0x[0-9a-fA-F]+)") + + +def find_addresses(text, max_frames=8): + """Pull the crash PC + up to max_frames backtrace addresses, in order, + deduplicated. Deliberately generic across the Windows (EBP-chain) and + Linux (backtrace_symbols) crash.c formats -- both print '#NN: 0xADDR...' + backtrace lines and a leading 'PC: 0xADDR' line (see port/src/crash.c).""" + addrs = [] + for line in text.splitlines(): + m = PC_LINE.match(line) or BACKTRACE_LINE.match(line) + if m and m.group(1) not in addrs: + addrs.append(m.group(1)) + if len(addrs) >= max_frames: + break + return addrs + + +def symbolicate(exe, addr): + """addr2line -f -C -> (function, file:line), or (None, None) if it + can't resolve (stripped exe, address outside any module, ...).""" + try: + out = subprocess.run([find_addr2line(), "-e", exe, "-f", "-C", addr], + capture_output=True, text=True, timeout=10) + except (OSError, subprocess.TimeoutExpired): + return None, None + lines = out.stdout.strip().splitlines() + if len(lines) < 2: + return None, None + func, loc = lines[0], lines[1] + if func in ("??", "") or loc.startswith("??"): + return (func if func not in ("??", "") else None, + None if loc.startswith("??") else loc) + return func, loc + + +def load_porting_notes_sections(): + """[(header_text, anchor, body_text), ...] from the '## ' sections.""" + path = os.path.join(ROOT, "docs", "porting-notes.md") + if not os.path.exists(path): + return [] + text = open(path, encoding="utf-8").read() + parts = re.split(r"^(## .+)$", text, flags=re.M) + sections = [] + for i in range(1, len(parts), 2): + header = parts[i][3:].strip() + body = parts[i + 1] if i + 1 < len(parts) else "" + sections.append((header, body)) + return sections + + +def load_findings_index(): + path = os.path.join(ROOT, "docs", "dev", "findings-index.csv") + rows = [] + if not os.path.exists(path): + return rows + import csv + with open(path, encoding="utf-8") as f: + for row in csv.DictReader(l for l in f if not l.startswith("#")): + rows.append(row) + return rows + + +def keyword_hits(func, fileloc, sections, findings, cap=6): + """Grep porting-notes sections + findings-index one-liners for the + resolved function name / file basename. + + Word-boundary match, not substring: a short file basename like "stan" + (stan.c) is a substring of "distance"/"instant"/"understand" and a naive + `needle in haystack` test matched nearly every finding in the log -- + caught live testing this script. Also drops needles under 5 chars (too + common to be distinctive) and caps the result so a hit list is a lead, + not the whole findings log pasted into the brief.""" + needles = [] + if func: + # split CamelCase / snake_case into tokens; keep the full name too + needles += re.split(r"[_A-Z]", func) + needles.append(func) + if fileloc: + # rsplit, not split: a Windows debug path is "C:/.../stan.c:2163" -- + # a plain split(":")[0] grabs the drive letter "C", not the path + # (caught live-testing this script). + base = os.path.basename(fileloc.rsplit(":", 1)[0]) + needles.append(os.path.splitext(base)[0]) + needles = sorted({n.lower() for n in needles if len(n) >= 5}) + patterns = [re.compile(r"\b%s\b" % re.escape(n)) for n in needles] + if not patterns: + return [], [] + + porting_hits, finding_hits = [], [] + for header, body in sections: + blob = (header + " " + body).lower() + if any(p.search(blob) for p in patterns): + porting_hits.append(header) + for row in findings: + blob = (row.get("one_liner", "") + " " + row.get("label", "")).lower() + if any(p.search(blob) for p in patterns): + finding_hits.append("%s (%s)" % (row["label"], row["one_liner"])) + return porting_hits[:cap], finding_hits[:cap] + + +def check_known_signatures(crash_text, frames): + hits = [] + for f in frames: + func = f.get("func") + if func in FUNC_LABELS: + hits.append(FUNC_LABELS[func]) + haystack = crash_text + " " + " ".join(f.get("func") or "" for f in frames) + for pattern, label, note in KNOWN_SIGNATURES: + if pattern.search(haystack): + hits.append((label, note)) + # dedupe, keep order + seen = set() + out = [] + for h in hits: + if h not in seen: + seen.add(h) + out.append(h) + return out + + +def render_brief(args, frames, known_hits, porting_hits, finding_hits): + top = frames[0] if frames else {} + task_sym = top.get("func") or "(unresolved)" + task_loc = top.get("loc") or top.get("addr", "?") + + lines = [] + lines.append("TASK: crash in %s (%s)%s." + % (task_sym, task_loc, + " level=%s" % args.level if args.level else "")) + read_first = ["docs/porting-notes.md"] + if porting_hits: + read_first.append("§" + ", §".join(dict.fromkeys(porting_hits))) + if finding_hits: + read_first.append("docs/dev/findings.md: " + + "; ".join(dict.fromkeys(finding_hits))) + lines.append("READ FIRST: " + "; ".join(read_first) + ".") + lines.append("FILES YOU MAY TOUCH: ." + % (os.path.basename(task_loc.rsplit(":", 1)[0]) + if task_loc != "?" else "")) + if known_hits: + lines.append("KNOWN-GOOD / RULED OUT: matches a PARKED signature -- " + + "; ".join("%s: %s" % (l, n) for l, n in known_hits) + + " -- confirm it's the same instance before spending " + "a budget re-deriving a known crash.") + else: + lines.append("KNOWN-GOOD / RULED OUT: .") + lines.append("PRE-FLIGHT ATTACHED: %s (%d frame%s symbolicated)." + % (args.crash_log, len(frames), "" if len(frames) == 1 else "s")) + for i, f in enumerate(frames): + lines.append(" #%02d %s %s %s" % (i, f["addr"], f.get("func") or "??", + f.get("loc") or "??")) + lines.append("BUDGET: . On expiry: revert probes, write up " + "with confidence.") + lines.append("CONSTRAINTS: no game-logic changes; ABI/layout/format only; " + "#ifdef PORT; documented in the finding log.") + verify = ("tools_pc/verify.sh %s" % args.level) if args.level else \ + "tools_pc/verify.sh (fill in which level reproduces this)" + lines.append("VERIFY: %s." % verify) + lines.append("REPORT: (a) root cause + file:line evidence (b) fix diff, " + "or why not (c) probes left in tree (d) confidence.\n" + " Append any generalisable quirk to " + "docs/porting-notes.md.") + return "\n".join(lines) + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--crash-log", default=os.path.join(ROOT, "ge007.crash.log")) + ap.add_argument("--exe") + ap.add_argument("--platform", choices=["win", "linux"]) + ap.add_argument("--level", help="level name/number this crash reproduces on " + "(fills VERIFY:)") + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv[1:]) + + # Content pulled in from findings.md/porting-notes.md (arrows, em-dashes, + # etc.) can contain non-cp1252 characters; native Windows Python's stdout + # defaults to the console codepage, not UTF-8, and raises on them. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + + if not os.path.exists(args.crash_log): + print("error: no crash log at %s" % args.crash_log, file=sys.stderr) + return 2 + + platform = args.platform or ("linux" if sys.platform.startswith("linux") else "win") + exe = args.exe or (os.path.join(ROOT, "build-linux", "ge007.x86_64") + if platform == "linux" + else os.path.join(ROOT, "build-pc", "ge007.x86_64.exe")) + if not os.path.exists(exe): + print("error: exe not found at %s (pass --exe)" % exe, file=sys.stderr) + return 2 + + crash_text = open(args.crash_log, encoding="utf-8", errors="replace").read() + addrs = find_addresses(crash_text) + if not addrs: + print("error: no PC/backtrace addresses found in %s" % args.crash_log, + file=sys.stderr) + return 2 + + frames = [] + for addr in addrs: + func, loc = symbolicate(exe, addr) + frames.append({"addr": addr, "func": func, "loc": loc}) + + sections = load_porting_notes_sections() + findings = load_findings_index() + porting_hits, finding_hits = [], [] + for f in frames: + p, fi = keyword_hits(f.get("func"), f.get("loc"), sections, findings) + porting_hits += p + finding_hits += fi + porting_hits = list(dict.fromkeys(porting_hits))[:6] + finding_hits = list(dict.fromkeys(finding_hits))[:6] + known_hits = check_known_signatures(crash_text, frames) + + brief = render_brief(args, frames, known_hits, porting_hits, finding_hits) + print(brief) + + if args.json: + print(json.dumps({ + "crash_log": args.crash_log, "exe": exe, "platform": platform, + "frames": frames, + "known_signature_matches": [{"label": l, "note": n} for l, n in known_hits], + "porting_notes_hits": list(dict.fromkeys(porting_hits)), + "findings_hits": list(dict.fromkeys(finding_hits)), + }, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tools_pc/gen_env_probes.py b/tools_pc/gen_env_probes.py new file mode 100644 index 00000000..0cc7eb9a --- /dev/null +++ b/tools_pc/gen_env_probes.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Drift check for docs/dev/GE-ENV-PROBES.md (speed-ups plan Step 4 / R9). + +That doc is a hand-curated table of every `GE_*` env probe -- var, file:line, +what it does, live/dead. It was last snapshotted by a manual grep and drifts +as probes are added/removed. This script does NOT regenerate the prose (the +curation is the value); it re-greps the live `getenv("GE_...")` sites and +reports: + * NEW -- a var with live sites that the doc never names + * GONE -- a var the doc names that has no live site any more + * a fresh var -> file:line site map (paste into the doc's File:line cells) + +Usage: + python tools_pc/gen_env_probes.py # print the drift report + python tools_pc/gen_env_probes.py --check # exit 1 if NEW or GONE +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DOC = ROOT / "docs" / "dev" / "GE-ENV-PROBES.md" +SCAN_DIRS = ["src", "port"] +GETENV_RE = re.compile(r'getenv\(\s*"(GE_[A-Z0-9_]+)"\s*\)') + + +def live_sites() -> dict[str, list[str]]: + sites: dict[str, list[str]] = {} + for d in SCAN_DIRS: + for path in sorted((ROOT / d).rglob("*")): + if path.suffix not in {".c", ".cpp", ".h", ".def", ".inc"}: + continue + try: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + continue + for n, line in enumerate(lines, 1): + for m in GETENV_RE.finditer(line): + rel = path.relative_to(ROOT).as_posix() + sites.setdefault(m.group(1), []).append(f"{rel}:{n}") + return sites + + +def doc_vars() -> tuple[set[str], set[str]]: + """Return (all vars named in the doc, vars whose row is an intentional tombstone).""" + if not DOC.exists(): + return set(), set() + named, tombstone = set(), set() + for line in DOC.read_text(encoding="utf-8").splitlines(): + vs = re.findall(r"GE_[A-Z0-9_]+", line) + named.update(vs) + if "tombstone" in line.lower(): + tombstone.update(vs) + return named, tombstone + + +def main() -> int: + check = "--check" in sys.argv[1:] + sites = live_sites() + documented, tombstoned = doc_vars() + live = set(sites) + + new = sorted(live - documented) + gone = sorted(v for v in documented - live if v not in ({"GE_DETERM"} | tombstoned)) + + out = [] + out.append(f"# GE_* probe drift vs {DOC.relative_to(ROOT).as_posix()}") + out.append(f"# {len(live)} vars with live sites, {len(documented)} named in the doc\n") + if new: + out.append("## NEW -- live sites, not in the doc (add a row):") + for v in new: + out.append(f" {v}") + for s in sites[v]: + out.append(f" {s}") + if gone: + out.append("\n## GONE -- named in the doc, no live site (mark removed):") + out += [f" {v}" for v in gone] + if not new and not gone: + out.append("doc is in sync with live getenv sites (site line numbers still drift -- see map below)") + + out.append("\n## full live site map:") + for v in sorted(sites): + out.append(f" {v}: " + ", ".join(sites[v])) + + print("\n".join(out)) + if check and (new or gone): + sys.stderr.write(f"\nGE-ENV-PROBES.md drift: {len(new)} new, {len(gone)} gone\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools_pc/gen_findings_index.py b/tools_pc/gen_findings_index.py new file mode 100644 index 00000000..724ced2e --- /dev/null +++ b/tools_pc/gen_findings_index.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Generate docs/dev/findings-index.csv from the §F/§H tables in findings.md. + +Agentic speed-ups plan Step 4 / R8: findings.md is ~200 KB. Agents should grep +this CSV (label, status, one-liner) to decide *which* Dxx entries to actually +read, instead of linear-reading the log. + +Heuristic parser: every markdown table row whose first cell matches a Dxx +label (`D12`, `D88.4`, `D75 · D77`, `D24-D30`, ...). The topic cell is +truncated to a one-liner; the status cell is normalised to a short bucket +(RESOLVED / OPEN / FIXED / PARTIAL / PROPOSED / ?) with the raw text kept. + +Usage: + python tools_pc/gen_findings_index.py # write the CSV + python tools_pc/gen_findings_index.py --check # exit 1 if stale + python tools_pc/gen_findings_index.py --stdout # print, don't write +""" +from __future__ import annotations + +import csv +import io +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "docs" / "dev" / "findings.md" +OUT = ROOT / "docs" / "dev" / "findings-index.csv" + +LABEL_RE = re.compile(r"^D\d+(?:\.\d+)?(?:[-–]D\d+)?(?:\s*[·,]\s*D\d+(?:\.\d+)?)*\+?$") + +STATUS_BUCKETS = [ + ("FIXED", re.compile(r"\bfixed\b", re.I)), + ("RESOLVED", re.compile(r"\bresolved\b", re.I)), + ("PARTIAL", re.compile(r"\bpartial|clamp landed|landed\b|safety-net", re.I)), + ("PROPOSED", re.compile(r"\bproposed|not applied|design\b", re.I)), + ("OPEN", re.compile(r"\bopen\b|not started|awaiting|owed|pending", re.I)), +] + + +def one_liner(cell: str, limit: int = 160) -> str: + """Strip markdown emphasis/code and collapse to a single short line.""" + txt = cell.replace("**", "").replace("`", "") + txt = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", txt) # [t](u) -> t + txt = re.sub(r"\s+", " ", txt).strip() + # first sentence-ish + m = re.match(r"(.{20,}?[.—:])\s", txt) + if m and len(m.group(1)) <= limit: + txt = m.group(1) + if len(txt) > limit: + txt = txt[: limit - 1].rstrip() + "…" + return txt + + +LEAD_RE = re.compile(r"^\**\s*(OPEN|RESOLVED|FIXED|PROPOSED|PARTIAL|LANDED|CLAMP|DISPROVEN)", re.I) + + +def bucket(status_cell: str) -> str: + # An explicit leading verdict token wins over prose keywords deeper in the + # cell (e.g. "OPEN (parked). ... D168 fixed ..." is OPEN, not FIXED). + lead = LEAD_RE.match(status_cell.strip()) + if lead: + tok = lead.group(1).upper() + return {"LANDED": "PARTIAL", "CLAMP": "PARTIAL", "DISPROVEN": "RESOLVED"}.get(tok, tok) + for name, rx in STATUS_BUCKETS: + if rx.search(status_cell): + return name + return "?" + + +def split_row(line: str) -> list[str] | None: + line = line.strip() + if not line.startswith("|"): + return None + # split on '|' that is not inside backticks + cells, buf, tick = [], [], False + for ch in line: + if ch == "`": + tick = not tick + buf.append(ch) + elif ch == "|" and not tick: + cells.append("".join(buf)) + buf = [] + else: + buf.append(ch) + cells.append("".join(buf)) + cells = [c.strip() for c in cells] + # leading/trailing empty from outer pipes + if cells and cells[0] == "": + cells = cells[1:] + if cells and cells[-1] == "": + cells = cells[:-1] + return cells or None + + +def rows(): + section = None + for raw in SRC.read_text(encoding="utf-8").splitlines(): + h = re.match(r"^#+\s+([A-Z])\.\s", raw) or re.match(r"^#+\s.*§?([FH])\b", raw) + if raw.startswith("### ") and re.search(r"\b([FH])\.", raw): + section = re.search(r"\b([FH])\.", raw).group(1) + cells = split_row(raw) + if not cells or len(cells) < 2: + continue + label = cells[0].replace("–", "-") + if not LABEL_RE.match(cells[0]): + continue + topic = cells[1] + status = cells[2] if len(cells) > 2 else "" + yield { + "label": label, + "bucket": bucket(status), + "one_liner": one_liner(topic), + "status_raw": one_liner(status, 200) if status else "", + } + + +def build_csv() -> str: + buf = io.StringIO() + w = csv.DictWriter( + buf, fieldnames=["label", "bucket", "one_liner", "status_raw"], lineterminator="\n" + ) + w.writeheader() + seen = set() + for r in rows(): + key = r["label"] + if key in seen: + continue + seen.add(key) + w.writerow(r) + return buf.getvalue() + + +def main() -> int: + args = set(sys.argv[1:]) + text = ( + "# GENERATED by tools_pc/gen_findings_index.py -- do not hand-edit.\n" + "# Grep this to pick which findings.md entries to read; it is not authoritative.\n" + + build_csv() + ) + if "--stdout" in args: + sys.stdout.write(text) + return 0 + want = text.encode("utf-8") + if "--check" in args: + cur = OUT.read_bytes() if OUT.exists() else b"" + if cur != want: + sys.stderr.write("findings-index.csv is stale: re-run gen_findings_index.py\n") + return 1 + return 0 + OUT.write_bytes(want) + print(f"wrote {OUT.relative_to(ROOT)} ({text.count(chr(10))} lines)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools_pc/verify.sh b/tools_pc/verify.sh new file mode 100644 index 00000000..b565f0a4 --- /dev/null +++ b/tools_pc/verify.sh @@ -0,0 +1,325 @@ +#!/bin/bash +# verify.sh — one command -> one machine-readable verdict (speed-ups plan Step 2). +# +# Wraps the manual recipe every session was re-deriving by hand (build, +# 640x480 ini pin, GE_PCDUMP capture, crash-symbolication, pixcount, +# framediff): see docs/dev/notes/AGENTIC-SPEEDUPS-PLAN.md Step 2 / N6. +# +# Usage: +# verify.sh single level: name (dam, bunker1, "surface 1") +# or -level_XX number (09, 33, ...) +# verify.sh sweep [subset] all 21 solo levels, or SWEEP_LEVELS-style +# "name:XX name:XX ..." subset +# verify.sh parity --against +# framediff this platform's capture against a +# capture directory from the other platform +# (there is no cross-OS launch here — feed it +# a directory of PPMs/PNGs pulled off that box) +# +# Options: +# --platform win|linux default: autodetect via `uname -s` +# --dump lo-hi:step GE_PCDUMP window (default: 200-440:120 single level, +# 80-400:40 sweep — matches the existing golden set +# and level_sweep.sh respectively) +# --script "..." GE_INPUTSCRIPT passthrough (scripted input) +# --json also emit one JSON object (or array, for sweep) to +# stdout: {level, platform, status, frames, +# worst_cell, crash_sym} +# --against DIR parity mode only: the other platform's capture dir +# +# Status values: PASS | CRASH | NO-FRAMES | STALLED | REGRESSION +# (STALLED is this tool's extension of the plan's 4-value enum — see +# level_sweep.sh's D117/D134 note: a level that renders a few frames then +# stops moving is a load-sensitivity flake, not a real pass or fail.) +# +# Known limitation (N6 / M-48): on --platform linux, GE_PCDUMP reads back +# black frames on llvmpipe/WSLg (a glReadPixels tooling bug, not a render +# bug — the live window renders fine). pixcount/framediff are therefore +# skipped on linux and the verdict is crash-detection only, flagged NOTE. +set -u +cd "$(dirname "$0")/.." +ROOT="$(pwd)" + +# --- arg parsing ------------------------------------------------------------- +MODE=""; LEVEL_ARG=""; PLATFORM=""; DUMP=""; SCRIPT=""; JSON=0; AGAINST="" +POSARGS=() +while [ $# -gt 0 ]; do + case "$1" in + --platform) PLATFORM="$2"; shift 2 ;; + --dump) DUMP="$2"; shift 2 ;; + --script) SCRIPT="$2"; shift 2 ;; + --against) AGAINST="$2"; shift 2 ;; + --json) JSON=1; shift ;; + -h|--help) sed -n '2,32p' "$0"; exit 0 ;; + *) POSARGS+=("$1"); shift ;; + esac +done +set -- "${POSARGS[@]:-}" + +case "${1:-}" in + sweep) MODE=sweep; shift; SWEEP_SUBSET="${*:-}" ;; + parity) MODE=parity; LEVEL_ARG="${2:-}"; shift 2 2>/dev/null || true ;; + ""|-h) sed -n '2,32p' "$0"; exit 0 ;; + *) MODE=single; LEVEL_ARG="$1" ;; +esac + +if [ -z "$PLATFORM" ]; then + case "$(uname -s)" in + Linux*) PLATFORM=linux ;; + *) PLATFORM=win ;; + esac +fi + +if [ "$PLATFORM" = win ]; then + BUILD_DIR=build-pc + EXE="$ROOT/build-pc/ge007.x86_64.exe" + export PATH="/c/msys64/mingw64/bin:$PATH" # addr2line + runtime DLLs + KILL() { taskkill //F //IM ge007.x86_64.exe >/dev/null 2>&1; } +else + BUILD_DIR=build-linux + EXE="$ROOT/build-linux/ge007.x86_64" + KILL() { pkill -f ge007.x86_64 >/dev/null 2>&1; } +fi + +# name -> -level_XX (mission order; matches playtest.sh / level_sweep.sh) +LEVELS=( + "dam:33" "facility:34" "runway:35" "surface1:36" "bunker1:09" "silo:20" + "frigate:26" "surface2:43" "bunker2:27" "statue:22" "archives:24" + "streets:29" "depot:30" "train:25" "jungle:37" "control:23" "caverns:39" + "cradle:41" "aztec:28" "egypt:32" "cuba:54" +) +norm() { echo "$1" | tr 'A-Z' 'a-z' | tr -d ' _-'; } +resolve_level() { + local arg="$1" + if [[ "$arg" =~ ^[0-9]+$ ]]; then echo "num:$(printf '%02d' "$arg")"; return; fi + local key; key=$(norm "$arg") + for e in "${LEVELS[@]}"; do + [ "$(norm "${e%%:*}")" = "$key" ] && { echo "${e%%:*}:${e##*:}"; return; } + done + echo "" +} +name_for_num() { + local num="$1" + for e in "${LEVELS[@]}"; do [ "${e##*:}" = "$num" ] && { echo "${e%%:*}"; return; }; done + echo "level_$num" +} + +# --- build-if-stale ------------------------------------------------------ +if [ ! -d "$BUILD_DIR" ]; then + echo "error: $BUILD_DIR missing — configure it first (./build-pc.sh for win;" >&2 + echo " cmake -S . -B build-linux -G Ninja -DROMID=ntsc-final for linux)" >&2 + exit 2 +fi +BUILD_LOG=$(mktemp) +if ! cmake --build "$BUILD_DIR" -j 8 >"$BUILD_LOG" 2>&1; then + echo "BUILD FAILED" + cat "$BUILD_LOG" >&2 + rm -f "$BUILD_LOG" + exit 2 +fi +rm -f "$BUILD_LOG" +if [ ! -x "$EXE" ] && [ ! -f "$EXE" ]; then + echo "error: build produced no $EXE" >&2 + exit 2 +fi + +# --- golden lookup: tools_pc/golden///, else the flat +# level_09 set (today's only golden — Step 3 fills the rest in) ---------- +golden_dir_for() { + local name="$1" + local per="$ROOT/tools_pc/golden/$name/$PLATFORM" + if [ -d "$per" ] && ls "$per"/*.png >/dev/null 2>&1; then echo "$per"; return; fi + if [ "$name" = "bunker1" ] && ls "$ROOT/tools_pc/golden"/*.png >/dev/null 2>&1; then + echo "$ROOT/tools_pc/golden"; return + fi + echo "" +} + +# --- ini pin: golden frames are captured at 640x480; back up + restore ----- +INI="$ROOT/data/ge007.ini" +INI_BAK="" +pin_ini_640x480() { + [ -f "$INI" ] || return + INI_BAK=$(mktemp) + cp "$INI" "$INI_BAK" + python3 - "$INI" <<'EOF' +import re, sys +p = sys.argv[1] +s = open(p, encoding="utf-8").read() +s = re.sub(r'(\[Window\]\s*\nWidth = )\d+', r'\g<1>640', s, count=1) +s = re.sub(r'(\[Window\]\s*\nWidth = 640\s*\nHeight = )\d+', r'\g<1>480', s, count=1) +open(p, "w", encoding="utf-8").write(s) +EOF +} +restore_ini() { [ -n "$INI_BAK" ] && cp "$INI_BAK" "$INI" && rm -f "$INI_BAK"; } +trap 'restore_ini; KILL' EXIT INT TERM + +# --- run one level -> sets globals: R_STATUS R_FRAMES R_SYM R_BRIEF ------- +run_one() { + local name="$1" num="$2" dump="$3" watchdog="${4:-90}" + local capdir; capdir=$(mktemp -d) + R_STATUS=""; R_FRAMES=0; R_SYM=""; R_BRIEF="" + + rm -f "$ROOT/ge007.crash.log" + KILL; sleep 1 + + ( cd "$ROOT" && export GE_PCDUMP="$dump" + [ -n "$SCRIPT" ] && export GE_INPUTSCRIPT="$SCRIPT" + timeout "$watchdog" "$EXE" "-level_$num" ) \ + >"$capdir/run.log" 2>&1 + mv "$ROOT"/ppm "$capdir/ppm" 2>/dev/null || mkdir -p "$capdir/ppm" + + R_FRAMES=$(ls "$capdir/ppm"/*.ppm 2>/dev/null | wc -l) + + if [ -f "$ROOT/ge007.crash.log" ]; then + R_STATUS=CRASH + cp "$ROOT/ge007.crash.log" "$capdir/crash.log" + # Step 9 (R5): crash_brief.py owns address extraction + symbolication + # (both platforms -- the old inline `0x1[0-9a-fA-F]{8,}` grep here + # assumed the Windows 0x140000000 image base and would never match a + # Linux no-PIE 0x20000000-based address) and prints a dispatch-ready + # brief with known-parked-signature and porting-notes/findings hits. + R_BRIEF=$(python "$ROOT/tools_pc/crash_brief.py" \ + --crash-log "$capdir/crash.log" --exe "$EXE" --platform "$PLATFORM" \ + --level "$name" --json 2>&1) + R_SYM=$(echo "$R_BRIEF" | python -c " +import json,sys +s=sys.stdin.read(); i=s.find('{') +try: + d,_=json.JSONDecoder().raw_decode(s[i:]) if i>=0 else (None,0) + f=d['frames'][0] + print('%s %s' % (f.get('func') or '??', f.get('loc') or '')) +except Exception: + print('') +" 2>/dev/null) + elif [ "$R_FRAMES" -eq 0 ]; then + R_STATUS=NO-FRAMES + else + R_STATUS=PASS + fi + CAPDIR="$capdir" +} + +# --- verdict emission -------------------------------------------------------- +emit_verdict() { + local name="$1" status="$2" frames="$3" worst="$4" sym="$5" note="${6:-}" + echo "$name: $status frames=$frames${worst:+ worst_cell=$worst}${sym:+ crash_sym=\"$sym\"}${note:+ ($note)}" + if [ "$JSON" = 1 ]; then + python3 - "$name" "$PLATFORM" "$status" "$frames" "$worst" "$sym" "$note" <<'EOF' +import json, sys +name, plat, status, frames, worst, sym, note = sys.argv[1:8] +obj = {"level": name, "platform": plat, "status": status, "frames": int(frames)} +if worst: obj["worst_cell"] = float(worst) +if sym: obj["crash_sym"] = sym +if note: obj["note"] = note +print(json.dumps(obj)) +EOF + fi +} + +verify_level() { + local name="$1" num="$2" dump="$3" watchdog="${4:-90}" + run_one "$name" "$num" "$dump" "$watchdog" + local status="$R_STATUS" frames="$R_FRAMES" sym="$R_SYM" worst="" note="" + + if [ "$status" = PASS ]; then + if [ "$PLATFORM" = linux ]; then + note="GE_PCDUMP reads black on llvmpipe/WSLg — pixcount/framediff skipped, crash-detect only" + else + local last; last=$(ls "$CAPDIR"/ppm/*.ppm 2>/dev/null | tail -1) + if [ -n "$last" ] && ! python3 "$ROOT/tools_pc/pixcount.py" "$last" >/dev/null 2>&1; then + status=NO-FRAMES; note="frames present but all-clear (pixcount 0)" + else + local gdir; gdir=$(golden_dir_for "$name") + if [ -n "$gdir" ]; then + local fdiff; fdiff=$(python3 "$ROOT/tools_pc/framediff.py" "$CAPDIR/ppm" --golden "$gdir" --json 2>&1) + local fexit=$? + # framediff --json sandwiches the JSON blob between human-readable + # per-frame lines and a summary line -- pull it out by scanning + # from the first '{' with a raw JSON decoder instead of assuming + # the whole stream parses. + worst=$(echo "$fdiff" | python3 -c " +import json,sys +s = sys.stdin.read() +i = s.find('{') +try: + d, _ = json.JSONDecoder().raw_decode(s[i:]) if i >= 0 else (None, 0) + print(max((r.get('worst_cell',0) for r in d['results']), default=0)) +except Exception: + print('') +" 2>/dev/null) + # exit 1 = a real per-frame threshold fail (pixel/cell/hash delta). + # exit 2 = a structural mismatch (missing/size) -- most commonly the + # golden's frame stems don't line up with this run's --dump window + # (today's only golden, level_09, was captured at 200-440:120; a + # sweep using a different stride has nothing to compare against + # rather than a real regression). Only exit 1 is a verdict fail. + case "$fexit" in + 1) status=REGRESSION; note="framediff vs $gdir" ;; + 2) note="golden frame stems don't match --dump window ($dump) for $gdir" ;; + esac + else + note="no golden for $name/$PLATFORM yet (Step 3)" + fi + fi + fi + fi + + local brief="$R_BRIEF" + rm -rf "$CAPDIR" + emit_verdict "$name" "$status" "$frames" "$worst" "$sym" "$note" + if [ "$status" = CRASH ] && [ -n "$brief" ]; then + # crash_brief.py's --json output sandwiches the JSON blob after the + # human-readable brief -- print only the brief part here. + echo "$brief" | awk '/^\{/{exit} {print}' + fi + case "$status" in PASS) return 0 ;; *) return 1 ;; esac +} + +# --- dispatch ---------------------------------------------------------------- +case "$MODE" in + single) + resolved=$(resolve_level "$LEVEL_ARG") + [ -z "$resolved" ] && { echo "unknown level '$LEVEL_ARG'" >&2; exit 2; } + name="${resolved%%:*}"; num="${resolved##*:}" + [ "${resolved%%:*}" = num ] && name=$(name_for_num "$num") + pin_ini_640x480 2>/dev/null || true + verify_level "$name" "$num" "${DUMP:-200-440:120}" + ;; + + sweep) + RESULTS_JSON=() + RC=0 + subset="${SWEEP_LEVELS:-${SWEEP_SUBSET:-}}" + entries=("${LEVELS[@]}") + [ -n "$subset" ] && read -ra entries <<< "$subset" + for e in "${entries[@]}"; do + n="${e%%:*}"; num="${e##*:}" + pin_ini_640x480 2>/dev/null || true + out=$(verify_level "$n" "$num" "${DUMP:-80-400:40}" 45) + rc=$? + [ "$rc" -ne 0 ] && RC=1 + echo "$out" | head -1 + done + echo "SWEEP DONE" + exit $RC + ;; + + parity) + [ -z "$LEVEL_ARG" ] && { echo "usage: verify.sh parity --against DIR" >&2; exit 2; } + [ -z "$AGAINST" ] && { echo "parity needs --against DIR (the other platform's capture)" >&2; exit 2; } + resolved=$(resolve_level "$LEVEL_ARG") + [ -z "$resolved" ] && { echo "unknown level '$LEVEL_ARG'" >&2; exit 2; } + name="${resolved%%:*}"; num="${resolved##*:}" + pin_ini_640x480 2>/dev/null || true + run_one "$name" "$num" "${DUMP:-200-440:120}" 90 + if [ "$R_STATUS" != PASS ]; then + emit_verdict "$name" "$R_STATUS" "$R_FRAMES" "" "$R_SYM" + rm -rf "$CAPDIR"; exit 1 + fi + python3 "$ROOT/tools_pc/framediff.py" "$CAPDIR/ppm" --golden "$AGAINST" --tol 2 --tol-pct 0.5 --exact ${JSON:+--json} + rc=$? + rm -rf "$CAPDIR" + exit $rc + ;; +esac