diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f03fb7a..bc3e7910 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,6 +113,12 @@ jobs: - name: Byte-compile Python tooling run: python3 -m compileall -q scripts tools_pc + # Generated dev-docs stay in sync with the tree (speed-ups plan Step 4). + - name: Check generated dev indexes are current + run: | + python3 tools_pc/gen_findings_index.py --check + python3 tools_pc/gen_env_probes.py --check + # US only. gen_romassets.py resolves every symbol for the US region from # the committed CSVs with no ROM present; PAL/JP each need a handful of # region-specific prop/font offsets that only come from that region's ROM diff --git a/CLAUDE.md b/CLAUDE.md index b2816cee..826d691c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,6 +54,9 @@ A subagent brief is INVALID and must not be sent unless it contains all of: write-up of a half-solved bug is a deliverable, not a failure. - **KNOWN-GOOD / RULED OUT**, **CONSTRAINTS** (no game-logic changes; ABI/layout/format only; `#ifdef PORT`; documented in §F), **REPORT** shape. +- **VERIFY** — the exact `tools_pc/verify.sh` / probe invocation that proves + the task done, or an explicit "runtime verification impossible here; a human + must run X". - Instruction to read `docs/porting-notes.md` first and append any generalisable quirk to it. diff --git a/docs/dev-process.md b/docs/dev-process.md index 79293c5c..35664b35 100644 --- a/docs/dev-process.md +++ b/docs/dev-process.md @@ -101,6 +101,7 @@ KNOWN-GOOD / RULED OUT: . PRE-FLIGHT ATTACHED: . BUDGET: run cycles / ~M min>. On expiry: revert probes, write up with confidence. CONSTRAINTS: no game-logic changes; ABI/layout/format only; #ifdef PORT; documented in the finding log. +VERIFY: . REPORT: (a) root cause + file:line evidence (b) fix diff, or why not (c) probes left in tree (d) confidence. Append any generalisable quirk to docs/porting-notes.md. diff --git a/docs/dev/GE-ENV-PROBES.md b/docs/dev/GE-ENV-PROBES.md index 9c645317..303aed69 100644 --- a/docs/dev/GE-ENV-PROBES.md +++ b/docs/dev/GE-ENV-PROBES.md @@ -1,7 +1,9 @@ # GE_* environment probes — consolidated list -Every `getenv("GE_…")` in the tree, from a grep of `src/` + `port/` on -2026-08-31 (master `31d036da`). All are **env-gated**: unset = zero behavior +Every `getenv("GE_…")` in the tree. The table prose is hand-curated; +**`tools_pc/gen_env_probes.py`** re-greps the live sites and reports drift +(NEW / GONE vars + a fresh file:line map) — run it before trusting the +File:line cells. Last reconciled 2026-09-04 (M-51). All are **env-gated**: unset = zero behavior change, so they are safe to leave in a release build (they only cost a `getenv` on the guarded path). "Dead" below means *the finding it was built for is closed* — the probe is strip-candidate scaffolding, not that it does @@ -22,6 +24,10 @@ Two classes: | `GE_INPUTLOG` | `port/src/input.c:329`, `port/src/config.c:53` (`configGetInputLog`) | Logs computed pad state (buttons + stick) each poll when non-zero. Also `[Debug] InputLog` ini. Also traces D165 `menuptr est/tgt/eff/stick`. | **live** | | `GE_STARTMENU` (+ `GE_STARTMENU_PAGE`, `GE_STARTMENU_DIFF`) | `src/game/lv.c:398-402` (`#ifdef PORT`) | Boot straight into a front-end menu id (13=MISSION_COMPLETE, 10=BRIEFING, 7=MISSION_SELECT, 12=MISSION_FAILED, 6=MODE_SELECT); `_PAGE`=folder row (def 1=Dam), `_DIFF`=0..3. Crash-test a screen fast. | **live** | | `GE_UNLOCK_ALL` | `src/game/debugmenu_handler.c:1098` (`get_debug_enable_all_levels_flag`, `#ifdef PORT`) | Mission-select shows every solo level (playtest jump-to-any-level aid). Cached in a `static int c`. | **live** | +| `GE_OPTIONSOVERLAY` | `port/src/optionsoverlay.c:231` | Force-enable the F10 port-layer options overlay (D184 / PR #10) regardless of ini. `atoi != 0`. | **live** | +| `GE_TEXPITCH` | `port/fast3d/gfx_pc.cpp:924` (`gfx_tex_pitch_fix`, cached) | `=0` disables the D183 defensive de-stride in `import_texture` (default on). RC2/RC3 texture-shear A/B knob. | **live** (RC3 not finished) | +| `GE_TEXRAW` | `port/fast3d/gfx_pc.cpp:1079` | With `GE_TEXDUMP`, also dumps the raw pre-import source bytes (`texdump/rNNN_f…_s…_WxH.bin`) so a decode bug can be told from a source-data bug offline (D183). | **live** (RC3 not finished) | +| `GE_WRAPFIX` | `port/fast3d/gfx_pc.cpp:3166` (`gfx_set_wrap_fix`) | RC3 test override for the `Video.WrapFix` D74/D167 wrap-block path (`atoi != 0` wins over the ini). | **live** (RC3 not finished) | | `GE_SAVELOG` | `src/game/file.c:46`, `src/game/file2.c:16` (`SAVELOG` macro), `src/boss.c:749`, `src/game/objective_status.c:297,343` | Traces the campaign save/unlock chain (objective completion → `end_of_mission_briefing` → `fileWriteSave` → EEPROM). Used to confirm D157. `g_savelogObjOnce` gates the per-criterion dump to the `bossReturnTitleStage` call. | **live** (strip once a few more level boundaries are playtested) | | `GE_D160` | `src/boss.c:744`, `src/game/bondview2.c:791`, `src/aicommands.def:10055,10181,10236,10242,10247` (all `#ifdef PORT`) | Dam exit-cutscene (D148/D160) diagnostic: trace points in `bossReturnTitleStage`, `bondviewSetCameraMode`, and AI cmds `EndLevel`/`exit_level`/`CameraLookAtBondFromPad`/`CameraSwitch`. **Investigation in progress** — user owes a `GE_D160=1` Dam run. | **live** | | `GE_DTEX` | `port/fast3d/gfx_pc.cpp:1001` | Per-`import_texture` param log (dims / line / size / lod / gen_mipmaps). Used to root-cause RC2 and D159. Kept as an inert diagnostic. | **live** (RC2/RC3 not finished) | @@ -33,7 +39,10 @@ Two classes: | Env var | File:line | Finding / what it logged | Status | |---|---|---|---| | `GE_D51` | `port/src/libultra.c:496`, `src/game/model.c:125,244,544` | msgQ 32-slot overflow watch / ModelSlot layout | dead (D51 closed) | -| `GE_D54` | — (all blocks stripped M-32, commit `49ce620a`) | music seq-table ABI / endianness (`ALMidiHdr`) | dead (D54 closed) — fully removed | +| `GE_D54` | — (all blocks stripped M-32, commit `49ce620a`) | music seq-table ABI / endianness (`ALMidiHdr`) | dead (D54 closed) — fully removed (kept here as a tombstone; the generator flags it GONE) | +| `GE_D154` | `src/game/bg.c:3418,3447,3632` (capped 64 calls) | bg room-GDL call trace (room / gdlidx / vtxoff / op / raw hdr words) | dead (D154 closed) | +| `GE_D176` | `src/game/bgfog.c:459,471`, `src/game/sky.c:325,381` (all `#ifdef PORT`) | D176 sky/fog env-match + sky-vert trace (Path B, PR #18) | **live** (D176(a)/(b) OPEN) | +| `GE_D178` | `src/game/front.c:6586,6594` (`#ifdef PORT`) | briefing-data u16 decode trace (blank-objectives, D143/D178) | dead (D178 closed) | | `GE_D56` | `src/game/model.c:227,830` | watch `Model` raw-offset reads | dead (D56 closed) | | `GE_D60` | `port/src/libultra.c:521,735` | gfx frame msgQ delivery trace | dead (D60 closed) | | `GE_D61` | `port/src/libultra.c:795` (`s_d61opened`) | opens a one-shot log file | dead (D61 closed) | diff --git a/docs/dev/findings-index.csv b/docs/dev/findings-index.csv new file mode 100644 index 00000000..b9f4f3b7 --- /dev/null +++ b/docs/dev/findings-index.csv @@ -0,0 +1,80 @@ +# GENERATED by tools_pc/gen_findings_index.py -- do not hand-edit. +# Grep this to pick which findings.md entries to read; it is not authoritative. +label,bucket,one_liner,status_raw +D24-D30,RESOLVED,green threads / kernel / crash handler,resolved +D31-D42,RESOLVED,"boot ABI: ANIM_DATA, music seq-table, libaudio banks, Globalimagetable rebase, GL context, rsp toggle",resolved +D43-D50,RESOLVED,"model-file loading → offline sidecar ""Plan B"" (d43_emit.py); D50.6 texCopyGdls full-slot copy",resolved +D51-D58,RESOLVED,"font pixeldata, osGetCount tick rate, model RW pools, cseq BE header, synth slots, RLE menu bg, watch raw offsets, gun-barrel DL",resolved +D59-D68,RESOLVED,"intro render: blood-RLE clobber, DMA validate, OSMesgQueue, HEADS/BODIES sentinels, romCopy width, image_entry, Globalimagetable BE→LE",resolved +D70-D74,RESOLVED,"intro-logo pixels: C-array bswap, UV path, sinf/cosf DVAL(), texture-import truncation",resolved +D75 · D77,OPEN,front-end 3D model transforms · audio,"OPEN (parked). M-32 triage (see §F ""D75 ADDENDUM""):" +D76 · D164,PROPOSED,disclaimer/legal screen only draws line 1 —,fix proposed (not applied — +D159,FIXED,"front-end wallet-Bond photo ""interlaced""/combed (RC1 / D149) —",FIXED (#ifdef PORT no-op the swap in image.c) +D161,FIXED,Depot (-level_30) ceiling = bright-blue speckle + radial rays (B2).,FIXED (port/fast3d/gfx_pc.cpp) +D165 · D166,FIXED,"input-layer polish (M-31, port-only, port/src/input.c):",FIXED +D180 · D181,PARTIAL,"native-PC input pass (QoL run, qol/native-pc-input-menu):","LANDED, config-gated, feel-checks owed" +D186,PARTIAL,"Video.FpsCap throttles the sim, not just presentation (M-39 diag, M-42 clamp).","CLAMP LANDED (port/fast3d/gfx_sdl2.cpp, port/src/video.c); real fix owed" +D187,FIXED,"HUD ammo-type/weapon icon vanished at -O2 (M-46 root-cause, M-47 fix + verify).","FIXED (src/game/gunfire.c, #ifdef AVOID_UB); PR #19" +D188,FIXED,Linux -O2 boot crash:,"FIXED (src/libultrare/libc/xprintf.c, src/sprintf.c, #ifdef PORT); PR #22" +D189,FIXED,Linux -fstack-protector aborts on the decomp's latent stack-buffer overruns (M-48) —,"FIXED (CMakeLists.txt, src/game/stan.c #ifdef PORT); PR #23" +D190,OPEN,"Linux -level_45 SIGSEGV at load in add_ptr_to_objective (objective.c:64) (M-48, open).",OPEN — not investigated (M-48) +D191,OPEN,Windows -O2 release-bundle crash:,OPEN — not investigated (M-49). +D193,OPEN,NPC AI locomotion is too slow —,"OPEN — root-cause hypothesis (timing family), not investigated (M-49)." +D194,OPEN,Mouse input needs another tuning pass —,"OPEN — root-caused, not fixed (M-49)" +D196,OPEN,"OS mouse cursor stays visible after closing the F10 options overlay in click-to-lock mode (M-49, user playtest).","OPEN — root-caused, 1-line fix identified, not applied (M-49)" +D192,OPEN,Front-end mission/level-select grid pointer still can't reach the outer cells —,"OPEN — root-caused, not fixed (M-49)" +D152+,FIXED,D152 addendum (M-31):,"FIXED (src/snd.c, port/src/libultra.c, #ifdef PORT) —" +D168,FIXED,GE_PCDUMP / F12 PPM captures were vertically flipped —,FIXED (port/fast3d/gfx_opengl.cpp); D114 / D116 reclassified as capture-orientation artifacts (below). +D169,FIXED,Front-end mouse pointer can't reach the outer cells of the mission/level-select grid —,"FIXED (M-33, port/src/input.c, port-only, no #ifdef PORT)." +D178,FIXED,Pre-mission briefing screen:,FIXED (M-36) — high confidence +D175,?,In-game stutter / brief hang during normal play (user QA report):,"Observed, not investigated" +D176,OPEN,"Surface exterior renders wrong (-level_36), two independent defects.","(a) root-caused, fix owed / (b) open" +D177,FIXED,Ladders non-functional —,FIXED (M-36) — high confidence; interactive climb test owed +D174,?,"""No blood effect"" (user QA report).",Observed; first step = BUNKER1 firefight with regenerated sidecars +D183,OPEN,"M-36 Family A (""texture line/pitch shear"") is DISPROVEN for the Surface repro (-level_36).",Hypothesis DISPROVEN + diagnostics shipped; D176(b)/D182(2) still OPEN +D173,?,Third-person Bond model spawns too high (user QA report):,"Observed, not investigated" +D179,FIXED,Packaged build crashes after the logos —,"Diagnosed, not fixed —" +D180,PARTIAL,"Native-PC input pass (M-XX QoL run, qol/native-pc-input-menu, port-only).","LANDED, port-only, no #ifdef PORT." +D181,PARTIAL,Game.ScreenShakeIntensity —,"LANDED behind config, default = original." +D184,PARTIAL,F10 port-layer options overlay —,"LANDED, port-only, no #ifdef PORT in src/." +D85,OPEN,room primary/secondary DL → texLoadFromGdl garbage,"OPEN (safety-netted; widen/pool fixes landed, geometry now renders)" +D86 · D87,RESOLVED,modelInitRwData truncated ptr · attract-demo BE ramromfilestructure,resolved +D88.4,RESOLVED,propDefs polymorphic record stream not byteswapped → setupDoor crash,resolved (d88_propdefs.py); layout audit M-20 → D132 +D132,PROPOSED,D88 propDefs layout audit (M-20):,proposed fix (not applied) — +D157,FIXED,"Campaign never unlocks the next level (M-30, user bug —","FIXED (src/bondtypes.h, #ifdef PORT —" +D160,?,Dam level-end cutscene (Bond off the dam / bungee) never plays —,"DIAGNOSTIC SHIPPED, not root-caused (c95713f5, #ifdef PORT + GE_D160=1):" +D89-D92,RESOLVED,"stage-load→frame: stan zero-fill overrun, portal address trunc, chr/AI spawn ptrs",resolved +D93-D102,RESOLVED,struct-pun / hardcoded-size pass:,resolved (D95 pool bump partial) +D103-D107,RESOLVED,"BUNKER1 viewport height, depth-buffer clear (G_CLEAR_DEPTH_EXT), portal near-plane, LOD mip tile",resolved +D108-D112,RESOLVED,skeletal models: d43_emit.py put_f32 byte-reversal bug,resolved +D113-D116,FIXED,portal BFS ok · matrix chain ok · player raw-offset audit + gunfire.c THROW* · HUD text X-mirror,"D114/D116 CLOSED — NOT A BUG (M-33, see D168):" +D117,RESOLVED,frame nondeterminism root-caused; GE_DETERM deferred; framediff.py structural,resolved (mode deferred) +D118,RESOLVED,SDL input layer (port/src/input.c); M-24 mouse-look rework:,resolved (D118a residual) +D150,FIXED,watch OBJECTIVES / BRIEF page crash:,FIXED (src/str.c #ifdef PORT); interactive re-verify pending +D137,FIXED,right-mouse (aim-sight) crash:,FIXED (src/game/gunfire.c #ifdef PORT — +D119 · D120,FIXED,guard-attack weapons_held[]->chr pun crash · blood-stain PointUsage[] chain hang,D119 fixed · D120 FIXED (M-30) +D120,FIXED,Blood-decal PointUsage[] chain —,FIXED (tools_pc/d43_emit.py). +D121,RESOLVED,WS1 frictionless per-level boot:,resolved +D122,FIXED,per-level prop/item model-load crash (Dam/Facility/Runway modelLoad/modelInitRwData):,converter fixed; residual chr/fast3d crashes on those levels are separate +D123,FIXED,crash class C1: chrIsNotDeadOrShot NULL deref on 6 levels (Dam/Runway/Frigate/Statue/Streets/Cradle).,"converter fixed (OBJ_ID_WORDS); C1 cleared on all 6, residual crashes are fast3d (C2)" +D125,RESOLVED,crash classes C3+C6 (Aztec/Bunker2/Surface2):,resolved (M-16b) — line 374 → 8 NULs; all 21 sidecars regen'd; Aztec -level_28 now PASSES (was C3 CRASH). +D126,RESOLVED,"crash classes C3r/C4/C6 (Bunker2 -level_27 door7F054FB4 propobj.c:13523, Depot -level_30 prop.c:902, Surface2 -level_43 loadobjectmodel.c:393):",resolved — d88_propdefs.py PROPDEF_PC_BYTES[30/32/33/35]=24 + typed handler; loadobjectmodel.c sizepropdef PORT returns 6. +D124,FIXED,crash class C2: fast3d bad texture pointer.,Jungle fixed (port/src/gimgfixup.c); Facility → D130 +D131,RESOLVED,crash class C2m (Jungle -level_37):,"resolved — seg_addr() restores the module high word for any fallthrough w1 in [0x40000000, 0x70000000) (module fixed-based at 0x140000000, no ASLR; DRAM/KSEG0/segmented/phys all handled earlier)." +D133,?,intro render triage (M-20):,"not a regression — parked (D75/D76, GRAPHICS-BACKLOG.md)" +D134,FIXED,"Frame-2 boot hang / sweep ""NO-FRAMES"" flakiness —",FIXED (port/src/libultra.c) +D135,FIXED,"Firefight crash — bgTestHitOnObj (propobj.c:8446) is an unported N64 GBI parser (M-22, WS6 playtest).","FIXED (src/game/propobj.c, #ifdef PORT —" +D136,?,"Stage-unload crash — objFree obj->prop reads float garbage (M-22, WS6 Facility playthrough).",ROOT-CAUSED → D139 (it was NOT a layout bug — +D139,FIXED,Stage-unload crash root cause + fix (M-23).,"FIXED (src/game/cleanup_objects.c, #ifdef PORT —" +D140,FIXED,Pause-menu crash (M-23 open → M-27 fixed).,"FIXED (src/game/bondview.h, src/game/bondview2.c; #ifdef PORT)." +D141,FIXED,"Watch-menu item-model crash (M-27), exposed by D140.","FIXED (src/game/gunfire.c, #ifdef PORT)." +D152,OPEN,"Mission-failed / objective-failed → permanent black screen (M-28, user bug report —","MITIGATED (port/src/libultra.c, M-28, #ifdef PORT/port-only)." +D154,FIXED,"bg.c room-geometry ray/hit-test GBI parser port (M-28, WRITTEN / UNVERIFIED / UNCOMMITTED).",M-30 re-audit + fixes (still playtest-gated for final verify). +D156,?,"Facility outro-cutscene ""hang"" —","GUARDED (src/game/model.c, #ifdef PORT):" +D155,FIXED,"Facility outro-cutscene end → ""hang"" = unclamped deltaFrames spiral (M-29, user bug report).","FIXED (src/game/frametiming.c, #ifdef PORT —" +D153,?,"-level_09 frame ~900–1400 0xc0000005 = D117/D134 load flakiness, NOT a bug (M-28).",NOT A BUG — closed. Run anchors on a quiet machine. +D151,FIXED,Watch data all blank —,"FIXED (src/bondtypes.h, #ifdef PORT —" +D130,RESOLVED,"crash class C2 (Facility -level_34, Runway -level_35):",resolved — stage all 6 N64 fields in a local f[6] before writing any. +D164,PROPOSED,"Disclaimer / legal screen only draws the first line (D76 root cause, M-31).","FIX PROPOSED, NOT APPLIED (front.c is owned by another agent this session)." +D165,FIXED,"Front-end menu cursor doesn't feel like a mouse pointer (M-31, user bug B5 follow-up).","FIXED (port/src/input.c; Input.MenuPointerMode, Input.HipfirePitchSpeed added)." +D166,FIXED,Hipfire mouse pitch is digital while yaw is analog —,FIXED (port/src/input.c). diff --git a/tools_pc/README.md b/tools_pc/README.md new file mode 100644 index 00000000..f31e46ca --- /dev/null +++ b/tools_pc/README.md @@ -0,0 +1,73 @@ +# `tools_pc/` — PC-port dev & test scripts + +Index for the ~60 scripts here (speed-ups plan Step 4 / N2). None of these are +part of the build; they are converters, verification tooling, and +investigation artifacts. + +**Marker:** +- **`living`** — kept and re-run; part of the verification ritual, the release + flow, or a converter the build depends on (via the sidecars it emits). +- **`artifact`** — added to chase one finding. Safe to move to + `tools_pc/archive//` once that finding is closed and the technique is + captured in `docs/dev/findings.md` / `docs/porting-notes.md`. + +## Living — converters the sidecars depend on + +| Script | Role | +|---|---| +| `d43_emit.py` | Offline N64→PC model-file converter → RZ sidecar + `manifest.csv` (D50 / Plan B). Regen after any model-format change. | +| `d43_convert.py` | Reference single-file model converter + full-512 layout/pointer validator (D43). | +| `d69_emit.py` | Offline converter for stage `bg/*.seg` + `Tbg_*_stanZ` → concatenated sidecar (D69/D78–D82). | +| `d88_emit.py` | Offline converter for per-level `Usetup*Z` stage-setup files → appended to the `pccg.bin` sidecar (D88). | +| `d88_propdefs.py` | The `propDefs` polymorphic-record stream N64→PC converter used by `d88_emit.py` (D88.4). | +| `d69_emit.py` / `d88_emit.py` / `d43_emit.py` | Run all three (+ `d88_propdefs`) to rebuild `data/` sidecars from the ROM. | + +## Living — verification & release + +| Script | Role | +|---|---| +| `framediff.py` | Visual regression: candidate `GE_PCDUMP` frames vs `tools_pc/golden/`, per-region divergence. Structural/tolerant by default; `--exact` after `GE_DETERM`. | +| `pixcount.py` | Count non-black pixels in a PPM dump — "did the scene render anything" as a number. | +| `level_sweep.sh` | Bare `-level_XX` boot of all 21 solo levels → PASS / NO-FRAMES / CRASH. (Step 2 folds this into `verify.sh sweep`.) | +| `playtest.sh` | Launch a level for `docs/dev/LEVEL-PLAYTEST.md` human validation (WS6). | +| `debug.ps1` / `repro_gdb.sh` / `attach_animgen.sh` | Launch (or attach to) the game under gdb so a crash always leaves a backtrace. | +| `bundle-win.sh` / `bundle-linux.sh` | Package a built tree as a distributable archive (exe + licenses + `prepare-assets/`). | +| `romverify.c` | One-shot `.z64` integrity check against the repo's ground truths. | +| `dump_objectives.py` | Dump per-level objectives + win/fail criteria from the ROM (playtest validation; Step 10 / N8 acceptance oracle). | +| `disasm.py` | Minimal MIPS disassembler for the BE ROM (RAM-addr → file offset). | +| `ppm2bmp.py` | PPM → 24-bit BMP, no deps — eyeball `GE_PCDUMP` frames without PIL. | +| `gen_findings_index.py` | Regenerate `docs/dev/findings-index.csv` (grep-before-you-read aid for the 200 KB finding log). `--check` in CI-style use. | +| `gen_env_probes.py` | Drift check for `docs/dev/GE-ENV-PROBES.md` — re-greps live `getenv("GE_*")` sites, reports NEW/GONE. | + +## Living — compiler-verified layout probes (kept: re-run when structs change) + +| Script | Role | +|---|---| +| `d43_layoutprobe.c` | Prints `sizeof`/`offsetof` for every struct `d43_emit.py` must reproduce, built with the port toolchain. | +| `d88_layoutprobe.c` | Same, for the `PROPDEF_*` record structs `d88_emit.py` converts. | +| `mtxtest.c` | Standalone numerical test of the fast3d matrix pipeline conventions (GE swapped-perspective, packing, MUL order). | + +## Investigation artifacts — D43 model-file format (finding closed; archive candidates) + +`d43_chainbound.py`, `d43_cover.py`, `d43_decode.py`, `d43_fullwalk.py`, +`d43_gdldump.py`, `d43_gdlhist.py`, `d43_gdlorder.py`, `d43_gdlseq.py`, +`d43_invariants.py`, `d43_layout.py`, `d43_layout1.py`, `d43_lutscan.py`, +`d43_pointusage.py`, `d43_seg5.py`, `d43_seg5ops.py`, `d43_seg5vtx.py`, +`d43_seg5vtx2.py`, `d43_sizes.py`, `d43_sizes2.py`, `d43_tree_dump.py`, +`d43_vtxfmt.py`, `d43_walk.py` — layout/pointer/GDL-order analysis passes that +derived the D43 converter spec. `d43_sizes.py` is still `exec()`'d by a couple +of the others. + +## Investigation artifacts — other closed/parked findings + +| Script | Finding | +|---|---| +| `d125_check.py` | D125 — is the emitted `propDefs` blob byte-identical to `d88_propdefs.convert_stream()`? | +| `d88_propdef_scan.py` | D88.4 — histogram of `PROPDEF_*` record types across shipped levels. | +| `d51_gdb.py` / `dump_animgen.cmd` | D51 texture-tile crash capture / Facility-outro-hang anim-state dump (gdb scripts). | + +## Other + +- `dist/` — staged bundle README template (`README.md.in`) + tokens. +- `golden/` — per-level golden frames for `framediff.py` (Step 3 fills this out). +- `__pycache__/` — ignored. diff --git a/tools_pc/gen_env_probes.py b/tools_pc/gen_env_probes.py new file mode 100644 index 00000000..0cc7eb9a --- /dev/null +++ b/tools_pc/gen_env_probes.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Drift check for docs/dev/GE-ENV-PROBES.md (speed-ups plan Step 4 / R9). + +That doc is a hand-curated table of every `GE_*` env probe -- var, file:line, +what it does, live/dead. It was last snapshotted by a manual grep and drifts +as probes are added/removed. This script does NOT regenerate the prose (the +curation is the value); it re-greps the live `getenv("GE_...")` sites and +reports: + * NEW -- a var with live sites that the doc never names + * GONE -- a var the doc names that has no live site any more + * a fresh var -> file:line site map (paste into the doc's File:line cells) + +Usage: + python tools_pc/gen_env_probes.py # print the drift report + python tools_pc/gen_env_probes.py --check # exit 1 if NEW or GONE +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DOC = ROOT / "docs" / "dev" / "GE-ENV-PROBES.md" +SCAN_DIRS = ["src", "port"] +GETENV_RE = re.compile(r'getenv\(\s*"(GE_[A-Z0-9_]+)"\s*\)') + + +def live_sites() -> dict[str, list[str]]: + sites: dict[str, list[str]] = {} + for d in SCAN_DIRS: + for path in sorted((ROOT / d).rglob("*")): + if path.suffix not in {".c", ".cpp", ".h", ".def", ".inc"}: + continue + try: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + continue + for n, line in enumerate(lines, 1): + for m in GETENV_RE.finditer(line): + rel = path.relative_to(ROOT).as_posix() + sites.setdefault(m.group(1), []).append(f"{rel}:{n}") + return sites + + +def doc_vars() -> tuple[set[str], set[str]]: + """Return (all vars named in the doc, vars whose row is an intentional tombstone).""" + if not DOC.exists(): + return set(), set() + named, tombstone = set(), set() + for line in DOC.read_text(encoding="utf-8").splitlines(): + vs = re.findall(r"GE_[A-Z0-9_]+", line) + named.update(vs) + if "tombstone" in line.lower(): + tombstone.update(vs) + return named, tombstone + + +def main() -> int: + check = "--check" in sys.argv[1:] + sites = live_sites() + documented, tombstoned = doc_vars() + live = set(sites) + + new = sorted(live - documented) + gone = sorted(v for v in documented - live if v not in ({"GE_DETERM"} | tombstoned)) + + out = [] + out.append(f"# GE_* probe drift vs {DOC.relative_to(ROOT).as_posix()}") + out.append(f"# {len(live)} vars with live sites, {len(documented)} named in the doc\n") + if new: + out.append("## NEW -- live sites, not in the doc (add a row):") + for v in new: + out.append(f" {v}") + for s in sites[v]: + out.append(f" {s}") + if gone: + out.append("\n## GONE -- named in the doc, no live site (mark removed):") + out += [f" {v}" for v in gone] + if not new and not gone: + out.append("doc is in sync with live getenv sites (site line numbers still drift -- see map below)") + + out.append("\n## full live site map:") + for v in sorted(sites): + out.append(f" {v}: " + ", ".join(sites[v])) + + print("\n".join(out)) + if check and (new or gone): + sys.stderr.write(f"\nGE-ENV-PROBES.md drift: {len(new)} new, {len(gone)} gone\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools_pc/gen_findings_index.py b/tools_pc/gen_findings_index.py new file mode 100644 index 00000000..724ced2e --- /dev/null +++ b/tools_pc/gen_findings_index.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Generate docs/dev/findings-index.csv from the §F/§H tables in findings.md. + +Agentic speed-ups plan Step 4 / R8: findings.md is ~200 KB. Agents should grep +this CSV (label, status, one-liner) to decide *which* Dxx entries to actually +read, instead of linear-reading the log. + +Heuristic parser: every markdown table row whose first cell matches a Dxx +label (`D12`, `D88.4`, `D75 · D77`, `D24-D30`, ...). The topic cell is +truncated to a one-liner; the status cell is normalised to a short bucket +(RESOLVED / OPEN / FIXED / PARTIAL / PROPOSED / ?) with the raw text kept. + +Usage: + python tools_pc/gen_findings_index.py # write the CSV + python tools_pc/gen_findings_index.py --check # exit 1 if stale + python tools_pc/gen_findings_index.py --stdout # print, don't write +""" +from __future__ import annotations + +import csv +import io +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "docs" / "dev" / "findings.md" +OUT = ROOT / "docs" / "dev" / "findings-index.csv" + +LABEL_RE = re.compile(r"^D\d+(?:\.\d+)?(?:[-–]D\d+)?(?:\s*[·,]\s*D\d+(?:\.\d+)?)*\+?$") + +STATUS_BUCKETS = [ + ("FIXED", re.compile(r"\bfixed\b", re.I)), + ("RESOLVED", re.compile(r"\bresolved\b", re.I)), + ("PARTIAL", re.compile(r"\bpartial|clamp landed|landed\b|safety-net", re.I)), + ("PROPOSED", re.compile(r"\bproposed|not applied|design\b", re.I)), + ("OPEN", re.compile(r"\bopen\b|not started|awaiting|owed|pending", re.I)), +] + + +def one_liner(cell: str, limit: int = 160) -> str: + """Strip markdown emphasis/code and collapse to a single short line.""" + txt = cell.replace("**", "").replace("`", "") + txt = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", txt) # [t](u) -> t + txt = re.sub(r"\s+", " ", txt).strip() + # first sentence-ish + m = re.match(r"(.{20,}?[.—:])\s", txt) + if m and len(m.group(1)) <= limit: + txt = m.group(1) + if len(txt) > limit: + txt = txt[: limit - 1].rstrip() + "…" + return txt + + +LEAD_RE = re.compile(r"^\**\s*(OPEN|RESOLVED|FIXED|PROPOSED|PARTIAL|LANDED|CLAMP|DISPROVEN)", re.I) + + +def bucket(status_cell: str) -> str: + # An explicit leading verdict token wins over prose keywords deeper in the + # cell (e.g. "OPEN (parked). ... D168 fixed ..." is OPEN, not FIXED). + lead = LEAD_RE.match(status_cell.strip()) + if lead: + tok = lead.group(1).upper() + return {"LANDED": "PARTIAL", "CLAMP": "PARTIAL", "DISPROVEN": "RESOLVED"}.get(tok, tok) + for name, rx in STATUS_BUCKETS: + if rx.search(status_cell): + return name + return "?" + + +def split_row(line: str) -> list[str] | None: + line = line.strip() + if not line.startswith("|"): + return None + # split on '|' that is not inside backticks + cells, buf, tick = [], [], False + for ch in line: + if ch == "`": + tick = not tick + buf.append(ch) + elif ch == "|" and not tick: + cells.append("".join(buf)) + buf = [] + else: + buf.append(ch) + cells.append("".join(buf)) + cells = [c.strip() for c in cells] + # leading/trailing empty from outer pipes + if cells and cells[0] == "": + cells = cells[1:] + if cells and cells[-1] == "": + cells = cells[:-1] + return cells or None + + +def rows(): + section = None + for raw in SRC.read_text(encoding="utf-8").splitlines(): + h = re.match(r"^#+\s+([A-Z])\.\s", raw) or re.match(r"^#+\s.*§?([FH])\b", raw) + if raw.startswith("### ") and re.search(r"\b([FH])\.", raw): + section = re.search(r"\b([FH])\.", raw).group(1) + cells = split_row(raw) + if not cells or len(cells) < 2: + continue + label = cells[0].replace("–", "-") + if not LABEL_RE.match(cells[0]): + continue + topic = cells[1] + status = cells[2] if len(cells) > 2 else "" + yield { + "label": label, + "bucket": bucket(status), + "one_liner": one_liner(topic), + "status_raw": one_liner(status, 200) if status else "", + } + + +def build_csv() -> str: + buf = io.StringIO() + w = csv.DictWriter( + buf, fieldnames=["label", "bucket", "one_liner", "status_raw"], lineterminator="\n" + ) + w.writeheader() + seen = set() + for r in rows(): + key = r["label"] + if key in seen: + continue + seen.add(key) + w.writerow(r) + return buf.getvalue() + + +def main() -> int: + args = set(sys.argv[1:]) + text = ( + "# GENERATED by tools_pc/gen_findings_index.py -- do not hand-edit.\n" + "# Grep this to pick which findings.md entries to read; it is not authoritative.\n" + + build_csv() + ) + if "--stdout" in args: + sys.stdout.write(text) + return 0 + want = text.encode("utf-8") + if "--check" in args: + cur = OUT.read_bytes() if OUT.exists() else b"" + if cur != want: + sys.stderr.write("findings-index.csv is stale: re-run gen_findings_index.py\n") + return 1 + return 0 + OUT.write_bytes(want) + print(f"wrote {OUT.relative_to(ROOT)} ({text.count(chr(10))} lines)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())