portx64: pd-extract: bg (extract roomgfxdata) and some model fixes

port/pdextract/common.h

@@ -1,20 +1,48 @@
-#ifndef _COMMON_H
-#define _COMMON_H
+#ifndef _PDE_COMMON_H
+#define _PDE_COMMON_H
+
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 
-#define srctoh16(value) be16toh(value)
-#define srctoh32(value) be32toh(value)
-#define srctoh64(value) be64toh(value)
+#include <winsock2.h>
+#include <sys/param.h>
+
+#if __BIG_ENDIAN__
+	#define htonll(x)   (x)
+	#define ntohll(x)   (x)
+#else
+	#define htonll(x)   ((((uint64_t)htonl(x&0xFFFFFFFF)) << 32) + htonl(x >> 32))
+	#define ntohll(x)   ((((uint64_t)ntohl(x&0xFFFFFFFF)) << 32) + ntohl(x >> 32))
+#endif
+
+#define htobe16(x) htons(x)
+#define htole16(x) (x)
+#define be16toh(x) ntohs(x)
+#define le16toh(x) (x)
+
+#define htobe32(x) htonl(x)
+#define htole32(x) (x)
+#define be32toh(x) ntohl(x)
+#define le32toh(x) (x)
+
+#define htobe64(x) htonll(x)
+#define htole64(x) (x)
+#define be64toh(x) ntohll(x)
+#define le64toh(x) (x)
+
+
+#define srctoh16(value) ntohs(value)
+#define srctoh32(value) ntohl(value)
+#define srctoh64(value) ntohll(value)
 
 #ifdef FORCE_BE32
-#define htodst16(value) htobe16(value)
-#define htodst32(value) htobe32(value)
-#define htodst64(value) htobe64(value)
-#define dsttoh16(value) be16toh(value)
-#define dsttoh32(value) be32toh(value)
-#define dsttoh64(value) be64toh(value)
+#define htodst16(value) htons(value)
+#define htodst32(value) htonl(value)
+#define htodst64(value) htonll(value)
+#define dsttoh16(value) ntohs(value)
+#define dsttoh32(value) ntohl(value)
+#define dsttoh64(value) ntohll(value)
 #else
 #define htodst16(value) (value)
 #define htodst32(value) (value)
@@ -66,6 +94,9 @@ int rzip_deflate(uint8_t *dst, size_t *dstlen, uint8_t *src, size_t srclen);
 void gbi_reset(void);
 void gbi_set_segment(int segment, uint32_t offset);
 void gbi_set_vtx(uint32_t src_offset, uint32_t dst_offset);
-uint32_t gbi_convert_gdl(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t srcpos);
+uint32_t gbi_convert_gdl(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t srcpos, int segment_cmds);
+void gbi_convert_vtx(uint8_t* dst, uint32_t dstpos, int count);
+void gbi_gdl_rewrite_addrs(uint8_t* dst, uint32_t offset);
+void gbi_add_tex_addr(uint32_t src_offset, uint32_t dst_offset);
 
 #endif

port/pdextract/ext_file_bg.c

@@ -3,6 +3,45 @@
 #include <stdlib.h>
 #include <string.h>
 #include "common.h"
+#include "system.h"
+
+struct ptrmarker {
+	uint32_t ptr_src;
+	uintptr_t ptr_host;
+};
+
+
+struct ptrmarker m_PtrMarkers[2048];
+int m_NumPtrMarkers;
+int m_SizeGDLs;
+
+static void add_marker(uint32_t ptr_src, uintptr_t ptr_host)
+{
+	if (m_NumPtrMarkers >= ARRAYCOUNT(m_PtrMarkers)) {
+		fprintf(stderr, "[BG] Marker limit exceeded\n");
+		exit(EXIT_FAILURE);
+	}
+
+	m_PtrMarkers[m_NumPtrMarkers].ptr_src = ptr_src;
+	m_PtrMarkers[m_NumPtrMarkers].ptr_host = ptr_host;
+	m_NumPtrMarkers++;
+}
+
+static struct ptrmarker* find_marker(uintptr_t ptr_src)
+{
+	for (int i = 0; i < m_NumPtrMarkers; i++) {
+		if (m_PtrMarkers[i].ptr_src == ptr_src) {
+			return &m_PtrMarkers[i];
+		}
+	}
+
+	return NULL;
+}
+
+static void reset_markers()
+{
+	m_NumPtrMarkers = 0;
+}
 
 /**
  * 4 bytes decompressed size of primary data
@@ -113,9 +152,57 @@ struct light {
 	int16_t bbox[4][3];
 };
 
+struct n64_roomblock {
+	uint8_t type;
+	uint32_t ptr_next;
+	uint32_t ptr_gdl;
+	uint32_t ptr_vertices;
+	uint32_t ptr_colours;
+};
+
+struct host_roomblock {
+	uint8_t type;
+	uintptr_t ptr_next;
+	uintptr_t ptr_gdl;
+	uintptr_t ptr_vertices;
+	uintptr_t ptr_colours;
+};
+
+struct n64_roomgfxdata {
+	uint32_t ptr_vertices;
+	uint32_t ptr_colours;
+	uint32_t ptr_opablocks;
+	uint32_t ptr_xlublocks;
+	int16_t lightsindex;
+	int16_t numlights;
+	int16_t numvertices;
+	int16_t numcolours;
+	// roomblocks[]
+};
+
+struct host_roomgfxdata {
+	uintptr_t ptr_vertices;
+	uintptr_t ptr_colours;
+	uintptr_t ptr_opablocks;
+	uintptr_t ptr_xlublocks;
+	int16_t lightsindex;
+	int16_t numlights;
+	int16_t numvertices;
+	int16_t numcolours;
+	// roomblocks[]
+};
+
+struct vtx {
+	int16_t coord[3];
+	uint8_t flag;
+	uint8_t color;
+	uint16_t s;
+	uint16_t t;
+};
+
 int m_NumRooms;
 
-uint16_t m_AllocSizes[512];
+uint32_t m_AllocSizes[512];
 
 static void convert_primary_rooms(uint8_t *dst, uint32_t *dstpos, uint8_t *src, uint32_t *srcpos)
 {
@@ -137,11 +224,11 @@ static void convert_primary_rooms(uint8_t *dst, uint32_t *dstpos, uint8_t *src,
 		host_rooms[i].br_light_max = n64_rooms[i].br_light_max;
 	}
 
-	*srcpos += sizeof(*n64_rooms) * (m_NumRooms + 1);
-	*dstpos += sizeof(*host_rooms) * (m_NumRooms + 1);
+	*srcpos += sizeof(*n64_rooms) * (m_NumRooms + 2);
+	*dstpos += sizeof(*host_rooms) * (m_NumRooms +2);
 }
 
-static void convert_primary_portals(uint8_t *dst, uint32_t *dstpos, uint8_t *src, uint32_t *srcpos)
+static void convert_primary_portals(uint8_t *dst, uint32_t *dstpos, uint8_t *src, uint32_t *srcpos, uint32_t *src_portalvtxs, uint32_t *dst_portalvtxs)
 {
 	struct portal *n64_portals = (struct portal *) &src[*srcpos];
 	struct portal *host_portals = (struct portal *) &dst[*dstpos];
@@ -164,13 +251,12 @@ static void convert_primary_portals(uint8_t *dst, uint32_t *dstpos, uint8_t *src
 	*srcpos += sizeof(*n64_portals) * (numportals + 1);
 	*dstpos += sizeof(*host_portals) * (numportals + 1);
 
-	// Convert portal vertices
-	uint8_t *n64_vtxdata = &src[*srcpos];
-	uint8_t *host_vtxdata = &dst[*dstpos];
+	*src_portalvtxs = *srcpos;
+	*dst_portalvtxs = *dstpos;
 
-	for (int i = 0; i < numportals; i++) {
-		struct portalvertices *n64_pvertices = (struct portalvertices*) &n64_vtxdata[*srcpos];
-		struct portalvertices *host_pvertices = (struct portalvertices*) &host_vtxdata[*dstpos];
+	while (1) {
+		struct portalvertices *n64_pvertices = (struct portalvertices*)&src[*srcpos];
+		struct portalvertices *host_pvertices = (struct portalvertices*)&dst[*dstpos];
 
 		host_pvertices->count = n64_pvertices->count;
 
@@ -182,10 +268,12 @@ static void convert_primary_portals(uint8_t *dst, uint32_t *dstpos, uint8_t *src
 
 		*srcpos += 4 + host_pvertices->count * 12;
 		*dstpos += 4 + host_pvertices->count * 12;
+
+		if (host_pvertices->count == 0)  break;
 	}
 }
 
-static void convert_primary_bgcmds(uint8_t *dst, uint32_t *dstpos, uint8_t *src, uint32_t *srcpos)
+static void convert_primary_bgcmds(uint8_t *dst, uint32_t *dstpos, uint8_t *src, uint32_t *srcpos, uint32_t dst_portalvtxs, uint32_t src_portalvtxs)
 {
 	struct bgcmd *n64_cmds = (struct bgcmd *) &src[*srcpos];
 	struct bgcmd *host_cmds = (struct bgcmd *) &dst[*dstpos];
@@ -201,6 +289,11 @@ static void convert_primary_bgcmds(uint8_t *dst, uint32_t *dstpos, uint8_t *src,
 		host_cmds[i].type = n64_cmds[i].type;
 		host_cmds[i].len = n64_cmds[i].len;
 		host_cmds[i].param = srctodst32(n64_cmds[i].param);
+
+		if (host_cmds[i].type == 0x64) {
+			uint32_t offset = host_cmds[i].param - src_portalvtxs;
+			host_cmds[i].param = dst_portalvtxs + offset;
+		}
 	}
 
 	*srcpos += sizeof(*n64_cmds) * num_cmds;
@@ -238,21 +331,196 @@ static void convert_primary_lights(uint8_t *dst, uint32_t *dstpos, uint8_t *src,
 	*dstpos += sizeof(*host_lights) * num_lights;
 }
 
-static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t srcpos)
+#define BLOCK_LEAF 0
+#define BLOCK_PARENT 1
+
+void relink_ptr(uintptr_t** ptr)
 {
-	return dstpos;
+	if (!*ptr) return;
+
+	struct ptrmarker* marker = find_marker(*ptr);
+	if (!marker) {
+		fprintf(stderr, "[BG] WARNING: Unable to relink pointer: %p\n", ptr);
+		exit(EXIT_FAILURE);
+	}
+	
+	*ptr = marker->ptr_host;
 }
 
-static size_t sum(uint8_t *ptr, uint8_t *end)
+static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t compsize, uint32_t src_roomoffset, uint32_t dst_roomoffset)
 {
-	size_t sum = 0;
+	reset_markers();
+	dst_roomoffset += 0x0f000000;
+	size_t infsize = rzip_get_infsize(src);
+	uint8_t* src_roomgfxdata = calloc(1, infsize);
 
-	while (ptr < end) {
-		sum = (sum + *ptr) & 0xffffffff;
-		ptr++;
+	int ret = rzip_inflate(src_roomgfxdata, infsize, src, compsize);
+
+	if (ret < 0) {
+		fprintf(stderr, "Unable to inflate roomgfxdata %d\n", ret);
+		exit(EXIT_FAILURE);
 	}
 
-	return sum;
+	uintptr_t gdls_addr[64];
+	int numgdls = 0;
+
+	struct n64_roomgfxdata *src_header = (struct n64_roomgfxdata*)src_roomgfxdata;
+	struct host_roomgfxdata *dst_header = (struct host_roomgfxdata*)&dst[dstpos];
+
+	dst_header->ptr_vertices = srctoh32(src_header->ptr_vertices);
+	dst_header->ptr_colours = srctoh32(src_header->ptr_colours);
+	dst_header->ptr_opablocks = srctoh32(src_header->ptr_opablocks);
+	dst_header->ptr_xlublocks = srctoh32(src_header->ptr_xlublocks);
+	dst_header->lightsindex = srctoh16(src_header->lightsindex);
+	dst_header->numlights = srctoh16(src_header->numlights);
+	dst_header->numvertices = srctoh16(src_header->numvertices);
+	dst_header->numcolours = srctoh16(src_header->numcolours);
+
+	intptr_t endpos = dst_header->ptr_vertices - src_roomoffset;
+	uintptr_t curpos_src = sizeof(struct n64_roomgfxdata);
+	uintptr_t curpos_dst = sizeof(struct host_roomgfxdata);
+
+	// roomblocks
+	int ncoords = 0;
+	
+	// save the ptr to roomblocks here to iterate later to fix the pointers
+	struct host_roomblock *ptr_dst_roomblocks = (struct host_roomblock*)&dst[curpos_dst];
+
+	int numblocks = 0;
+	while ((intptr_t)(curpos_src + sizeof(struct n64_roomblock)) <= endpos) {
+		struct n64_roomblock *src_roomblock = (struct n64_roomblock*)&src_roomgfxdata[curpos_src];
+		struct host_roomblock *dest_roomblock = (struct host_roomblock*)&dst[curpos_dst];
+
+		dest_roomblock->type = src_roomblock->type;
+		dest_roomblock->ptr_next = srctoh32(src_roomblock->ptr_next);
+		dest_roomblock->ptr_gdl = srctoh32(src_roomblock->ptr_gdl);
+		dest_roomblock->ptr_vertices = srctoh32(src_roomblock->ptr_vertices);
+		dest_roomblock->ptr_colours = srctoh32(src_roomblock->ptr_colours);
+
+		add_marker(curpos_src + src_roomoffset, curpos_dst + dst_roomoffset);
+
+		if (src_roomblock->type == BLOCK_PARENT) {
+			ncoords++;
+			uintptr_t vtx_ptr = dest_roomblock->ptr_vertices - src_roomoffset;
+			if (vtx_ptr < endpos) endpos = vtx_ptr;
+		}
+		else if (src_roomblock->ptr_gdl) {
+			gdls_addr[numgdls++] = srctoh32(src_roomblock->ptr_gdl) - src_roomoffset;
+		}
+
+		curpos_src += sizeof(struct n64_roomblock);
+		curpos_dst += sizeof(struct host_roomblock);
+		numblocks++;
+	}
+
+	// block coords
+	for (size_t i = 0; i < ncoords; i++) {
+		uint32_t *src_coord = (uint32_t*)(src_roomgfxdata + curpos_src);
+		uint32_t *dst_coord = (uint32_t*)(dst + curpos_dst);
+
+		dst_coord[0] = srctoh32(src_coord[0]);
+		dst_coord[1] = srctoh32(src_coord[1]);
+		dst_coord[2] = srctoh32(src_coord[2]);
+		
+		dst_coord[3] = srctoh32(src_coord[3]);
+		dst_coord[4] = srctoh32(src_coord[4]);
+		dst_coord[5] = srctoh32(src_coord[5]);
+
+		add_marker(curpos_src + src_roomoffset, curpos_dst + dst_roomoffset);
+
+		curpos_src += 6 * sizeof(uint32_t);
+		curpos_dst += 6 * sizeof(uint32_t);
+	}
+
+	curpos_dst = ALIGN8(curpos_dst);
+
+	// vertices
+	uintptr_t ptr_src_vertices = curpos_src = ALIGN8(dst_header->ptr_vertices - src_roomoffset);
+	uintptr_t ptr_dst_vertices = curpos_dst + dst_roomoffset;
+
+	add_marker(curpos_src + src_roomoffset, curpos_dst + dst_roomoffset);
+
+	uintptr_t vtx_end = dst_header->ptr_colours - src_roomoffset;
+	vtx_end = dst_header->ptr_colours == 0 ? infsize : vtx_end;
+	
+	while (curpos_src < vtx_end) {
+		struct vtx *src_vtx = (struct vtx*)(src_roomgfxdata + curpos_src);
+		struct vtx *dst_vtx = (struct vtx*)(dst + curpos_dst);
+
+		dst_vtx->coord[0] = srctoh16(src_vtx->coord[0]);
+		dst_vtx->coord[1] = srctoh16(src_vtx->coord[1]);
+		dst_vtx->coord[2] = srctoh16(src_vtx->coord[2]);
+		
+		dst_vtx->flag = src_vtx->flag;
+		dst_vtx->color= src_vtx->color;
+
+		dst_vtx->s = srctoh16(src_vtx->s);
+		dst_vtx->t = srctoh16(src_vtx->t);
+
+		curpos_src += sizeof(struct vtx);
+		curpos_dst += sizeof(struct vtx);
+	}
+
+	curpos_dst = ALIGN8(curpos_dst);
+
+	// colors
+	uintptr_t ptr_src_colors = 0;
+	uintptr_t ptr_dst_colors = 0;
+
+	if (dst_header->ptr_colours) {
+		ptr_src_colors = curpos_src = ALIGN8(dst_header->ptr_colours - src_roomoffset);
+		ptr_dst_colors = curpos_dst + dst_roomoffset;
+
+		add_marker(curpos_src + src_roomoffset, curpos_dst + dst_roomoffset);
+		
+		uintptr_t col_end = numgdls > 0 != 0 ? gdls_addr[0] : infsize;
+		size_t col_len = col_end - curpos_src;
+
+		memcpy(dst + curpos_dst, src_roomgfxdata + curpos_src, col_len);
+		curpos_src += col_len;
+		curpos_dst += col_len;
+	}
+
+	// gdls
+	gbi_set_vtx(src_header->ptr_vertices, ptr_src_vertices);
+	gbi_set_segment(0x0e, dst_header->ptr_vertices);
+	
+	m_SizeGDLs = 0;
+	uintptr_t gdlstart = curpos_dst;
+	for (size_t i = 0; i < numgdls; i++) {
+		add_marker(gdls_addr[i] + src_roomoffset, curpos_dst + dst_roomoffset);
+		curpos_dst = gbi_convert_gdl(dst, curpos_dst, src_roomgfxdata, gdls_addr[i], 1);
+	}
+	m_SizeGDLs = curpos_dst - gdlstart;
+
+	// relink ptrs: roomblocks
+	for (size_t i = 0; i < numblocks; i++) {
+		struct host_roomblock *block = (struct host_roomblock*)&ptr_dst_roomblocks[i];
+		
+		relink_ptr(&block->ptr_next);
+
+		if (block->type == BLOCK_LEAF) {
+			relink_ptr(&block->ptr_gdl);
+			uint32_t offset_vtx = block->ptr_vertices - ptr_src_vertices - src_roomoffset;
+			uint32_t offset_col = block->ptr_colours - ptr_src_colors - src_roomoffset;
+
+			block->ptr_vertices = ptr_dst_vertices + offset_vtx;
+			block->ptr_colours = ptr_dst_colors + offset_col;
+		}
+		else {
+			relink_ptr(&block->ptr_gdl); // child
+			relink_ptr(&block->ptr_vertices); // coord
+		}
+	}
+	
+	// relink ptrs: header
+	relink_ptr(&dst_header->ptr_vertices);
+	relink_ptr(&dst_header->ptr_colours);
+	relink_ptr(&dst_header->ptr_opablocks);
+	relink_ptr(&dst_header->ptr_xlublocks);
+
+	free(src_roomgfxdata);
+	return curpos_dst;
 }
 
 static uint32_t convert_section1(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t srcpos, struct fileheader *n64_header)
@@ -276,51 +544,67 @@ static uint32_t convert_section1(uint8_t *dst, uint32_t dstpos, uint8_t *src, ui
 
 	// Convert primary
 	struct host_primaryheader *host_primary_header = (struct host_primaryheader *) &dstbin[dstbinpos];
+	struct n64_primaryheader *n64_primary_header = (struct n64_primaryheader *) &srcbin[srcbinpos];
 	srcbinpos += sizeof(struct n64_primaryheader);
 	dstbinpos += sizeof(*host_primary_header);
 
+	uint32_t src_bgcmds = srctodst32(n64_primary_header->ptr_bgcmds) - 0x0f000000;
+
 	host_primary_header->unused1 = 0;
 	host_primary_header->unused2 = 0;
 
-	host_primary_header->ptr_rooms = dstbinpos;
+	host_primary_header->ptr_rooms = dstbinpos + 0x0f000000;
 	convert_primary_rooms(dstbin, &dstbinpos, srcbin, &srcbinpos);
+	
+	if (n64_primary_header->ptr_lights) {
+		host_primary_header->ptr_lights = dstbinpos + 0x0f000000;
+		srcbinpos = srctodst32(n64_primary_header->ptr_lights) - 0x0f000000;
+		convert_primary_lights(dstbin, &dstbinpos, srcbin, &srcbinpos, src_bgcmds);
+	}
 
-	host_primary_header->ptr_portals = dstbinpos;
-	convert_primary_portals(dstbin, &dstbinpos, srcbin, &srcbinpos);
+	host_primary_header->ptr_portals = dstbinpos + 0x0f000000;
+	srcbinpos = srctodst32(n64_primary_header->ptr_portals) - 0x0f000000;
+	uint32_t src_portalvtxs, dst_portalvtxs;
+	convert_primary_portals(dstbin, &dstbinpos, srcbin, &srcbinpos, &src_portalvtxs, &dst_portalvtxs);
 
-	host_primary_header->ptr_bgcmds = dstbinpos;
-	convert_primary_bgcmds(dstbin, &dstbinpos, srcbin, &srcbinpos);
-
-	host_primary_header->ptr_lights = dstbinpos;
-	convert_primary_lights(dstbin, &dstbinpos, srcbin, &srcbinpos, infsize);
+	host_primary_header->ptr_bgcmds = dstbinpos + 0x0f000000;
+	srcbinpos = src_bgcmds;
+	convert_primary_bgcmds(dstbin, &dstbinpos, srcbin, &srcbinpos, dst_portalvtxs, src_portalvtxs);
 
 	// Convert room gfxdata
-	uint32_t n64_primary_diff = n64_header->primary_infsize - n64_header->primary_cmpsize;
-	struct n64_room *n64_rooms = (struct n64_room *) &src[sizeof(*n64_header)];
-	struct host_room *host_rooms = (struct host_room *) &dst[dstbinpos];
-
-	uint8_t *roomgfxbuf = calloc(1, 1024 * 1024);
-	uint8_t *allcmp = calloc(1, 1024 * 1024);
-	int roomgfxbufpos = 0;
+	uint32_t n64_primary_diff = srctoh32(n64_header->primary_infsize) - srctoh32(n64_header->primary_cmpsize) - 0xc;
+	struct n64_room *n64_rooms = (struct n64_room *) &srcbin[sizeof(struct n64_primaryheader)];
+	struct host_room *host_rooms = (struct host_room *) &dstbin[sizeof(struct host_primaryheader)];
+	
+	uint8_t *roomgfxbuf = calloc(1, (size_t)1024 * 1024);
+	uint8_t *allcmp = calloc(1, (size_t)1024 * 1024);
 	int cmppos = 0;
 
-	//for (int i = 0; i < m_NumRooms; i++) {
-	//	host_rooms[i].ptr_gfxdata = dstbinpos + cmppos;
+	host_rooms[0].ptr_gfxdata = 0;
+	for (int i = 1; i <= m_NumRooms; i++) {
+		host_rooms[i].ptr_gfxdata = dstbinpos + cmppos + 0x0f000000;
 
-	//	srcpos = srctoh32(n64_rooms[i].ptr_gfxdata) - n64_primary_diff;
-	//	roomgfxbufpos = convert_section1_roomgfxdata(roomgfxbuf, 0, src, srcpos);
+		if (i == m_NumRooms) break;
 
-	//	size_t ziplen = 0;
-	//	ret = rzip_deflate(&allcmp[cmppos], &ziplen, roomgfxbuf, roomgfxbufpos);
+		uint32_t gfxdatacompsize = srctoh32(n64_rooms[i + 1].ptr_gfxdata) - srctoh32(n64_rooms[i].ptr_gfxdata);
+		srcpos = srctoh32(n64_rooms[i].ptr_gfxdata) - n64_primary_diff - 0x0f000000;
+		uint32_t src_roomoffset = srctoh32(n64_rooms[i].ptr_gfxdata);
+		uint32_t roomgfxbufpos = convert_section1_roomgfxdata(roomgfxbuf, 0, src+srcpos, gfxdatacompsize, src_roomoffset, dstbinpos + cmppos);
 
-	//	if (ret < 0) {
-	//		fprintf(stderr, "Unable to compress BG roomgfxdata for room 0x%x: %d\n", i, ret);
-	//		exit(EXIT_FAILURE);
-	//	}
+		size_t ziplen = 0;
+		ret = rzip_deflate(&allcmp[cmppos], &ziplen, roomgfxbuf, roomgfxbufpos);
+		if (ret < 0) {
+			fprintf(stderr, "Unable to compress BG roomgfxdata for room 0x%x: %d\n", i, ret);
+			exit(EXIT_FAILURE);
+		}
 
-	//	ziplen = ALIGN16(ziplen);
-	//	cmppos += ziplen;
-	//}
+		ziplen = ALIGN16(ziplen);
+		cmppos += ziplen;
+
+		m_AllocSizes[i - 1] = roomgfxbufpos + (ziplen > m_SizeGDLs ? ziplen*4 : m_SizeGDLs*4);
+	}
+	
+	host_rooms[m_NumRooms+1].ptr_gfxdata = 0;
 
 	// Zip primary
 	size_t ziplen = 0;
@@ -392,7 +676,7 @@ static void convert_section3_allocsizes(uint8_t *dst, uint32_t *dstpos, uint8_t
 	uint16_t *dst_sizes = (uint16_t *) &dst[*dstpos];
 
 	for (int i = 0; i < m_NumRooms; i++) {
-		dst_sizes[i] = m_AllocSizes[i];
+		dst_sizes[i] = (m_AllocSizes[i] - 0x100) / 0x10;
 	}
 
 	*srcpos += sizeof(uint16_t) * (m_NumRooms - 1);
@@ -465,11 +749,17 @@ void extract_file_bg(char *filename, uint32_t romoffset, size_t len)
 			return;
 		}
 	}
-	return;
+
+	if (strcmp(filename, "ob/ob_mid.seg") == 0) {
+		return;
+	}
 
 	size_t dstlen = len * 2;
 	uint8_t *dst = calloc(1, dstlen);
 	uint32_t dstpos = 0;
+	m_NumPtrMarkers = 0;
+	m_SizeGDLs = 0;
+	gbi_reset();
 
 	printf("--------------------------------------------------------------------------------\n");
 	printf("-- %s\n", filename);
@@ -489,6 +779,8 @@ void extract_file_bg(char *filename, uint32_t romoffset, size_t len)
 	dstpos = convert_zipped_section(3, dst, dstpos, src, n64_section3_start + sizeof(*n64_section3header), n64_section3header);
 	dstpos = ALIGN16(dstpos);
 
+	sysLogPrintf(LOG_NOTE, "%s	%d	%d	%.3f", filename, len, dstpos, (f32)dstpos / len);
+
 	char outfilename[1024];
 	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
 

port/pdextract/ext_file_model.c

@@ -2,6 +2,12 @@
 #include <stdlib.h>
 #include <string.h>
 #include "common.h"
+#include "system.h"
+
+#include <PR/ultratypes.h>
+
+#undef near
+#undef far
 
 enum contenttype {
 	/* 0*/ CT_MODELDEF,
@@ -45,7 +51,7 @@ struct alignconfig m_AlignConfigs[] = {
 	/* 2*/ { 4,  4 },
 	/* 3*/ { 8,  8 },
 	/* 4*/ { 4,  4 },
-	/* 5*/ { 4,  8 },
+	/* 5*/ { 8,  8 },
 	/* 6*/ { 8,  8 },
 	/* 7*/ { 4,  4 },
 	/* 8*/ { 4,  4 },
@@ -396,6 +402,8 @@ static void populate_markers(uint8_t *src)
 	for (int i = 0; i < m_NumMarkers; i++) {
 		struct marker *marker = &m_Markers[i];
 		void *src_thing = &src[marker->src_offset];
+		int numvtx = 0;
+		uint32_t colstart = 0;
 
 		switch (marker->type) {
 		case CT_MODELDEF:
@@ -439,9 +447,14 @@ static void populate_markers(uint8_t *src)
 			break;
 		case CT_RODATA_GUNDL:
 			struct src_rodata_gundl *src_gundl = src_thing;
+			numvtx = srctoh16(src_gundl->numvertices);
+			colstart = srctoh32(src_gundl->ptr_vertices) + numvtx * 12;
+			colstart = ALIGN(colstart, m_AlignConfigs[marker->type].before);
+
 			set_marker(srctoh32(src_gundl->ptr_opagdl), CT_GDL, marker->src_offset);
 			set_marker(srctoh32(src_gundl->ptr_xlugdl), CT_GDL, marker->src_offset);
 			set_marker(srctoh32(src_gundl->ptr_vertices), CT_VTXCOL, marker->src_offset);
+			set_marker(colstart, CT_VTXCOL, marker->src_offset);
 			break;
 		case CT_RODATA_DISTANCE:
 			struct src_rodata_distance *src_dist = src_thing;
@@ -462,11 +475,18 @@ static void populate_markers(uint8_t *src)
 			break;
 		case CT_RODATA_STARGUNFIRE:
 			struct src_rodata_stargunfire *src_stargunfire = src_thing;
+			numvtx = srctoh16(src_stargunfire->unk00);
+
 			set_marker(srctoh32(src_stargunfire->ptr_vertices), CT_VTXCOL, marker->src_offset);
+			set_marker(srctoh32(src_stargunfire->ptr_vertices) + numvtx * 12, CT_VTXCOL, marker->src_offset);
 			set_marker(srctoh32(src_stargunfire->ptr_gdl), CT_GDL, marker->src_offset);
 			break;
 		case CT_RODATA_DL:
 			struct src_rodata_dl *src_dl = src_thing;
+			numvtx = srctoh16(src_dl->numvertices);
+			colstart = srctoh32(src_dl->ptr_vertices) + numvtx * 12;
+			colstart = ALIGN(colstart, 8);
+
 			set_marker(srctoh32(src_dl->ptr_opagdl), CT_GDL, marker->src_offset);
 			set_marker(srctoh32(src_dl->ptr_xlugdl), CT_GDL, marker->src_offset);
 			set_marker(srctoh32(src_dl->ptr_colours), CT_VTXCOL, marker->src_offset);
@@ -609,10 +629,7 @@ static uint32_t convert_content(uint8_t *dst, uint8_t *src, uint32_t src_file_le
 				dst_vtx = dsttoh32(dst_rodata->ptr_vertices) & 0x00ffffff;
 			}
 
-			gbi_set_vtx(src_vtx, dst_vtx);
-			gbi_set_segment(0x04, dst_vtx);
-			gbi_set_segment(0x05, 0);
-			dstpos = gbi_convert_gdl(dst, dstpos, src, marker->src_offset);
+			dstpos = gbi_convert_gdl(dst, dstpos, src, marker->src_offset, 0);
 			break;
 		case CT_RODATA_CHRINFO:
 			struct generic_rodata_chrinfo *src_chrinfo = src_thing;
@@ -803,6 +820,7 @@ static void relink_pointers(uint8_t *dst, uint8_t *src)
 			struct dst_texconfig *dst_texconfig = dst_thing;
 			if ((srctoh32(src_texconfig->ptr) & 0xff000000) == 0x05000000) {
 				dst_texconfig->ptr = htodst32(resolve_pointer(srctoh32(src_texconfig->ptr)));
+				gbi_add_tex_addr(srctoh32(src_texconfig->ptr), dst_texconfig->ptr);
 			}
 			break;
 		case CT_PARTS:
@@ -825,6 +843,17 @@ static void relink_pointers(uint8_t *dst, uint8_t *src)
 			dst_gundl->ptr_opagdl = htodst32(resolve_pointer(srctoh32(src_gundl->ptr_opagdl)));
 			dst_gundl->ptr_xlugdl = htodst32(resolve_pointer(srctoh32(src_gundl->ptr_xlugdl)));
 			dst_gundl->ptr_vertices = htodst32(resolve_pointer(srctoh32(src_gundl->ptr_vertices)));
+			
+			gbi_convert_vtx(dst, dst_gundl->ptr_vertices & 0x00ffffff, dst_gundl->numvertices);
+
+			gbi_set_vtx(srctoh32(src_gundl->ptr_vertices), dst_gundl->ptr_vertices);
+			gbi_set_segment(0x04, dst_gundl->ptr_vertices);
+
+			gbi_gdl_rewrite_addrs(dst, dst_gundl->ptr_opagdl);
+			gbi_gdl_rewrite_addrs(dst, dst_gundl->ptr_xlugdl);
+
+			if (dst_gundl->ptr_opagdl) dst_gundl->ptr_opagdl |= 1;
+			if (dst_gundl->ptr_xlugdl) dst_gundl->ptr_xlugdl |= 1;
 			break;
 		case CT_RODATA_DISTANCE:
 			struct src_rodata_distance *src_dist = src_thing;
@@ -841,6 +870,9 @@ static void relink_pointers(uint8_t *dst, uint8_t *src)
 			struct src_rodata_chrgunfire *src_chrgunfire = src_thing;
 			struct dst_rodata_chrgunfire *dst_chrgunfire = dst_thing;
 			dst_chrgunfire->ptr_texture = htodst32(resolve_pointer(srctoh32(src_chrgunfire->ptr_texture)));
+			if ((srctoh32(dst_chrgunfire->ptr_texture) & 0xff000000) == 0x05000000) {
+				gbi_add_tex_addr(srctoh32(dst_chrgunfire->ptr_texture), dst_chrgunfire->ptr_texture);
+			}
 			break;
 		case CT_RODATA_TOGGLE:
 			struct src_rodata_toggle *src_toggle = src_thing;
@@ -852,6 +884,13 @@ static void relink_pointers(uint8_t *dst, uint8_t *src)
 			struct dst_rodata_stargunfire *dst_stargunfire = dst_thing;
 			dst_stargunfire->ptr_vertices = htodst32(resolve_pointer(srctoh32(src_stargunfire->ptr_vertices)));
 			dst_stargunfire->ptr_gdl = htodst32(resolve_pointer(srctoh32(src_stargunfire->ptr_gdl)));
+
+			gbi_convert_vtx(dst, dst_stargunfire->ptr_vertices & 0x00ffffff, dst_stargunfire->unk00);
+
+			gbi_set_vtx(srctoh32(src_stargunfire->ptr_vertices), dst_stargunfire->ptr_vertices);
+			gbi_set_segment(0x04, dst_stargunfire->ptr_vertices);
+			gbi_gdl_rewrite_addrs(dst, dst_stargunfire->ptr_gdl);
+			if (dst_stargunfire->ptr_gdl) dst_stargunfire->ptr_gdl |= 1;
 			break;
 		case CT_RODATA_DL:
 			struct src_rodata_dl *src_dl = src_thing;
@@ -860,6 +899,16 @@ static void relink_pointers(uint8_t *dst, uint8_t *src)
 			dst_dl->ptr_xlugdl = htodst32(resolve_pointer(srctoh32(src_dl->ptr_xlugdl)));
 			dst_dl->ptr_colours = htodst32(resolve_pointer(srctoh32(src_dl->ptr_colours)));
 			dst_dl->ptr_vertices = htodst32(resolve_pointer(srctoh32(src_dl->ptr_vertices)));
+			
+			gbi_convert_vtx(dst, dst_dl->ptr_vertices & 0x00ffffff, dst_dl->numvertices);
+
+			gbi_set_vtx(srctoh32(src_dl->ptr_vertices), dst_dl->ptr_vertices);
+			gbi_set_segment(0x04, dst_dl->ptr_vertices);
+			gbi_gdl_rewrite_addrs(dst, dst_dl->ptr_opagdl);
+			gbi_gdl_rewrite_addrs(dst, dst_dl->ptr_xlugdl);
+			
+			if (dst_dl->ptr_opagdl) dst_dl->ptr_opagdl |= 1;
+			if (dst_dl->ptr_xlugdl) dst_dl->ptr_xlugdl |= 1;
 			break;
 		case CT_TEXDATA:
 		case CT_VTXCOL:
@@ -904,6 +953,7 @@ static int convert_model(uint8_t *dst, uint8_t *src, uint32_t srclen)
 
 void extract_file_model(char *filename, uint32_t romoffset, size_t len)
 {
+	gbi_reset();
 	// Unzip it
 	size_t infsize = rzip_get_infsize(&g_Rom[romoffset]);
 	uint8_t *src = malloc(infsize);
@@ -916,7 +966,7 @@ void extract_file_model(char *filename, uint32_t romoffset, size_t len)
 	}
 
 	// Convert it
-	uint8_t *dst = calloc(1, infsize * 4);
+	uint8_t *dst = calloc(1, infsize * 2);
 
 	int dstpos = convert_model(dst, src, infsize);
 
@@ -942,6 +992,8 @@ void extract_file_model(char *filename, uint32_t romoffset, size_t len)
 	char outfilename[1024];
 	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
 
+	sysLogPrintf(LOG_NOTE, "%s	%d	%d	%.3f", filename, infsize, dstpos, (f32)dstpos/infsize);
+
 	FILE *fp = openfile(outfilename);
 	fwrite(zipped, ziplen, 1, fp);
 	fclose(fp);

port/pdextract/gbi.c

@@ -21,71 +21,69 @@ struct vtx {
 	int16_t t;
 };
 
+struct texaddr {
+	uint32_t src;
+	uint32_t dst;
+};
+
 uint32_t m_Segments[16];
-uint8_t m_Seen[1024];
 uint32_t m_SrcVtxOffset;
 uint32_t m_DstVtxOffset;
 
+struct texaddr m_TexAddrs[64];
+int m_NumTexAddr;
+
 void gbi_reset(void)
 {
 	for (int i = 0; i < ARRAYCOUNT(m_Segments); i++) {
 		m_Segments[i] = 0;
 	}
-
-	for (int i = 0; i < ARRAYCOUNT(m_Seen); i++) {
-		m_Seen[i] = 0;
-	}
-
+	
 	m_SrcVtxOffset = 0;
 	m_DstVtxOffset = 0;
+	m_NumTexAddr = 0;
 }
 
 void gbi_set_segment(int segment, uint32_t offset)
 {
-	m_Segments[segment] = offset;
+	m_Segments[segment] = offset & 0x00ffffff;
 }
 
 void gbi_set_vtx(uint32_t src_offset, uint32_t dst_offset)
 {
-	m_SrcVtxOffset = src_offset;
-	m_DstVtxOffset = dst_offset;
+	m_SrcVtxOffset = src_offset & 0x00ffffff;
+	m_DstVtxOffset = dst_offset & 0x00ffffff;
 }
 
-static void gbi_convert_col(uint8_t *dst, uint32_t dstpos, int count)
+void gbi_add_tex_addr(uint32_t src_offset, uint32_t dst_offset)
 {
-	for (int i = 0; i < count; i++) {
-		int index = dstpos / 4;
+	m_TexAddrs[m_NumTexAddr].src   = src_offset & 0x00ffffff;
+	m_TexAddrs[m_NumTexAddr++].dst = dst_offset & 0x00ffffff;
+}
 
-		if (!m_Seen[index]) {
-			uint32_t *col = (uint32_t *) &dst[dstpos];
-
-			*col = srctodst32(*col);
-
-			m_Seen[index] = 1;
-		}
-
-		dstpos += sizeof(uint32_t);
+uint32_t gbi_find_tex_addr(uint32_t src_offset)
+{
+	for (int i = 0; i < m_NumTexAddr; i++) {
+		if (m_TexAddrs[i].src == src_offset)
+			return m_TexAddrs[i].dst;
 	}
+
+	return 0;
 }
 
-static void gbi_convert_vtx(uint8_t *dst, uint32_t dstpos, int count)
+void gbi_convert_vtx(uint8_t *dst, uint32_t offset, int count)
 {
+	if (!offset) return;
+	struct vtx* vtxs = (struct vtx*)(dst + offset);
+
 	for (int i = 0; i < count; i++) {
-		int index = dstpos / 4;
+		struct vtx* vtx = &vtxs[i];
 
-		if (!m_Seen[index]) {
-			struct vtx *vtx = (struct vtx *) &dst[dstpos];
-
-			vtx->x = srctodst16(vtx->x);
-			vtx->y = srctodst16(vtx->y);
-			vtx->z = srctodst16(vtx->z);
-			vtx->s = srctodst16(vtx->s);
-			vtx->t = srctodst16(vtx->t);
-
-			m_Seen[index] = 1;
-		}
-
-		dstpos += 12;
+		vtx->x = srctodst16(vtx->x);
+		vtx->y = srctodst16(vtx->y);
+		vtx->z = srctodst16(vtx->z);
+		vtx->s = srctodst16(vtx->s);
+		vtx->t = srctodst16(vtx->t);
 	}
 }
 
@@ -94,93 +92,105 @@ static void gbi_convert_vtx(uint8_t *dst, uint32_t dstpos, int count)
 #define CMD_IS_COL(cmd)      ((cmd & 0xff00000000000000) == 0x0700000000000000)
 #define CMD_IS_ENDDL(cmd)    ((cmd & 0xff00000000000000) == 0xb800000000000000)
 #define CMD_IS_MOVEWORD(cmd) ((cmd & 0xff00000000000000) == 0xbc00000000000000)
+#define CMD_IS_MOVEMEM(cmd)  ((cmd & 0xff00000000000000) == 0x0300000000000000)
+#define CMD_IS_MTX(cmd)      ((cmd & 0xff00000000000000) == 0x0100000000000000)
+#define CMD_IS_SETTIMG(cmd)  ((cmd & 0xff00000000000000) == 0xfd00000000000000)
+#define CMD_IS_SETZIMG(cmd)  ((cmd & 0xff00000000000000) == 0xfe00000000000000)
+#define CMD_IS_SETCIMG(cmd)  ((cmd & 0xff00000000000000) == 0xff00000000000000)
 
 #define CMD_HAS_ADDR(cmd) CMD_IS_VTX(cmd) \
 	|| CMD_IS_COL(cmd) \
-	|| CMD_IS_MOVEWORD(cmd)
+	|| CMD_IS_MOVEWORD(cmd) \
+	|| CMD_IS_SETTIMG(cmd)
 
-static void gbi_convert_vtx_by_cmd(uint64_t cmd, uint8_t *dst)
-{
-	uint8_t segment = cmd & 0xff000000;
-	uint8_t offset = cmd & 0x00ffffff;
-	uint32_t num_bytes = (cmd >> 32) & 0xffff;
-	int num_vertices = num_bytes / 12;
 
-	if (segment == 5) {
-		uint32_t dst_offset = offset - m_SrcVtxOffset + m_DstVtxOffset;
-		gbi_convert_vtx(dst, dst_offset, num_vertices);
-	} else {
-		gbi_convert_vtx(dst, m_Segments[segment] + offset, num_vertices);
-	}
-}
-
-static void gbi_convert_col_by_cmd(uint64_t cmd, uint8_t *dst)
-{
-	uint8_t segment = cmd & 0xff000000;
-	uint8_t offset = cmd & 0x00ffffff;
-	uint32_t num_bytes = (cmd >> 32) & 0xffff;
-	int num_colours = num_bytes / 4;
-
-	if (segment == 5) {
-		uint32_t dst_offset = offset - m_SrcVtxOffset + m_DstVtxOffset;
-		gbi_convert_col(dst, dst_offset, num_colours);
-	} else {
-		gbi_convert_col(dst, m_Segments[segment] + offset, num_colours);
-	}
-}
+#define CMD_IS_SEGMENTED(cmd) ( CMD_IS_MOVEMEM(cmd) \
+	|| CMD_IS_MTX(cmd) \
+	|| CMD_IS_VTX(cmd) \
+	|| CMD_IS_COL(cmd) \
+	|| CMD_IS_DL(cmd) \
+	|| CMD_IS_SETTIMG(cmd) \
+	|| CMD_IS_SETCIMG(cmd) \
+	|| CMD_IS_SETZIMG(cmd))
 
 /**
  * Segment 5 is the start of the model file. The data in these files shifts
  * depending on the pointer size, so this function adjusts the offset accordingly.
  */
-static uint64_t gbi_rewrite_addr(uint64_t cmd)
+static uint64_t gbi_rewrite_addr(uint64_t cmd, uint8_t opcode)
 {
-	uint8_t segment = cmd & 0xff000000;
-	uint8_t offset = cmd & 0x00ffffff;
+	uint8_t segment = (cmd & 0x0f000000) >> 24;
+	uint32_t offset = cmd & 0x00ffffff;
 
 	if (segment == 5) {
-		if (offset < m_SrcVtxOffset) {
-			printf("Tried to load data from offset 0x%x but segment starts at 0x%x\n", offset, m_SrcVtxOffset);
-			exit(EXIT_FAILURE);
+		if (opcode == 0xfd) {
+			offset = gbi_find_tex_addr(offset);
 		}
+		else {
+			if (offset < m_SrcVtxOffset) {
+				printf("Tried to load data from offset 0x%x but segment starts at 0x%x\n", offset, m_SrcVtxOffset);
+				exit(EXIT_FAILURE);
+			}
 
-		offset = offset - m_SrcVtxOffset + m_DstVtxOffset;
+			offset = offset - m_SrcVtxOffset + m_DstVtxOffset;
+		}
 	}
 
 	return (cmd & 0xffffffffff000000) | offset;
 }
 
-uint32_t gbi_convert_gdl(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t srcpos)
+void gbi_gdl_rewrite_addrs(uint8_t *dst, uint32_t offset)
+{
+	if (!offset) return;
+
+	uint64_t *cmds = (uint64_t*)(dst + (offset & 0x00ffffff));
+	if (!cmds) return;
+
+	uint64_t cmd;
+
+	do {
+		cmd = *cmds;
+
+		if (CMD_HAS_ADDR((cmd << 32))) {
+			uint8_t opcode = (cmd >> 24) & 0xff;
+			cmds[1] = gbi_rewrite_addr(cmds[1], opcode);
+		}
+
+		if (CMD_IS_SEGMENTED((cmd << 32))) {
+			cmds[1] |= 1;
+		}
+
+		cmds += 2;
+	} while (!CMD_IS_ENDDL((cmd << 32)));
+}
+
+uint32_t gbi_convert_gdl(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t srcpos, int segment_cmds)
 {
 	dstpos = ALIGN8(dstpos);
 
-	uint64_t *n64_cmd = (uint64_t *) &src[srcpos];
-	uint64_t *host_cmd = (uint64_t *) &dst[dstpos];
+	uint64_t *n64_cmd = (uint64_t*)&src[srcpos];
+	uint64_t *host_cmd = (uint64_t*)&dst[dstpos];
 	uint64_t cmd;
 
 	do {
 		cmd = srctoh64(*n64_cmd);
 
 #if HOST_DWORDS_PER_CMD == 2
-		if (CMD_HAS_ADDR(cmd)) {
-			cmd = gbi_rewrite_addr(cmd);
-			host_cmd[0] = htodst64(cmd & 0xffffffff00000000);
-			host_cmd[1] = htodst64(cmd & 0x00000000ffffffff);
-		} else {
-			host_cmd[0] = htodst64(cmd);
-			host_cmd[1] = 0;
+		host_cmd[0] = htodst64((cmd & 0xffffffff00000000) >> 32);
+		host_cmd[1] = htodst64(cmd & 0xffffffff);
+
+		if (segment_cmds && CMD_IS_SEGMENTED(cmd)) {
+			host_cmd[1] |= 1;
 		}
 #else
 		host_cmd[0] = htodst64(cmd);
-#endif
-		dstpos += sizeof(*host_cmd) * HOST_DWORDS_PER_CMD;
 
-		if (CMD_IS_VTX(cmd)) {
-			gbi_convert_vtx_by_cmd(cmd, dst);
-		} else if (CMD_IS_COL(cmd)) {
-			gbi_convert_col_by_cmd(cmd, dst);
+		if (segment_cmds && CMD_IS_SEGMENTED(cmd)) {
+			host_cmd[0] |= 1;
 		}
+#endif
 
+		dstpos += sizeof(*host_cmd) * HOST_DWORDS_PER_CMD;
 		host_cmd += HOST_DWORDS_PER_CMD;
 		n64_cmd++;
 	} while (!CMD_IS_ENDDL(cmd));