portx64: integrate pd-extract functions with romdata

port/pdextract/common.h

@@ -6,8 +6,17 @@
 #include <stdlib.h>
 
 #include <winsock2.h>
+// this BS comes from windows.h
+#undef near
+#undef far
+
 #include <sys/param.h>
 
+#include <PR/ultratypes.h>
+
+#include "constants.h"
+
+
 #if __BIG_ENDIAN__
 	#define htonll(x)   (x)
 	#define ntohll(x)   (x)
@@ -60,37 +69,6 @@
 #define uintptr_t uint32_t
 #endif
 
-enum romversion {
-	ROMVERSION_NTSC10,
-	ROMVERSION_NTSCFINAL,
-	ROMVERSION_PALFINAL,
-	ROMVERSION_JPNFINAL,
-	ROMVERSION_NTSCBETA,
-	ROMVERSION_PALBETA,
-	ROMVERSION_UNKNOWN,
-};
-
-extern uint8_t *g_Rom;
-extern enum romversion g_RomVersion;
-extern char g_OutPath[1024];
-
-typedef uint32_t n64ptr_t;
-
-#define ALIGN(val, size) ((val + (size - 1)) & ~(size - 1))
-#define ALIGN16(val) ((val + 0xf) & ~0xf)
-#define ALIGN8(val) ((val + 0x7) & ~0x7)
-#define ALIGN4(val) ((val + 0x3) & ~0x3)
-
-#define ARRAYCOUNT(arr) (int) (sizeof(arr) / sizeof(arr[0]))
-
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
-
-FILE *openfile(char *filename);
-size_t rzip_get_infsize(uint8_t *src);
-int rzip_inflate(uint8_t *dst, size_t dstlen, uint8_t *src, size_t srclen);
-int rzip_deflate(uint8_t *dst, size_t *dstlen, uint8_t *src, size_t srclen);
-
 void gbi_reset(void);
 void gbi_set_segment(int segment, uint32_t offset);
 void gbi_set_vtx(uint32_t src_offset, uint32_t dst_offset);

port/pdextract/ext_anims.c

@@ -1,78 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "common.h"
-
-struct animtableentry {
-	uint16_t numframes;
-	uint16_t bytesperframe;
-	uint32_t data;
-	uint16_t headerlen;
-	uint8_t framelen;
-	uint8_t flags;
-};
-
-static void extract_anims_data(uint32_t romoffset, size_t len)
-{
-	char outfilename[1024];
-	sprintf(outfilename, "%s/anims_data.bin", g_OutPath);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(&g_Rom[romoffset], len, 1, fp);
-	fclose(fp);
-}
-
-static void extract_anims_table(uint32_t data_offset, uint32_t table_offset)
-{
-	int num_anims = srctoh32(*(uint32_t *) &g_Rom[table_offset]);
-	struct animtableentry *srcanims = (struct animtableentry *) &g_Rom[table_offset + 4];
-
-	size_t dstlen = ALIGN16(4 + num_anims * sizeof(struct animtableentry));
-	uint8_t *dst = calloc(1, dstlen);
-	*(uint32_t *) dst = htodst32(num_anims);
-	struct animtableentry *dstanims = (struct animtableentry *) (dst + 4);
-
-	for (int i = 0; i < num_anims; i++) {
-		dstanims[i].numframes = srctodst16(srcanims[i].numframes);
-		dstanims[i].bytesperframe = srctodst16(srcanims[i].bytesperframe);
-		dstanims[i].data = srctodst32(srcanims[i].data);
-		dstanims[i].headerlen = srctodst16(srcanims[i].headerlen);
-		dstanims[i].framelen = srcanims[i].framelen;
-		dstanims[i].flags = srcanims[i].flags;
-	}
-
-	char outfilename[1024];
-	sprintf(outfilename, "%s/segs/animations", g_OutPath);
-
-	FILE *fp = openfile(outfilename);
-	size_t datalen = table_offset - data_offset;
-	fwrite(&g_Rom[data_offset], datalen, 1, fp);
-	fwrite(dst, dstlen, 1, fp);
-	fclose(fp);
-
-	free(dst);
-}
-
-void extract_anims(void)
-{
-	uint32_t table_offset = 0;
-	uint32_t data_offset = 0;
-
-	switch (g_RomVersion) {
-	case ROMVERSION_NTSC10:
-	case ROMVERSION_NTSCFINAL:
-		data_offset = 0x1a15c0;
-		table_offset = 0x7cd1a0;
-		break;
-	case ROMVERSION_PALFINAL:
-		data_offset = 0x18cdc0;
-		table_offset = 0x7b89a0;
-		break;
-	case ROMVERSION_JPNFINAL:
-		data_offset = 0x190c50;
-		table_offset = 0x7bc830;
-		break;
-	}
-
-	extract_anims_table(data_offset, table_offset);
-}

port/pdextract/ext_audio.c

@@ -391,95 +391,34 @@ static int convert_audio_bank(uint8_t *dst, int dstpos, uint8_t *src, int srcpos
 	return dstpos;
 }
 
-static int convert_audio_bankfile(uint8_t *dst, int dstpos, uint8_t *src, int srcpos)
+static int convert_audio_bankfile(uint8_t *dst, uint8_t *src)
 {
-	struct n64_bankfile *n64_bankfile = (struct n64_bankfile *) &src[srcpos];
-	struct host_bankfile *host_bankfile = (struct host_bankfile *) &dst[dstpos];
+	struct n64_bankfile *n64_bankfile = (struct n64_bankfile *)src;
+	struct host_bankfile *host_bankfile = (struct host_bankfile *)dst;
 	int bankCount = srctoh16(n64_bankfile->bankCount);
 
 	host_bankfile->revision = srctodst16(n64_bankfile->revision);
 	host_bankfile->bankCount = htodst16(bankCount);
 
-	dstpos = dstpos + sizeof(struct host_bankfile) + sizeof(uintptr_t) * (bankCount - 1);
+
+	uint32_t dstpos = sizeof(struct host_bankfile) + sizeof(uintptr_t) * (bankCount - 1);
 
 	for (int i = 0; i < bankCount; i++) {
 		host_bankfile->bankArray[i] = htodst32(dstpos);
-		srcpos = srctoh32(n64_bankfile->bankArray[i]);
+		uint32_t srcpos = srctoh32(n64_bankfile->bankArray[i]);
 		dstpos = convert_audio_bank(dst, dstpos, src, srcpos);
 	}
 
 	return dstpos;
 }
 
-static void extract_audio_ctl(const char *filename, uint32_t romoffset, size_t srclen)
+uint8_t *preprocessALBankFile_x64(uint8_t *src, uint32_t size, uint32_t *outSize)
 {
-	size_t dstlen = srclen * 4;
-	uint8_t *src = &g_Rom[romoffset];
+	size_t dstlen = size * 4;
 	uint8_t *dst = calloc(1, dstlen);
-	int dstpos = 0;
 
-	dstpos = convert_audio_bankfile(dst, dstpos, src, 0);
+	*outSize = convert_audio_bankfile(dst, src);
+	*outSize = ALIGN16(*outSize);
 
-	char outfilename[1024];
-	sprintf(outfilename, "%s/segs/%sctl", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(dst, ALIGN16(dstpos), 1, fp);
-	fclose(fp);
-
-	free(dst);
-}
-
-static void extract_audio_tbl(const char *filename, uint32_t romoffset, size_t len)
-{
-	char outfilename[1024];
-	sprintf(outfilename, "%s/segs/%stbl", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(&g_Rom[romoffset], len, 1, fp);
-	fclose(fp);
-}
-
-void extract_audio(void)
-{
-	uint32_t sfx_ctl = 0;
-	uint32_t sfx_tbl = 0;
-	uint32_t sfx_end = 0;
-
-	uint32_t seq_ctl = 0;
-	uint32_t seq_tbl = 0;
-	uint32_t seq_end = 0;
-
-	switch (g_RomVersion) {
-	case ROMVERSION_NTSC10:
-	case ROMVERSION_NTSCFINAL:
-		sfx_ctl = 0x80a250;
-		sfx_tbl = 0x839dd0;
-		sfx_end = 0xcfbf30;
-		seq_ctl = 0xcfbf30;
-		seq_tbl = 0xd05f90;
-		seq_end = 0xe82000;
-		break;
-	case ROMVERSION_PALFINAL:
-		sfx_ctl = 0x7f87e0;
-		sfx_tbl = 0x828360;
-		sfx_end = 0xcea4c0;
-		seq_ctl = 0xcea4c0;
-		seq_tbl = 0xcf4520;
-		seq_end = 0xe70590;
-		break;
-	case ROMVERSION_JPNFINAL:
-		sfx_ctl = 0x7fc670;
-		sfx_tbl = 0x82c1f0;
-		sfx_end = 0xcee350;
-		seq_ctl = 0xcee350;
-		seq_tbl = 0xcf83b0;
-		seq_end = 0xe74420;
-		break;
-	}
-
-	extract_audio_ctl("sfx", sfx_ctl, sfx_tbl - sfx_ctl);
-	extract_audio_tbl("sfx", sfx_tbl, sfx_end - sfx_tbl);
-	extract_audio_ctl("seq", seq_ctl, seq_tbl - seq_ctl);
-	extract_audio_tbl("seq", seq_tbl, seq_end - seq_tbl);
+	return dst;
 }

port/pdextract/ext_banners.c

@@ -1,39 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "common.h"
-
-void extract_banner(const char *filename, uint32_t romoffset, size_t len)
-{
-	char outfilename[1024];
-	sprintf(outfilename, "%s/%s.bin", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(&g_Rom[romoffset], len, 1, fp);
-	fclose(fp);
-}
-
-void extract_banners(void)
-{
-	uint32_t accessingpak = 0;
-	uint32_t copyright = 0;
-
-	switch (g_RomVersion) {
-	case ROMVERSION_NTSC10:
-	case ROMVERSION_NTSCFINAL:
-		copyright = 0x1ffea20;
-		accessingpak = 0x1fff550;
-		break;
-	case ROMVERSION_PALFINAL:
-		copyright = 0x1ff5500;
-		accessingpak = 0x1ff6030;
-		break;
-	case ROMVERSION_JPNFINAL:
-		copyright = 0x1ffd6b0;
-		accessingpak = 0x1ffe1e0;
-		break;
-	}
-
-	extract_banner("copyright", copyright, 0xb30);
-	extract_banner("accessingpak", accessingpak, 0x8b0);
-}

port/pdextract/ext_file_audio.c

@@ -1,14 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "common.h"
-
-void extract_file_audio(char *filename, uint32_t romoffset, size_t len)
-{
-	char outfilename[1024];
-	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(&g_Rom[romoffset], len, 1, fp);
-	fclose(fp);
-}

port/pdextract/ext_file_bg.c

@@ -4,6 +4,7 @@
 #include <string.h>
 #include "common.h"
 #include "system.h"
+#include "platform.h"
 
 struct ptrmarker {
 	uint32_t ptr_src;
@@ -18,7 +19,7 @@ int m_SizeGDLs;
 static void add_marker(uint32_t ptr_src, uintptr_t ptr_host)
 {
 	if (m_NumPtrMarkers >= ARRAYCOUNT(m_PtrMarkers)) {
-		fprintf(stderr, "[BG] Marker limit exceeded\n");
+		sysLogPrintf(LOG_ERROR, "[BG] Marker limit exceeded");
 		exit(EXIT_FAILURE);
 	}
 
@@ -216,7 +217,7 @@ static void convert_primary_rooms(uint8_t *dst, uint32_t *dstpos, uint8_t *src,
 	}
 
 	for (int i = 0; i < m_NumRooms + 1; i++) {
-		host_rooms[i].ptr_gfxdata = 0; // Not known at this point
+		host_rooms[i].ptr_gfxdata = srctodst32(n64_rooms[i].ptr_gfxdata);
 		host_rooms[i].pos[0] = srctodst32(n64_rooms[i].pos[0]);
 		host_rooms[i].pos[1] = srctodst32(n64_rooms[i].pos[1]);
 		host_rooms[i].pos[2] = srctodst32(n64_rooms[i].pos[2]);
@@ -273,9 +274,9 @@ static void convert_primary_portals(uint8_t *dst, uint32_t *dstpos, uint8_t *src
 	}
 }
 
-static void convert_primary_bgcmds(uint8_t *dst, uint32_t *dstpos, uint8_t *src, uint32_t *srcpos, uint32_t dst_portalvtxs, uint32_t src_portalvtxs)
+static void convert_primary_bgcmds(uint8_t *dst, uint32_t *dstpos, uint8_t *src, uint32_t dst_portalvtxs, uint32_t src_portalvtxs)
 {
-	struct bgcmd *n64_cmds = (struct bgcmd *) &src[*srcpos];
+	struct bgcmd *n64_cmds = (struct bgcmd *) src;
 	struct bgcmd *host_cmds = (struct bgcmd *) &dst[*dstpos];
 	int num_cmds = 0;
 
@@ -296,7 +297,6 @@ static void convert_primary_bgcmds(uint8_t *dst, uint32_t *dstpos, uint8_t *src,
 		}
 	}
 
-	*srcpos += sizeof(*n64_cmds) * num_cmds;
 	*dstpos += sizeof(*host_cmds) * num_cmds;
 }
 
@@ -347,25 +347,17 @@ void relink_ptr(uintptr_t** ptr)
 	*ptr = marker->ptr_host;
 }
 
-static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t compsize, uint32_t src_roomoffset, uint32_t dst_roomoffset)
+static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint8_t *src, uint32_t infsize, uint32_t src_ofs)
 {
 	reset_markers();
-	dst_roomoffset += 0x0f000000;
-	size_t infsize = rzip_get_infsize(src);
-	uint8_t* src_roomgfxdata = calloc(1, infsize);
 
-	int ret = rzip_inflate(src_roomgfxdata, infsize, src, compsize);
-
-	if (ret < 0) {
-		fprintf(stderr, "Unable to inflate roomgfxdata %d\n", ret);
-		exit(EXIT_FAILURE);
-	}
+	uint32_t dst_roomoffset = src_ofs;
 
 	uintptr_t gdls_addr[64];
 	int numgdls = 0;
 
-	struct n64_roomgfxdata *src_header = (struct n64_roomgfxdata*)src_roomgfxdata;
-	struct host_roomgfxdata *dst_header = (struct host_roomgfxdata*)&dst[dstpos];
+	struct n64_roomgfxdata *src_header = (struct n64_roomgfxdata*)src;
+	struct host_roomgfxdata *dst_header = (struct host_roomgfxdata*)dst;
 
 	dst_header->ptr_vertices = srctoh32(src_header->ptr_vertices);
 	dst_header->ptr_colours = srctoh32(src_header->ptr_colours);
@@ -376,7 +368,7 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 	dst_header->numvertices = srctoh16(src_header->numvertices);
 	dst_header->numcolours = srctoh16(src_header->numcolours);
 
-	intptr_t endpos = dst_header->ptr_vertices - src_roomoffset;
+	intptr_t endpos = dst_header->ptr_vertices - src_ofs;
 	uintptr_t curpos_src = sizeof(struct n64_roomgfxdata);
 	uintptr_t curpos_dst = sizeof(struct host_roomgfxdata);
 
@@ -388,7 +380,7 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 
 	int numblocks = 0;
 	while ((intptr_t)(curpos_src + sizeof(struct n64_roomblock)) <= endpos) {
-		struct n64_roomblock *src_roomblock = (struct n64_roomblock*)&src_roomgfxdata[curpos_src];
+		struct n64_roomblock *src_roomblock = (struct n64_roomblock*)&src[curpos_src];
 		struct host_roomblock *dest_roomblock = (struct host_roomblock*)&dst[curpos_dst];
 
 		dest_roomblock->type = src_roomblock->type;
@@ -397,15 +389,15 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 		dest_roomblock->ptr_vertices = srctoh32(src_roomblock->ptr_vertices);
 		dest_roomblock->ptr_colours = srctoh32(src_roomblock->ptr_colours);
 
-		add_marker(curpos_src + src_roomoffset, curpos_dst + dst_roomoffset);
+		add_marker(curpos_src + src_ofs, curpos_dst + dst_roomoffset);
 
 		if (src_roomblock->type == BLOCK_PARENT) {
 			ncoords++;
-			uintptr_t vtx_ptr = dest_roomblock->ptr_vertices - src_roomoffset;
+			uintptr_t vtx_ptr = dest_roomblock->ptr_vertices - src_ofs;
 			if (vtx_ptr < endpos) endpos = vtx_ptr;
 		}
 		else if (src_roomblock->ptr_gdl) {
-			gdls_addr[numgdls++] = srctoh32(src_roomblock->ptr_gdl) - src_roomoffset;
+			gdls_addr[numgdls++] = srctoh32(src_roomblock->ptr_gdl) - src_ofs;
 		}
 
 		curpos_src += sizeof(struct n64_roomblock);
@@ -415,7 +407,7 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 
 	// block coords
 	for (size_t i = 0; i < ncoords; i++) {
-		uint32_t *src_coord = (uint32_t*)(src_roomgfxdata + curpos_src);
+		uint32_t *src_coord = (uint32_t*)(src + curpos_src);
 		uint32_t *dst_coord = (uint32_t*)(dst + curpos_dst);
 
 		dst_coord[0] = srctoh32(src_coord[0]);
@@ -426,7 +418,7 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 		dst_coord[4] = srctoh32(src_coord[4]);
 		dst_coord[5] = srctoh32(src_coord[5]);
 
-		add_marker(curpos_src + src_roomoffset, curpos_dst + dst_roomoffset);
+		add_marker(curpos_src + src_ofs, curpos_dst + dst_roomoffset);
 
 		curpos_src += 6 * sizeof(uint32_t);
 		curpos_dst += 6 * sizeof(uint32_t);
@@ -435,16 +427,15 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 	curpos_dst = ALIGN8(curpos_dst);
 
 	// vertices
-	uintptr_t ptr_src_vertices = curpos_src = ALIGN8(dst_header->ptr_vertices - src_roomoffset);
+	uintptr_t ptr_src_vertices = curpos_src = ALIGN8(dst_header->ptr_vertices - src_ofs);
 	uintptr_t ptr_dst_vertices = curpos_dst + dst_roomoffset;
 
-	add_marker(curpos_src + src_roomoffset, curpos_dst + dst_roomoffset);
+	add_marker(curpos_src + src_ofs, curpos_dst + dst_roomoffset);
 
-	uintptr_t vtx_end = dst_header->ptr_colours - src_roomoffset;
-	vtx_end = dst_header->ptr_colours == 0 ? infsize : vtx_end;
-	
-	while (curpos_src < vtx_end) {
-		struct vtx *src_vtx = (struct vtx*)(src_roomgfxdata + curpos_src);
+	uintptr_t vtx_end = dst_header->ptr_colours - src_ofs;
+
+	while (dst_header->ptr_colours && curpos_src < vtx_end) {
+		struct vtx *src_vtx = (struct vtx*)(src + curpos_src);
 		struct vtx *dst_vtx = (struct vtx*)(dst + curpos_dst);
 
 		dst_vtx->coord[0] = srctoh16(src_vtx->coord[0]);
@@ -452,7 +443,7 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 		dst_vtx->coord[2] = srctoh16(src_vtx->coord[2]);
 		
 		dst_vtx->flag = src_vtx->flag;
-		dst_vtx->color= src_vtx->color;
+		dst_vtx->color = src_vtx->color;
 
 		dst_vtx->s = srctoh16(src_vtx->s);
 		dst_vtx->t = srctoh16(src_vtx->t);
@@ -468,15 +459,15 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 	uintptr_t ptr_dst_colors = 0;
 
 	if (dst_header->ptr_colours) {
-		ptr_src_colors = curpos_src = ALIGN8(dst_header->ptr_colours - src_roomoffset);
+		ptr_src_colors = curpos_src = ALIGN8(dst_header->ptr_colours - src_ofs);
 		ptr_dst_colors = curpos_dst + dst_roomoffset;
 
-		add_marker(curpos_src + src_roomoffset, curpos_dst + dst_roomoffset);
-		
+		add_marker(curpos_src + src_ofs, curpos_dst + dst_roomoffset);
+
 		uintptr_t col_end = numgdls > 0 != 0 ? gdls_addr[0] : infsize;
 		size_t col_len = col_end - curpos_src;
 
-		memcpy(dst + curpos_dst, src_roomgfxdata + curpos_src, col_len);
+		memcpy(dst + curpos_dst, src + curpos_src, col_len);
 		curpos_src += col_len;
 		curpos_dst += col_len;
 	}
@@ -488,8 +479,8 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 	m_SizeGDLs = 0;
 	uintptr_t gdlstart = curpos_dst;
 	for (size_t i = 0; i < numgdls; i++) {
-		add_marker(gdls_addr[i] + src_roomoffset, curpos_dst + dst_roomoffset);
-		curpos_dst = gbi_convert_gdl(dst, curpos_dst, src_roomgfxdata, gdls_addr[i], 1);
+		add_marker(gdls_addr[i] + src_ofs, curpos_dst + dst_roomoffset);
+		curpos_dst = gbi_convert_gdl(dst, curpos_dst, src, gdls_addr[i], 1);
 	}
 	m_SizeGDLs = curpos_dst - gdlstart;
 
@@ -501,8 +492,8 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 
 		if (block->type == BLOCK_LEAF) {
 			relink_ptr(&block->ptr_gdl);
-			uint32_t offset_vtx = block->ptr_vertices - ptr_src_vertices - src_roomoffset;
-			uint32_t offset_col = block->ptr_colours - ptr_src_colors - src_roomoffset;
+			uint32_t offset_vtx = block->ptr_vertices - ptr_src_vertices - src_ofs;
+			uint32_t offset_col = block->ptr_colours - ptr_src_colors - src_ofs;
 
 			block->ptr_vertices = ptr_dst_vertices + offset_vtx;
 			block->ptr_colours = ptr_dst_colors + offset_col;
@@ -519,122 +510,40 @@ static uint32_t convert_section1_roomgfxdata(uint8_t *dst, uint32_t dstpos, uint
 	relink_ptr(&dst_header->ptr_opablocks);
 	relink_ptr(&dst_header->ptr_xlublocks);
 
-	free(src_roomgfxdata);
 	return curpos_dst;
 }
 
-static uint32_t convert_section1(uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t srcpos, struct fileheader *n64_header)
+static uint32_t convert_section1(uint8_t *dst, uint8_t *src, uint32_t ofs)
 {
-	struct fileheader *host_header = (struct fileheader *) &dst[dstpos];
-	dstpos += sizeof(*host_header);
-
-	// Unzip primary
-	size_t infsize = srctoh32(n64_header->primary_infsize);
-	uint8_t *srcbin = calloc(1, infsize);
-	uint8_t *dstbin = calloc(1, infsize * 2);
-	uint32_t srcbinpos = 0;
-	uint32_t dstbinpos = 0;
-
-	int ret = rzip_inflate(srcbin, infsize, &src[srcpos], srctoh32(n64_header->primary_cmpsize));
-
-	if (ret < 0) {
-		fprintf(stderr, "Unable to inflate BG primary data: %d\n", ret);
-		exit(EXIT_FAILURE);
-	}
-
 	// Convert primary
-	struct host_primaryheader *host_primary_header = (struct host_primaryheader *) &dstbin[dstbinpos];
-	struct n64_primaryheader *n64_primary_header = (struct n64_primaryheader *) &srcbin[srcbinpos];
-	srcbinpos += sizeof(struct n64_primaryheader);
-	dstbinpos += sizeof(*host_primary_header);
+	uint32_t srcpos = 0;
+	uint32_t dstpos = 0;
+	struct host_primaryheader *host_primary_header = (struct host_primaryheader *) dst;
+	struct n64_primaryheader *n64_primary_header = (struct n64_primaryheader *) src;
+	srcpos += sizeof(struct n64_primaryheader);
+	dstpos += sizeof(*host_primary_header);
 
-	uint32_t src_bgcmds = srctodst32(n64_primary_header->ptr_bgcmds) - 0x0f000000;
+	uint32_t src_bgcmds = srctodst32(n64_primary_header->ptr_bgcmds) - ofs;
 
 	host_primary_header->unused1 = 0;
 	host_primary_header->unused2 = 0;
 
-	host_primary_header->ptr_rooms = dstbinpos + 0x0f000000;
-	convert_primary_rooms(dstbin, &dstbinpos, srcbin, &srcbinpos);
+	host_primary_header->ptr_rooms = dstpos + ofs;
+	convert_primary_rooms(dst, &dstpos, src, &srcpos);
 	
 	if (n64_primary_header->ptr_lights) {
-		host_primary_header->ptr_lights = dstbinpos + 0x0f000000;
-		srcbinpos = srctodst32(n64_primary_header->ptr_lights) - 0x0f000000;
-		convert_primary_lights(dstbin, &dstbinpos, srcbin, &srcbinpos, src_bgcmds);
+		host_primary_header->ptr_lights = dstpos + ofs;
+		srcpos = srctodst32(n64_primary_header->ptr_lights) - ofs;
+		convert_primary_lights(dst, &dstpos, src, &srcpos, src_bgcmds);
 	}
 
-	host_primary_header->ptr_portals = dstbinpos + 0x0f000000;
-	srcbinpos = srctodst32(n64_primary_header->ptr_portals) - 0x0f000000;
+	host_primary_header->ptr_portals = dstpos + ofs;
+	srcpos = srctodst32(n64_primary_header->ptr_portals) - ofs;
 	uint32_t src_portalvtxs, dst_portalvtxs;
-	convert_primary_portals(dstbin, &dstbinpos, srcbin, &srcbinpos, &src_portalvtxs, &dst_portalvtxs);
+	convert_primary_portals(dst, &dstpos, src, &srcpos, &src_portalvtxs, &dst_portalvtxs);
 
-	host_primary_header->ptr_bgcmds = dstbinpos + 0x0f000000;
-	srcbinpos = src_bgcmds;
-	convert_primary_bgcmds(dstbin, &dstbinpos, srcbin, &srcbinpos, dst_portalvtxs, src_portalvtxs);
-
-	// Convert room gfxdata
-	uint32_t n64_primary_diff = srctoh32(n64_header->primary_infsize) - srctoh32(n64_header->primary_cmpsize) - 0xc;
-	struct n64_room *n64_rooms = (struct n64_room *) &srcbin[sizeof(struct n64_primaryheader)];
-	struct host_room *host_rooms = (struct host_room *) &dstbin[sizeof(struct host_primaryheader)];
-	
-	uint8_t *roomgfxbuf = calloc(1, (size_t)1024 * 1024);
-	uint8_t *allcmp = calloc(1, (size_t)1024 * 1024);
-	int cmppos = 0;
-
-	host_rooms[0].ptr_gfxdata = 0;
-	for (int i = 1; i <= m_NumRooms; i++) {
-		host_rooms[i].ptr_gfxdata = dstbinpos + cmppos + 0x0f000000;
-
-		if (i == m_NumRooms) break;
-
-		uint32_t gfxdatacompsize = srctoh32(n64_rooms[i + 1].ptr_gfxdata) - srctoh32(n64_rooms[i].ptr_gfxdata);
-		srcpos = srctoh32(n64_rooms[i].ptr_gfxdata) - n64_primary_diff - 0x0f000000;
-		uint32_t src_roomoffset = srctoh32(n64_rooms[i].ptr_gfxdata);
-		uint32_t roomgfxbufpos = convert_section1_roomgfxdata(roomgfxbuf, 0, src+srcpos, gfxdatacompsize, src_roomoffset, dstbinpos + cmppos);
-
-		size_t ziplen = 0;
-		ret = rzip_deflate(&allcmp[cmppos], &ziplen, roomgfxbuf, roomgfxbufpos);
-		if (ret < 0) {
-			fprintf(stderr, "Unable to compress BG roomgfxdata for room 0x%x: %d\n", i, ret);
-			exit(EXIT_FAILURE);
-		}
-
-		ziplen = ALIGN16(ziplen);
-		cmppos += ziplen;
-
-		m_AllocSizes[i - 1] = roomgfxbufpos + (ziplen > m_SizeGDLs ? ziplen*4 : m_SizeGDLs*4);
-	}
-	
-	host_rooms[m_NumRooms+1].ptr_gfxdata = 0;
-
-	// Zip primary
-	size_t ziplen = 0;
-
-	printf("dstbin:%d = %p\n", __LINE__, dstbin);
-	printf("dstpos:%d = %d\n", __LINE__, dstpos);
-	printf("dstbinpos:%d = %d\n", __LINE__, dstbinpos);
-	ret = rzip_deflate(&dst[dstpos], &ziplen, dstbin, dstbinpos);
-
-	if (ret < 0) {
-		fprintf(stderr, "Unable to compress BG primary data: %d\n", ret);
-		exit(EXIT_FAILURE);
-	}
-
-	printf("ziplen:%d = %d\n", __LINE__, ziplen);
-	ziplen = ALIGN16(ziplen);
-	dstpos += ziplen;
-
-	// Paste compressed roomgfxdata after it
-	memcpy(&dst[dstpos], allcmp, cmppos);
-	dstpos += cmppos;
-
-	free(roomgfxbuf);
-	free(allcmp);
-	free(srcbin);
-	free(dstbin);
-
-	host_header->primary_infsize = dstbinpos;
-	host_header->section1_cmpsize = ziplen + cmppos;
-	host_header->primary_cmpsize = ziplen;
+	host_primary_header->ptr_bgcmds = dstpos + ofs;
+	convert_primary_bgcmds(dst, &dstpos, &src[src_bgcmds], dst_portalvtxs, src_portalvtxs);
 
 	return dstpos;
 }
@@ -691,102 +600,36 @@ static void convert_section3_lightcounts(uint8_t *dst, uint32_t *dstpos, uint8_t
 	*dstpos += (m_NumRooms - 1);
 }
 
-static uint32_t convert_zipped_section(int section_num, uint8_t *dst, uint32_t dstpos, uint8_t *src, uint32_t srcpos, struct sectionheader *n64_header)
+#ifdef PLATFORM_64BIT
+void preprocessBgSection1(uint8_t *data, uint32_t size, uint32_t ofs)
 {
-	struct sectionheader *host_header = (struct sectionheader *) &dst[dstpos];
-	dstpos += sizeof(*host_header);
+	uint8_t *dst = sysMemZeroAlloc(size);
 
-	// Unzip
-	size_t infsize = rzip_get_infsize(&src[srcpos]);
-	uint8_t *srcbin = calloc(1, infsize);
-	uint8_t *dstbin = calloc(1, infsize);
-	uint32_t srcbinpos = 0;
-	uint32_t dstbinpos = 0;
+	uint32_t newSize = convert_section1(dst, data, ofs);
 
-	int ret = rzip_inflate(srcbin, infsize, &src[srcpos], srctoh16(n64_header->cmpsize));
-
-	if (ret < 0) {
-		fprintf(stderr, "Unable to inflate BG section %d: %d\n", section_num, ret);
+	if (newSize > size) {
+		sysLogPrintf(LOG_ERROR, "overflow when trying to preprocess a bg file, size %d newsize %d", size, newSize);
 		exit(EXIT_FAILURE);
 	}
 
-	// Convert
-	if (section_num == 2) {
-		convert_section2_texturenums(dstbin, &dstbinpos, srcbin, &srcbinpos, infsize);
-	} else if (section_num == 3) {
-		convert_section3_bboxes(dstbin, &dstbinpos, srcbin, &srcbinpos);
-		convert_section3_allocsizes(dstbin, &dstbinpos, srcbin, &srcbinpos);
-		convert_section3_lightcounts(dstbin, &dstbinpos, srcbin, &srcbinpos);
-	}
+	memcpy(data, dst, newSize);
+	sysMemFree(dst);
+}
 
-	// Zip
-	size_t ziplen = 0;
+uint32_t preprocessBgRoom(uint8_t *data, uint32_t size, uint32_t room_ofs)
+{
+	size *= 2;
+	uint8_t *dst = sysMemZeroAlloc(size);
+	uint32_t newSize = convert_section1_roomgfxdata(dst, data, size, room_ofs);
 
-	ret = rzip_deflate(&dst[dstpos], &ziplen, dstbin, dstbinpos);
-
-	if (ret < 0) {
-		fprintf(stderr, "Unable to compress BG section %d: %d\n", section_num, ret);
+	if (newSize > size) {
+		sysLogPrintf(LOG_ERROR, "overflow when trying to preprocess a bg room, size %d newsize %d", size, newSize);
 		exit(EXIT_FAILURE);
 	}
 
-	ziplen = ALIGN16(ziplen);
+	memcpy(data, dst, newSize);
+	sysMemFree(dst);
 
-	host_header->infsize = dstbinpos;
-	host_header->cmpsize = ziplen;
-
-	dstpos += ziplen;
-
-	free(srcbin);
-	free(dstbin);
-
-	return dstpos;
-}
-
-void extract_file_bg(char *filename, uint32_t romoffset, size_t len)
-{
-	if (0) {
-		if (strcmp(filename, "bgdata/bg_mp5.seg") != 0) {
-			return;
-		}
-	}
-
-	if (strcmp(filename, "ob/ob_mid.seg") == 0) {
-		return;
-	}
-
-	size_t dstlen = len * 2;
-	uint8_t *dst = calloc(1, dstlen);
-	uint32_t dstpos = 0;
-	m_NumPtrMarkers = 0;
-	m_SizeGDLs = 0;
-	gbi_reset();
-
-	printf("--------------------------------------------------------------------------------\n");
-	printf("-- %s\n", filename);
-
-	uint8_t *src = &g_Rom[romoffset];
-	struct fileheader *n64_fileheader = (struct fileheader *) src;
-	uint32_t n64_section2_start = sizeof(*n64_fileheader) + srctoh32(n64_fileheader->section1_cmpsize);
-	struct sectionheader *n64_section2header = (struct sectionheader *) &src[n64_section2_start];
-	uint32_t n64_section3_start = n64_section2_start + sizeof(*n64_section2header) + (srctoh16(n64_section2header->cmpsize) & 0x7fff);
-	struct sectionheader *n64_section3header = (struct sectionheader *) &src[n64_section3_start];
-
-	printf("section2_start 0x%x\n", n64_section2_start);
-	printf("section3_start 0x%x\n", n64_section3_start);
-
-	dstpos = convert_section1(dst, dstpos, src, sizeof(*n64_fileheader), n64_fileheader);
-	dstpos = convert_zipped_section(2, dst, dstpos, src, n64_section2_start + sizeof(*n64_section2header), n64_section2header);
-	dstpos = convert_zipped_section(3, dst, dstpos, src, n64_section3_start + sizeof(*n64_section3header), n64_section3header);
-	dstpos = ALIGN16(dstpos);
-
-	sysLogPrintf(LOG_NOTE, "%s	%d	%d	%.3f", filename, len, dstpos, (f32)dstpos / len);
-
-	char outfilename[1024];
-	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(dst, dstpos, 1, fp);
-	fclose(fp);
-
-	free(dst);
+	return newSize;
 }
+#endif

port/pdextract/ext_file_lang.c

@@ -2,6 +2,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "common.h"
+#include "system.h"
 
 static int get_next_nonzero(uint8_t *src, uint32_t offset, size_t len)
 {
@@ -30,28 +31,16 @@ static int get_num_strings(uint8_t *src, size_t len)
 	return 1;
 }
 
-void extract_file_lang(char *filename, uint32_t romoffset, size_t srclen)
+uint32_t convert_lang_file(uint8_t *dst, uint8_t *src, size_t srclen)
 {
-	// Unzip it
-	size_t infsize = rzip_get_infsize(&g_Rom[romoffset]);
-	uint8_t *src = malloc(infsize);
-
-	int ret = rzip_inflate(src, infsize, &g_Rom[romoffset], srclen);
-
-	if (ret < 0) {
-		fprintf(stderr, "%s: Unable to inflate file: %d\n", filename, ret);
-		exit(EXIT_FAILURE);
-	}
-
 	// Convert it
 	uint32_t *src_offsets = (uint32_t *) src;
 	size_t src_ptr_table_len = srctoh32(src_offsets[0]);
-	size_t src_str_table_len = infsize - src_ptr_table_len;
+	size_t src_str_table_len = srclen - src_ptr_table_len;
 	int num_strings = get_num_strings(src, srclen);
 	size_t dst_ptr_table_len = num_strings * sizeof(uintptr_t);
 	size_t dst_str_table_len = src_str_table_len;
 
-	uint8_t *dst = calloc(1, dst_ptr_table_len + dst_str_table_len + 0x20);
 	int cur_dst_offset = dst_ptr_table_len;
 
 	uintptr_t *dst_offsets = (uintptr_t *) dst;
@@ -67,7 +56,7 @@ void extract_file_lang(char *filename, uint32_t romoffset, size_t srclen)
 			uint32_t end = get_next_nonzero(src, (i + 1) * 4, num_strings * 4);
 
 			if (!end) {
-				end = infsize;
+				end = srclen;
 			}
 
 			size_t len = end - src_offset;
@@ -81,30 +70,24 @@ void extract_file_lang(char *filename, uint32_t romoffset, size_t srclen)
 		}
 	}
 
-	cur_dst_offset = ALIGN16(cur_dst_offset);
+	return ALIGN16(cur_dst_offset);
+}
 
-	// Zip it
-	uint8_t *zipped = calloc(1, cur_dst_offset);
-	size_t ziplen = cur_dst_offset;
+uint8_t *preprocessLangFile_x64(uint8_t *data, uint32_t size, uint32_t *outSize) {
+	uint32_t newSizeEstimated = (uint32_t)(size * 1.3);
+	uint8_t *dst = sysMemZeroAlloc(newSizeEstimated);
 
-	ret = rzip_deflate(zipped, &ziplen, dst, cur_dst_offset);
+	uint32_t newSize = convert_lang_file(dst, data, size);
 
-	if (ret < 0) {
-		fprintf(stderr, "%s: Unable to compress file: %d\n", filename, ret);
+	if (newSize > newSizeEstimated) {
+		sysLogPrintf(LOG_ERROR, "overflow when trying to preprocess a lang file, size %d newsize %d", size, newSize);
 		exit(EXIT_FAILURE);
 	}
 
-	ziplen = ALIGN16(ziplen);
+	memcpy(data, dst, newSize);
+	sysMemFree(dst);
 
-	// Write it
-	char outfilename[1024];
-	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
+	*outSize = newSize;
 
-	FILE *fp = openfile(outfilename);
-	fwrite(zipped, ziplen, 1, fp);
-	fclose(fp);
-
-	free(zipped);
-	free(dst);
-	free(src);
+	return 0;
 }

port/pdextract/ext_file_model.c

@@ -4,11 +4,6 @@
 #include "common.h"
 #include "system.h"
 
-#include <PR/ultratypes.h>
-
-#undef near
-#undef far
-
 enum contenttype {
 	/* 0*/ CT_MODELDEF,
 	/* 1*/ CT_NODE,
@@ -923,21 +918,6 @@ static void relink_pointers(uint8_t *dst, uint8_t *src)
 	}
 }
 
-int calc_align(uint32_t value)
-{
-	if ((value % 16) == 0) {
-		return 16;
-	} else if ((value % 8) == 0) {
-		return 8;
-	} else if ((value % 4) == 0) {
-		return 4;
-	} else if ((value % 2) == 0) {
-		return 2;
-	} else {
-		return 1;
-	}
-}
-
 static int convert_model(uint8_t *dst, uint8_t *src, uint32_t srclen)
 {
 	uint32_t dstpos;
@@ -951,54 +931,24 @@ static int convert_model(uint8_t *dst, uint8_t *src, uint32_t srclen)
 	return dstpos;
 }
 
-void extract_file_model(char *filename, uint32_t romoffset, size_t len)
+uint8_t *preprocessModelFile_x64(uint8_t *data, uint32_t size, uint32_t *outSize)
 {
 	gbi_reset();
-	// Unzip it
-	size_t infsize = rzip_get_infsize(&g_Rom[romoffset]);
-	uint8_t *src = malloc(infsize);
 
-	int ret = rzip_inflate(src, infsize, &g_Rom[romoffset], len);
+	uint32_t newSizeEstimated = (uint32_t)(size * 1.7);
+	uint8_t *dst = sysMemZeroAlloc(newSizeEstimated);
 
-	if (ret < 0) {
-		fprintf(stderr, "%s: Unable to inflate file: %d\n", filename, ret);
+	uint32_t newSize = convert_model(dst, data, size);
+
+	if (newSize > newSizeEstimated) {
+		sysLogPrintf(LOG_ERROR, "overflow when trying to preprocess model, size %d newsize %d", size, newSize);
 		exit(EXIT_FAILURE);
 	}
 
-	// Convert it
-	uint8_t *dst = calloc(1, infsize * 2);
+	memcpy(data, dst, newSize);
+	sysMemFree(dst);
 
-	int dstpos = convert_model(dst, src, infsize);
+	*outSize = newSize;
 
-	if (dstpos > infsize * 2) {
-		fprintf(stderr, "%s: overflow\n", filename);
-		exit(EXIT_FAILURE);
-	}
-
-	// Zip it
-	uint8_t *zipped = calloc(1, dstpos);
-	size_t ziplen = dstpos;
-
-	ret = rzip_deflate(zipped, &ziplen, dst, dstpos);
-
-	if (ret < 0) {
-		fprintf(stderr, "%s: Unable to compress file: %d\n", filename, ret);
-		exit(EXIT_FAILURE);
-	}
-
-	ziplen = ALIGN16(ziplen);
-
-	// Write it
-	char outfilename[1024];
-	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
-
-	sysLogPrintf(LOG_NOTE, "%s	%d	%d	%.3f", filename, infsize, dstpos, (f32)dstpos/infsize);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(zipped, ziplen, 1, fp);
-	fclose(fp);
-
-	free(zipped);
-	free(dst);
-	free(src);
+	return 0;
 }

port/pdextract/ext_file_pads.c

@@ -2,6 +2,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "common.h"
+#include "system.h"
 
 #define PADFLAG_INTPOS          0x0001
 #define PADFLAG_UPALIGNTOX      0x0002
@@ -276,7 +277,7 @@ static int convert_cover(uint8_t *dst, int dstpos, uint8_t *src, int srcpos, int
 	return dstpos;
 }
 
-static int convert_binary(uint8_t *dst, uint8_t *src)
+static int convert_pads_file(uint8_t *dst, uint8_t *src)
 {
 	int dstpos = 0;
 	struct n64_header *n64_header = (struct n64_header *) src;
@@ -307,45 +308,21 @@ static int convert_binary(uint8_t *dst, uint8_t *src)
 	return dstpos;
 }
 
-void extract_file_pads(char *filename, uint32_t romoffset, size_t len)
-{
-	// Unzip it
-	size_t infsize = rzip_get_infsize(&g_Rom[romoffset]);
-	uint8_t *src = malloc(infsize);
+u8* preprocessPadsFile_x64(u8 *data, u32 size, u32 *outSize) {
+	u32 newSizeEstimated = (u32)(size * 1.7);
+	u8* dst = sysMemZeroAlloc(newSizeEstimated);
 
-	int ret = rzip_inflate(src, infsize, &g_Rom[romoffset], len);
+	u32 newSize = convert_pads_file(dst, data);
 
-	if (ret < 0) {
-		fprintf(stderr, "%s: Unable to inflate file: %d\n", filename, ret);
+	if (newSize > newSizeEstimated) {
+		sysLogPrintf(LOG_ERROR, "overflow when trying to preprocess a pads file, size %d newsize %d", size, newSize);
 		exit(EXIT_FAILURE);
 	}
 
-	// Convert it
-	uint8_t *dst = calloc(1, infsize * 2);
-	int dstpos = convert_binary(dst, src);
+	memcpy(data, dst, newSize);
+	sysMemFree(dst);
 
-	dstpos = ALIGN16(dstpos);
+	*outSize = newSize;
 
-	// Zip it
-	uint8_t *zipped = calloc(1, dstpos);
-	size_t ziplen = dstpos;
-
-	ret = rzip_deflate(zipped, &ziplen, dst, dstpos);
-
-	if (ret < 0) {
-		fprintf(stderr, "%s: Unable to compress file: %d\n", filename, ret);
-		exit(EXIT_FAILURE);
-	}
-
-	// Write it
-	char outfilename[1024];
-	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(zipped, ALIGN16(ziplen), 1, fp);
-	fclose(fp);
-
-	free(src);
-	free(dst);
-	free(zipped);
+	return 0;
 }

port/pdextract/ext_file_tiles.c

@@ -2,6 +2,8 @@
 #include <stdlib.h>
 #include <string.h>
 #include "common.h"
+#include "system.h"
+
 
 struct tile {
 	uint8_t type;
@@ -17,19 +19,8 @@ struct tile {
 	uint16_t floorcol;
 };
 
-void extract_file_tiles(char *filename, uint32_t romoffset, size_t len)
+uint32_t convert_tiles(uint8_t *dst, uint8_t *src, size_t srclen)
 {
-	// Unzip it
-	size_t infsize = rzip_get_infsize(&g_Rom[romoffset]);
-	uint8_t *src = malloc(infsize);
-
-	int ret = rzip_inflate(src, infsize, &g_Rom[romoffset], len);
-
-	if (ret < 0) {
-		fprintf(stderr, "%s: Unable to inflate file: %d\n", filename, ret);
-		exit(EXIT_FAILURE);
-	}
-
 	int num_rooms = srctoh32(*(uint32_t *) src);
 
 	size_t src_ptr_table_len = (num_rooms + 1) * 4;
@@ -37,13 +28,11 @@ void extract_file_tiles(char *filename, uint32_t romoffset, size_t len)
 
 	uint32_t *src_offsets = (uint32_t *) &src[4];
 
-	size_t data_len = infsize - src_ptr_table_len - 4;
-
-	uint8_t *dst = calloc(1, dst_ptr_table_len + data_len + 0x20);
+	size_t data_len = srclen - src_ptr_table_len - 4;
 
 	*(uint32_t *) dst = htodst32(num_rooms);
 
-	uint32_t*dst_offsets = (uint32_t*) (dst + sizeof(uint32_t));
+	uint32_t *dst_offsets = (uint32_t *) (dst + sizeof(uint32_t));
 	uint32_t cur_dst_offset = dst_ptr_table_len + sizeof(uint32_t);
 
 	uint32_t cur_src_offset = srctoh32(src_offsets[0]);
@@ -92,31 +81,24 @@ void extract_file_tiles(char *filename, uint32_t romoffset, size_t len)
 		}
 	}
 
-	cur_dst_offset = ALIGN16(cur_dst_offset);
+	return ALIGN16(cur_dst_offset);
+}
 
-	// Zip it
-	uint8_t *zipped = calloc(1, cur_dst_offset);
-	size_t ziplen = cur_dst_offset;
+uint8_t *preprocessTilesFile_x64(uint8_t *data, uint32_t size, uint32_t *outSize)
+{
+	uint8_t *dst = sysMemZeroAlloc(size);
 
-	ret = rzip_deflate(zipped, &ziplen, dst, cur_dst_offset);
+	uint32_t newSize = convert_tiles(dst, data, size);
 
-	if (ret < 0) {
-		fprintf(stderr, "%s: Unable to compress file: %d\n", filename, ret);
+	if (newSize > size) {
+		sysLogPrintf(LOG_ERROR, "overflow when trying to preprocess a tiles file, size %d newsize %d", size, newSize);
 		exit(EXIT_FAILURE);
 	}
 
-	ziplen = ALIGN16(ziplen);
+	memcpy(data, dst, newSize);
+	sysMemFree(dst);
 
-	// Write it
-	char outfilename[1024];
+	*outSize = newSize;
 
-	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(zipped, ziplen, 1, fp);
-	fclose(fp);
-
-	free(zipped);
-	free(dst);
-	free(src);
+	return 0;
 }

port/pdextract/ext_files.c

@@ -1,100 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "common.h"
-
-void extract_file_audio(char *filename, uint32_t romoffset, size_t len);
-void extract_file_bg(char *filename, uint32_t romoffset, size_t len);
-void extract_file_lang(char *filename, uint32_t romoffset, size_t len);
-void extract_file_model(char *filename, uint32_t romoffset, size_t len);
-void extract_file_pads(char *filename, uint32_t romoffset, size_t len);
-void extract_file_tiles(char *filename, uint32_t romoffset, size_t len);
-
-static void extract_file(int filenum, uint32_t romoffset, size_t len)
-{
-	uint32_t name_uint32_table = 0;
-
-	switch (g_RomVersion) {
-	case ROMVERSION_NTSC10:
-	case ROMVERSION_NTSCFINAL:
-		name_uint32_table = 0x1d5ca00;
-		break;
-	case ROMVERSION_PALFINAL:
-		name_uint32_table = 0x1d534e0;
-		break;
-	case ROMVERSION_JPNFINAL:
-		name_uint32_table = 0x1d58a20;
-		break;
-	}
-
-	uint32_t name_offset = name_uint32_table + srctoh32(*(uint32_t *) &g_Rom[name_uint32_table + filenum * 4]);
-
-	char *filename = (char *) &g_Rom[name_offset];
-
-	if (strstr(filename, ".seg")) {
-		extract_file_bg(filename, romoffset, len);
-	} else if (strstr(filename, "padsZ")) {
-		extract_file_pads(filename, romoffset, len);
-	} else if (strstr(filename, "tilesZ")) {
-		extract_file_tiles(filename, romoffset, len);
-	} else if (filename[0] == 'A') {
-		extract_file_audio(filename, romoffset, len);
-	} else if (filename[0] == 'C' || filename[0] == 'G' || filename[0] == 'P') {
-		extract_file_model(filename, romoffset, len);
-	} else if (filename[0] == 'L') {
-		extract_file_lang(filename, romoffset, len);
-	} else if (filename[0] == 'U') {
-		// Setup file - don't extract
-	} else {
-		fprintf(stderr, "Unrecognised file type: '%s'\n", filename);
-		exit(EXIT_FAILURE);
-	}
-}
-
-void extract_files(void)
-{
-	size_t infsize = rzip_get_infsize(&g_Rom[0x39850]);
-	uint8_t *data = malloc(infsize);
-
-	int ret = rzip_inflate(data, infsize, &g_Rom[0x39850], infsize);
-
-	if (ret < 0) {
-		fprintf(stderr, "Unable to inflate data segment: %d\n", ret);
-		exit(EXIT_FAILURE);
-	}
-
-	uint32_t offset = 0;
-
-	switch (g_RomVersion) {
-	case ROMVERSION_NTSC10:
-	case ROMVERSION_NTSCFINAL:
-		offset = 0x28080;
-		break;
-	case ROMVERSION_PALFINAL:
-		offset = 0x28910;
-		break;
-	case ROMVERSION_JPNFINAL:
-		offset = 0x28800;
-		break;
-	}
-
-	uint32_t *offsets = (uint32_t *) &data[offset];
-	int i = 0;
-
-	while (1) {
-		uint32_t curr = srctoh32(offsets[i]);
-		uint32_t next = srctoh32(offsets[i + 1]);
-
-		if (next == 0) {
-			break;
-		}
-
-		if (curr) {
-			extract_file(i, curr, next - curr);
-		}
-
-		i++;
-	}
-
-	free(data);
-}

port/pdextract/ext_fonts.c

@@ -4,6 +4,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "common.h"
+#include "system.h"
 
 struct n64fontchar {
 	uint8_t index;
@@ -41,15 +42,33 @@ struct hostfontcharjpn {
 	uintptr_t pixeldata;
 };
 
-static void extract_font(const char *filename, uint32_t romoffset, size_t srclen, int num_chars)
+struct fontdef {
+	const char *name;
+	uint32_t offsets[3];
+	int pal_extra_chars;
+};
+
+u8* preprocessFont_x64(u8 *src, u32 srclen, u32 *outSize)
 {
+	int num_chars = 94;
+#if VERSION == VERSION_PAL_FINAL
+	// PAL has more characters in these fonts
+	extern u8* _fonthandelgothicsmSegmentRomStart;
+	extern u8* _fonthandelgothicxsSegmentRomStart;
+	extern u8* _fonthandelgothicmdSegmentRomStart;
+	if (src == _fonthandelgothicsmSegmentRomStart
+		|| src == _fonthandelgothicxsSegmentRomStart
+		|| src == _fonthandelgothicmdSegmentRomStart) {
+		numchars = 135;
+	}
+#endif
+
 	size_t dstlen = srclen - (num_chars * sizeof(struct n64fontchar)) + (num_chars * sizeof(struct hostfontchar)) + 0x20;
-	uint8_t *src = &g_Rom[romoffset];
-	uint8_t *dst = calloc(1, dstlen);
+	uint8_t* dst = sysMemZeroAlloc(dstlen);
 
 	// Kerning table
-	int32_t *src_kerning_table = (int32_t *) src;
-	int32_t *dst_kerning_table = (int32_t *) dst;
+	int32_t *src_kerning_table = (int32_t *)src;
+	int32_t *dst_kerning_table = (int32_t *)dst;
 
 	for (int i = 0; i < 13 * 13; i++) {
 		dst_kerning_table[i] = srctodst32(src_kerning_table[i]);
@@ -59,7 +78,7 @@ static void extract_font(const char *filename, uint32_t romoffset, size_t srclen
 	uint32_t dst_offset = ALIGN(13 * 13 * 4, sizeof(uintptr_t));
 
 	// Character table
-	if (g_RomVersion == ROMVERSION_JPNFINAL) {
+#if VERSION == VERSION_JPN_FINAL
 		struct n64fontcharjpn *src_char_table = (struct n64fontcharjpn *) &src[src_offset];
 		struct hostfontcharjpn *dst_char_table = (struct hostfontcharjpn *) &dst[dst_offset];
 		uint32_t diff = (dst_offset + num_chars * sizeof(struct hostfontcharjpn)) - (src_offset + num_chars * sizeof(struct n64fontcharjpn));
@@ -75,7 +94,7 @@ static void extract_font(const char *filename, uint32_t romoffset, size_t srclen
 
 		src_offset += num_chars * sizeof(struct n64fontcharjpn);
 		dst_offset += num_chars * sizeof(struct hostfontcharjpn);
-	} else {
+#else
 		struct n64fontchar *src_char_table = (struct n64fontchar *) &src[src_offset];
 		struct hostfontchar *dst_char_table = (struct hostfontchar *) &dst[dst_offset];
 		uint32_t diff = (dst_offset + num_chars * sizeof(struct hostfontchar)) - (src_offset + num_chars * sizeof(struct n64fontchar));
@@ -91,67 +110,13 @@ static void extract_font(const char *filename, uint32_t romoffset, size_t srclen
 
 		src_offset += num_chars * sizeof(struct n64fontchar);
 		dst_offset += num_chars * sizeof(struct hostfontchar);
-	}
+#endif
 
 	// Pixel data
-	size_t len = srclen - src_offset;
-
+	u32 len = srclen - src_offset;
 	memcpy(&dst[dst_offset], &src[src_offset], len);
 
-	dst_offset = ALIGN16(dst_offset + len);
+	if (outSize) *outSize = ALIGN16(dst_offset + len);
 
-	// Write it
-	char outfilename[1024];
-	sprintf(outfilename, "%s/segs/font%s", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(dst, dst_offset, 1, fp);
-	fclose(fp);
-
-	free(dst);
-}
-
-struct fontdef {
-	const char *name;
-	uint32_t offsets[3];
-	int pal_extra_chars;
-};
-
-void extract_fonts(void)
-{
-	struct fontdef defs[] = {
-        //                      ntsc       pal       jpn
-		{"bankgothic",     {0x7f2390, 0x7ddb90, 0x7e1a20}, 0},
-		{"zurich",         {0x7f4930, 0x7e0130, 0x7e3fc0}, 0},
-        {"tahoma",         {0x7f7860, 0x7e3060, 0x7e6ef0}, 0},
-        {"numeric",        {0x7f8b20, 0x7e4320, 0x7e81b0}, 0},
-        {"handelgothicsm", {0x7f9d30, 0x7e5530, 0x7e93c0}, 1},
-        {"handelgothicxs", {0x7fbfb0, 0x7e87b0, 0x7ec640}, 1},
-        {"handelgothicmd", {0x7fdd80, 0x7eae20, 0x7eecb0}, 1},
-        {"handelgothiclg", {0x8008e0, 0x7eee70, 0x7f2d00}, 0},
-        {"ocramd",         {0x803da0, 0x7f2330, 0x7f61c0}, 0},
-        {"ocralg",         {0x806ac0, 0x7f5050, 0x7f8ee0}, 0},
-        {"end",            {0x80a250, 0x7f87e0, 0x7fc670}, 0},
-    };
-
-	int index = 0;
-
-	switch (g_RomVersion) {
-	case ROMVERSION_NTSC10:
-	case ROMVERSION_NTSCFINAL:
-		index = 0;
-		break;
-	case ROMVERSION_PALFINAL:
-		index = 1;
-		break;
-	case ROMVERSION_JPNFINAL:
-		index = 2;
-		break;
-	}
-
-	for (int i = 0; i < ARRAYCOUNT(defs) - 1; i++) {
-		int num_chars = g_RomVersion == ROMVERSION_PALFINAL && defs[i].pal_extra_chars ? 135 : 94;
-		size_t len = defs[i + 1].offsets[index] - defs[i].offsets[index];
-		extract_font(defs[i].name, defs[i].offsets[index], len, num_chars);
-	}
+	return dst;
 }

port/pdextract/ext_jpnfonts.c

@@ -1,32 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "common.h"
-
-static void extract_jpnfont(const char *filename, uint32_t romoffset, size_t len)
-{
-	char outfilename[1024];
-	sprintf(outfilename, "%s/segs/%s", g_OutPath, filename);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(&g_Rom[romoffset], len, 1, fp);
-	fclose(fp);
-}
-
-void extract_jpnfonts(void)
-{
-	switch (g_RomVersion) {
-	case ROMVERSION_NTSC10:
-	case ROMVERSION_NTSCFINAL:
-		extract_jpnfont("fontjpnsingle", 0x194440, 0xb020);
-		extract_jpnfont("fontjpnmulti", 0x19fb40, 0x1a80);
-		break;
-	case ROMVERSION_PALFINAL:
-		extract_jpnfont("fontjpnsingle", 0x17fc40, 0xb010);
-		extract_jpnfont("fontjpnmulti", 0x18b340, 0x1a80);
-		break;
-	case ROMVERSION_JPNFINAL:
-		extract_jpnfont("fontjpn", 0x178c40, 0x17920);
-		break;
-	}
-}

port/pdextract/ext_setups.c

@@ -2,21 +2,21 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "types.h"
+
 #include "common.h"
 #include "system.h"
 #include "assert.h"
 #include "setups_decl.h"
 
-#include "types.h"
 
-static inline f32 convF32(f32 *dst, f32 src) { *(u32*)dst = srctoh32(*(u32*)&src); }
-static inline u32 convU32(u32 *dst, u32 src) { *dst = srctoh32(src); }
-static inline s32 convS32(s32 *dst, s32 src) { *dst = srctoh32(src); }
-static inline u16 convU16(u16 *dst, u32 src) { *dst = srctoh16(src); }
-static inline s16 convS16(s16 *dst, s32 src) { *dst = srctoh16(src); }
-static inline u8 cpyByte(u8 *dst, u8 src) { *dst = src; }
-static inline void* convPtr(void** dst, u32 src) { *dst = srctoh32(src); }
-static inline struct coord convCoord(struct coord* dst, struct n64_coord src) { convF32(&dst->x, src.x); convF32(&dst->y, src.y); convF32(&dst->z, src.z); }
+static inline void convF32(f32 *dst, f32 src) { *(u32*)dst = srctoh32(*(u32*)&src); }
+static inline void convU32(u32 *dst, u32 src) { *dst = srctoh32(src); }
+static inline void convS32(s32 *dst, s32 src) { *dst = srctoh32(src); }
+static inline void convU16(u16 *dst, u32 src) { *dst = srctoh16(src); }
+static inline void convS16(s16 *dst, s32 src) { *dst = srctoh16(src); }
+static inline void cpyByte(u8 *dst, u8 src) { *dst = src; }
+static inline void convCoord(struct coord* dst, struct n64_coord src) { convF32(&dst->x, src.x); convF32(&dst->y, src.y); convF32(&dst->z, src.z); }
 static u32 convUnk(void* x, u32 y) { __debugbreak(); }
 
 #define PD_CONV_VAL(dst, src) _Generic((dst), \
@@ -1231,54 +1231,21 @@ void test_conv_functions()
 	PD_CONV_ARRAY2D(dst8, src8);
 }
 
-void extract_file_setups(char* filename, u32 romoffset, size_t len)
-{
-	// Unzip it
-	size_t infsize = rzip_get_infsize(&g_Rom[romoffset]);
-	u8* src = malloc(infsize);
+u8 *preprocessSetupFile_x64(u8 *data, u32 size, u32 *outSize) { 
+	u32 newSizeEstimated = (u32)(size * 3.3);
+	u8 *dst = sysMemZeroAlloc(newSizeEstimated);
 
-	int ret = rzip_inflate(src, infsize, &g_Rom[romoffset], len);
+	u32 newSize = convert_setup(dst, data, size);
 
-	if (ret < 0) {
-		sysLogPrintf(LOG_ERROR, "%s: Unable to inflate file: %d", filename, ret);
+	if (newSize > newSizeEstimated) {
+		sysLogPrintf(LOG_ERROR, "overflow when trying to preprocess a model file, size %d newsize %d", size, newSize);
 		exit(EXIT_FAILURE);
 	}
 
-	// Convert it
-	u8* dst = calloc(1, infsize * 4);
+	memcpy(data, dst, newSize);
+	sysMemFree(dst);
 
-	int* mark = strstr(filename, "mp_setupdam");
-	u32 dstpos = convert_setup(dst, src, infsize);
+	*outSize = newSize;
 
-	if (dstpos > infsize * 4) {
-		sysLogPrintf(LOG_ERROR, "%s: overflow", filename);
-		exit(EXIT_FAILURE);
-	}
-
-	sysLogPrintf(LOG_NOTE, "%s	%d	%d	%.3f", filename, infsize, dstpos, (f32)dstpos / infsize);
-
-	// Zip it
-	u8* zipped = calloc(1, dstpos);
-	size_t ziplen = dstpos;
-
-	ret = rzip_deflate(zipped, &ziplen, dst, dstpos);
-
-	if (ret < 0) {
-		sysLogPrintf(LOG_ERROR, "%s: Unable to compress file: %d", filename, ret);
-		exit(EXIT_FAILURE);
-	}
-
-	ziplen = ALIGN16(ziplen);
-
-	// Write it
-	char outfilename[1024];
-	sprintf(outfilename, "%s/files/%s", g_OutPath, filename);
-
-	FILE* fp = openfile(outfilename);
-	fwrite(zipped, ziplen, 1, fp);
-	fclose(fp);
-
-	free(zipped);
-	free(dst);
-	free(src);
-}
+	return 0;
+}

port/pdextract/ext_textures.c

@@ -1,76 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "common.h"
-
-struct texture {
-	uint32_t soundsurfacetype : 4;
-	uint32_t surfacetype : 4;
-	uint32_t dataoffset : 24;
-	uint32_t unk04;
-};
-
-static void extract_textures_data(uint32_t romoffset, size_t len)
-{
-	char outfilename[1024];
-	sprintf(outfilename, "%s/textures_data.bin", g_OutPath);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(&g_Rom[romoffset], len, 1, fp);
-	fclose(fp);
-}
-
-static void extract_textures_table(uint32_t romoffset, size_t len)
-{
-	int num_textures = len / sizeof(struct texture);
-	uint8_t *dst = calloc(1, len);
-
-	struct texture *n64texes = (struct texture *) &g_Rom[romoffset];
-	struct texture *hosttexes = (struct texture *) dst;
-
-	for (int i = 0; i < num_textures; i++) {
-		hosttexes[i].soundsurfacetype = n64texes[i].soundsurfacetype;
-		hosttexes[i].surfacetype = n64texes[i].surfacetype;
-		hosttexes[i].dataoffset = srctodst32(n64texes[i].dataoffset);
-		hosttexes[i].unk04 = 0;
-	}
-
-	// Write it
-	char outfilename[1024];
-	sprintf(outfilename, "%s/textures_table.bin", g_OutPath);
-
-	FILE *fp = openfile(outfilename);
-	fwrite(dst, len, 1, fp);
-	fclose(fp);
-
-	free(dst);
-}
-
-void extract_textures(void)
-{
-	uint32_t data_offset = 0;
-	uint32_t table_offset = 0;
-	uint32_t end = 0;
-
-	switch (g_RomVersion) {
-	case ROMVERSION_NTSC10:
-	case ROMVERSION_NTSCFINAL:
-		data_offset = 0x1d65f40;
-		table_offset = 0x1ff7ca0;
-		end = 0x1ffea20;
-		break;
-	case ROMVERSION_PALFINAL:
-		data_offset = 0x1d5ca20;
-		table_offset = 0x1fee780;
-		end = 0x1ff5500;
-		break;
-	case ROMVERSION_JPNFINAL:
-		data_offset = 0x1d61f90;
-		table_offset = 0x1ff68f0;
-		end = 0x1ffd6b0;
-		break;
-	}
-
-	extract_textures_data(data_offset, table_offset - data_offset);
-	extract_textures_table(table_offset, end - table_offset);
-}

port/pdextract/main.c

@@ -1,173 +0,0 @@
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "common.h"
-
-#define ROM_SIZE (1024 * 1024 * 32)
-
-uint8_t *g_Rom = NULL;
-enum romversion g_RomVersion = 0;
-char g_OutPath[1024] = {'\0'};
-
-void extract_anims(void);
-void extract_audio(void);
-void extract_banners(void);
-void extract_files(void);
-void extract_fonts(void);
-void extract_jpnfonts(void);
-void extract_textures(void);
-
-/**
- * We allow modded ROMs to be used, so we can't simply compare the ROM checksum.
- */
-static enum romversion identify_rom_version(uint8_t *rom)
-{
-	char game_code[5];
-	game_code[0] = rom[0x3b];
-	game_code[1] = rom[0x3c];
-	game_code[2] = rom[0x3d];
-	game_code[3] = rom[0x3e];
-	game_code[4] = '\0';
-
-	char version_byte = rom[0x3f];
-
-	if (strncmp((char *) &rom[0x20], "Perfect Dark DBGNTSC", 20) == 0) {
-		return ROMVERSION_NTSCBETA;
-	}
-
-	if (strcmp(game_code, "NPDJ") == 0) {
-		return ROMVERSION_JPNFINAL;
-	}
-
-	if (strcmp(game_code, "NPDE") == 0) {
-		if (version_byte == 1) {
-			return ROMVERSION_NTSCFINAL;
-		}
-
-		if (version_byte == 0) {
-			return ROMVERSION_NTSC10;
-		}
-	}
-
-	if (strcmp(game_code, "NPDP") == 0) {
-		if (rom[0x100f] == 0xe0) {
-			return ROMVERSION_PALBETA;
-		}
-
-		if (rom[0x100f] == 0xf0) {
-			return ROMVERSION_PALFINAL;
-		}
-	}
-
-	return ROMVERSION_UNKNOWN;
-}
-
-static void flip_pairs(uint8_t *rom)
-{
-	uint16_t *ptr = (uint16_t *) rom;
-
-	for (int i = 0; i < ROM_SIZE / 2; i++) {
-		ptr[i] = __bswap_16(ptr[i]);
-	}
-}
-
-static void flip_words(uint8_t *rom)
-{
-	uint32_t *ptr = (uint32_t *) rom;
-
-	for (int i = 0; i < ROM_SIZE / 4; i++) {
-		ptr[i] = __bswap_32(ptr[i]);
-	}
-}
-
-static void extract_rom(char *romfile)
-{
-	FILE *fp = fopen(romfile, "rb");
-
-	if (fp == NULL) {
-		fprintf(stderr, "Unable to open '%s' for reading.\n", romfile);
-		exit(EXIT_FAILURE);
-	}
-
-	fseek(fp, 0, SEEK_END);
-
-	if (ftell(fp) != ROM_SIZE) {
-		fprintf(stderr, "The ROM size is not 32MB. Is this a Perfect Dark ROM?\n");
-		exit(EXIT_FAILURE);
-	}
-
-	fseek(fp, 0, SEEK_SET);
-
-	g_Rom = malloc(ROM_SIZE);
-	fread(g_Rom, 1, ROM_SIZE, fp);
-	fclose(fp);
-
-	uint32_t magic = (g_Rom[0] << 24) | (g_Rom[1] << 16) | (g_Rom[2] << 8) | g_Rom[3];
-
-	switch (magic) {
-	case 0x80371240:
-		break;
-	case 0x37804012:
-		flip_pairs(g_Rom);
-		break;
-	case 0x40123780:
-		flip_words(g_Rom);
-		break;
-	default:
-		fprintf(stderr, "Unable to detect ROM header. Is this an N64 ROM?\n");
-		exit(EXIT_FAILURE);
-		break;
-	}
-
-	g_RomVersion = identify_rom_version(g_Rom);
-
-	if (g_RomVersion == ROMVERSION_NTSCBETA || g_RomVersion == ROMVERSION_PALBETA) {
-		fprintf(stderr, "The beta versions are not supported.\n");
-		exit(EXIT_FAILURE);
-	}
-
-	if (g_RomVersion == ROMVERSION_UNKNOWN) {
-		fprintf(stderr, "Unable to determine your Perfect Dark version. Is this a Perfect Dark ROM?\n");
-		exit(EXIT_FAILURE);
-	}
-
-	extract_anims();
-	extract_audio();
-	extract_files();
-	extract_fonts();
-	extract_jpnfonts();
-	extract_textures();
-	extract_banners();
-
-	free(g_Rom);
-	g_Rom = NULL;
-}
-
-int main(int argc, char **argv)
-{
-	char *romfile = NULL;
-
-	if (argc < 2) {
-		fprintf(stderr, "Usage: pd-extract <ROMFILE> [PATH]\n");
-		fprintf(stderr, "Read ROMFILE, writing assets to PATH.\n");
-		fprintf(stderr, "With no PATH, assume \"extracted/\".\n");
-		return EXIT_FAILURE;
-	}
-
-	romfile = argv[1];
-
-	if (argc > 2) {
-		strcpy(g_OutPath, argv[2]);
-	} else {
-		strcpy(g_OutPath, "extracted");
-	}
-
-	if (g_OutPath[strlen(g_OutPath) - 1] == '/') {
-		g_OutPath[strlen(g_OutPath) - 1] = '\0';
-	}
-
-	extract_rom(romfile);
-
-	return EXIT_SUCCESS;
-}

port/pdextract/setups_decl.h

@@ -1,8 +1,6 @@
 #ifndef _IN_SETUP_DECL
 #define _IN_SETUP_DECL
 
-
-#include <PR/ultratypes.h>
 #include "common.h"
 
 

port/src/preprocess.c

@@ -36,8 +36,8 @@ static inline u32 swapUnk(u32 x) { assert(0 && "unknown type"); return x; }
 
 #define PD_SWAP_PTR(x) x = swapPtr((void *)(x))
 
-#define PD_PTR_BASE(x, b) (void *)((u8 *)b + (u32)x)
-#define PD_PTR_BASEOFS(x, b, d) (void *)((u8 *)b - d + (u32)x)
+#define PD_PTR_BASE(x, b) (void *)((u8 *)b + (uintptr_t)x)
+#define PD_PTR_BASEOFS(x, b, d) (void *)((u8 *)b - d + (uintptr_t)x)
 
 // HACK: to prevent double-swapping stuff, flag swapped offsets in a bitmap
 
@@ -1007,7 +1007,7 @@ static void preprocessIntroScript(s32 *cmd)
 	PD_SWAP_VAL(cmd[0]);
 }
 
-void preprocessAnimations(u8 *data, u32 size)
+u8* preprocessAnimations(u8* data, u32 size, u32* outSize)
 {
 	// set the anim table pointers as well
 	extern u8 *_animationsTableRomStart;
@@ -1032,9 +1032,10 @@ void preprocessAnimations(u8 *data, u32 size)
 			anim->data = 0xffffffff;
 		}
 	}
+	return 0;
 }
 
-void preprocessMpConfigs(u8 *data, u32 size)
+u8* preprocessMpConfigs(u8* data, u32 size, u32* outSize)
 {
 	const u32 count = size / sizeof(struct mpconfig);
 	struct mpconfig *cfg = (struct mpconfig *)data;
@@ -1064,10 +1065,12 @@ void preprocessMpConfigs(u8 *data, u32 size)
 #endif
 		}
 	}
+	return 0;
 }
 
-void preprocessFont(u8 *data, u32 size)
+u8* preprocessFont(u8* data, u32 size, u32* outSize)
 {
+	return;
 	struct font *fnt = (struct font *)data;
 
 	for (s32 i = 0; i < ARRAYCOUNT(fnt->kerning); ++i) {
@@ -1092,14 +1095,16 @@ void preprocessFont(u8 *data, u32 size)
 		PD_SWAP_VAL(fnt->chars[i].kerningindex);
 		PD_SWAP_PTR(fnt->chars[i].pixeldata);
 	}
+	return 0;
 }
 
-void preprocessJpnFont(u8 *data, u32 size)
+u8* preprocessJpnFont(u8* data, u32 size, u32* outSize)
 {
 	// ???
+	return 0;
 }
 
-void preprocessALBankFile(u8 *data, u32 size)
+u8* preprocessALBankFile(u8* data, u32 size, u32* outSize)
 {
 	memset(swapmap, 0, sizeof(swapmap));
 
@@ -1111,6 +1116,7 @@ void preprocessALBankFile(u8 *data, u32 size)
 		PD_SWAP_PTR(bankf->bankArray[i]);
 		preprocessALBank(PD_PTR_BASE(bankf->bankArray[i], data), data);
 	}
+	return 0;
 }
 
 void preprocessALCMidiHdr(u8 *data, u32 size)
@@ -1122,7 +1128,7 @@ void preprocessALCMidiHdr(u8 *data, u32 size)
 	}
 }
 
-void preprocessSequences(u8 *data, u32 size)
+u8* preprocessSequences(u8* data, u32 size, u32* outSize)
 {
 	struct seqtable *seq = (struct seqtable *)data;
 	PD_SWAP_VAL(seq->count);
@@ -1132,9 +1138,10 @@ void preprocessSequences(u8 *data, u32 size)
 		PD_SWAP_VAL(seq->entries[i].ziplen);
 		PD_SWAP_VAL(seq->entries[i].romaddr);
 	}
+	return 0;
 }
 
-void preprocessTexturesList(u8 *data, u32 size)
+u8* preprocessTexturesList(u8* data, u32 size, u32* outSize)
 {
 	struct texture *tex = (struct texture *)data;
 	const u32 count = size / sizeof(*tex);
@@ -1148,6 +1155,7 @@ void preprocessTexturesList(u8 *data, u32 size)
 		tex->soundsurfacetype = tex->surfacetype;
 		tex->surfacetype = tmp;
 	}
+	return 0;
 }
 
 void preprocessTextureRGBA32Embedded(u32* dest, u32 size_bytes)
@@ -1218,15 +1226,16 @@ void preprocessModel(u8 *base, u32 ofs)
 	}
 }
 
-void preprocessLangFile(u8 *data, u32 size)
+u8* preprocessLangFile(u8* data, u32 size, u32* outSize)
 {
 	// lang banks are just an offset table + text data right after
 	// offsets are from the beginning of the bank
 	// however we cannot byteswap them in advance because the length of the
 	// offset table is unknown, so that's done in lang.c on demand instead
+	return 0;
 }
 
-void preprocessPadsFile(u8 *data, u32 size)
+u8* preprocessPadsFile(u8* data, u32 size, u32* outSize)
 {
 	struct padsfileheader *hdr = (void *)data;
 
@@ -1297,9 +1306,10 @@ void preprocessPadsFile(u8 *data, u32 size)
 			++wg;
 		}
 	}
+	return 0;
 }
 
-void preprocessTilesFile(u8 *data, u32 size)
+u8* preprocessTilesFile(u8* data, u32 size, u32* outSize)
 {
 	u32 *roomTable = (u32 *)data;
 	PD_SWAP_VAL(roomTable[0]);
@@ -1368,9 +1378,10 @@ void preprocessTilesFile(u8 *data, u32 size)
 				break;
 		}
 	}
+	return 0;
 }
 
-void preprocessSetupFile(u8 *data, u32 size)
+u8* preprocessSetupFile(u8* data, u32 size, u32* outSize)
 {
 	struct stagesetup *set = (void *)data;
 
@@ -1422,16 +1433,19 @@ void preprocessSetupFile(u8 *data, u32 size)
 	}
 
 	// the rest of the pointers are set after loading
+	return 0;
 }
 
-void preprocessModelFile(u8 *data, u32 size)
+u8* preprocessModelFile(u8* data, u32 size, u32* outSize)
 {
 	preprocessModel(data, 0x5000000);
+	return 0;
 }
 
-void preprocessGunFile(u8 *data, u32 size)
+u8* preprocessGunFile(u8* data, u32 size, u32* outSize)
 {
 	preprocessModel(data, 0x5000000);
+	return 0;
 }
 
 void preprocessBgSection1Header(u8 *data, u32 size)
@@ -1442,6 +1456,7 @@ void preprocessBgSection1Header(u8 *data, u32 size)
 	PD_SWAP_VAL(header[2]); // primcompsize
 }
 
+#ifndef PLATFORM_64BIT
 void preprocessBgSection1(u8 *data, u32 ofs) {
 	struct {
 		u32 unk00;
@@ -1505,6 +1520,7 @@ void preprocessBgSection1(u8 *data, u32 ofs) {
 		// unused, so won't swap the contents
 	}
 }
+#endif
 
 void preprocessBgSection2Header(u8 *data, u32 size)
 {
@@ -1549,6 +1565,7 @@ void preprocessBgSection3(u8 *data, u32 size)
 	}
 }
 
+#ifndef PLATFORM_64BIT
 void preprocessBgRoom(u8 *data, u32 ofs) {
 	struct roomgfxdata *rgfx = (void *)data;
 
@@ -1607,6 +1624,7 @@ void preprocessBgRoom(u8 *data, u32 ofs) {
 		}
 	}
 }
+#endif
 
 void preprocessBgLights(u8 *data, u32 ofs)
 {

port/src/romdata.c

@@ -4,7 +4,6 @@
 #include <string.h>
 #include <ctype.h>
 #include <PR/ultratypes.h>
-#include <PR/ultratypes.h>
 #include "lib/rzip.h"
 #include "romdata.h"
 #include "fs.h"
@@ -99,6 +98,13 @@ static struct romfile fileSlots[ROMDATA_MAX_FILES] = {
 #define ROMSEG_START(n) _ ## n ## SegmentRomStart
 #define ROMSEG_END(n) _ ## n ## SegmentRomEnd
 
+// for x64 preprocess functions
+#ifdef PLATFORM_64BIT
+#define F(preprocessFunc) preprocessFunc##_x64
+#else
+#define F(preprocessFunc) preprocessFunc
+#endif
+
 /* segment table for ntsc-final                                                     */
 /* size will get calculated automatically if it is 0                                */
 /* if there are replacement files in the data dir, they will be loaded instead      */
@@ -116,15 +122,15 @@ static struct romfile fileSlots[ROMDATA_MAX_FILES] = {
 	ROMSEG_DECL_SEG(mpstringsS,         0x7e2f20,  0x7ce720,  0x7d25b0,  0x3700,   NULL                    ) \
 	ROMSEG_DECL_SEG(mpstringsI,         0x7e6620,  0x7d1e20,  0x7d5cb0,  0x3700,   NULL                    ) \
 	ROMSEG_DECL_SEG(firingrange,        0x7e9d20,  0x7d5520,  0x7d93b0,  0x1550,   NULL                    ) \
-	ROMSEG_DECL_SEG(fonttahoma,         0x7f7860,  0x7e3060,  0x7e6ef0,  0x0,      preprocessFont          ) \
-	ROMSEG_DECL_SEG(fontnumeric,        0x7f8b20,  0x7e4320,  0x7e81b0,  0x0,      preprocessFont          ) \
-	ROMSEG_DECL_SEG(fonthandelgothicsm, 0x7f9d30,  0x7e5530,  0x7e93c0,  0x0,      preprocessFont          ) \
-	ROMSEG_DECL_SEG(fonthandelgothicxs, 0x7fbfb0,  0x7e87b0,  0x7ec640,  0x0,      preprocessFont          ) \
-	ROMSEG_DECL_SEG(fonthandelgothicmd, 0x7fdd80,  0x7eae20,  0x7eecb0,  0x0,      preprocessFont          ) \
-	ROMSEG_DECL_SEG(fonthandelgothiclg, 0x8008e0,  0x7eee70,  0x7f2d00,  0x0,      preprocessFont          ) \
-	ROMSEG_DECL_SEG(sfxctl,             0x80a250,  0x7f87e0,  0x7fc670,  0x2fb80,  preprocessALBankFile    ) \
+	ROMSEG_DECL_SEG(fonttahoma,         0x7f7860,  0x7e3060,  0x7e6ef0,  0x0,      F(preprocessFont)       ) \
+	ROMSEG_DECL_SEG(fontnumeric,        0x7f8b20,  0x7e4320,  0x7e81b0,  0x0,      F(preprocessFont)       ) \
+	ROMSEG_DECL_SEG(fonthandelgothicsm, 0x7f9d30,  0x7e5530,  0x7e93c0,  0x0,      F(preprocessFont)       ) \
+	ROMSEG_DECL_SEG(fonthandelgothicxs, 0x7fbfb0,  0x7e87b0,  0x7ec640,  0x0,      F(preprocessFont)       ) \
+	ROMSEG_DECL_SEG(fonthandelgothicmd, 0x7fdd80,  0x7eae20,  0x7eecb0,  0x0,      F(preprocessFont)       ) \
+	ROMSEG_DECL_SEG(fonthandelgothiclg, 0x8008e0,  0x7eee70,  0x7f2d00,  0x0,      F(preprocessFont)       ) \
+	ROMSEG_DECL_SEG(sfxctl,             0x80a250,  0x7f87e0,  0x7fc670,  0x2fb80,  F(preprocessALBankFile) ) \
 	ROMSEG_DECL_SEG(sfxtbl,             0x839dd0,  0x828360,  0x82c1f0,  0x4c2160, NULL                    ) \
-	ROMSEG_DECL_SEG(seqctl,             0xcfbf30,  0xcea4c0,  0xcee350,  0xa060,   preprocessALBankFile    ) \
+	ROMSEG_DECL_SEG(seqctl,             0xcfbf30,  0xcea4c0,  0xcee350,  0xa060,   F(preprocessALBankFile) ) \
 	ROMSEG_DECL_SEG(seqtbl,             0xd05f90,  0xcf4520,  0xcf83b0,  0x17c070, NULL                    ) \
 	ROMSEG_DECL_SEG(sequences,          0xe82000,  0xe70590,  0xe74420,  0x563a0,  preprocessSequences     ) \
 	ROMSEG_DECL_SEG(texturesdata,       0x1d65f40, 0x1d5ca20, 0x1d61f90, 0x0,      NULL                    ) \
@@ -165,14 +171,16 @@ static struct romfile romSegs[] = {
 static preprocessfunc filePreprocFuncs[] = {
 	/* LOADTYPE_NONE  */ NULL,
 	/* LOADTYPE_BG    */ NULL, // loaded in parts
-	/* LOADTYPE_TILES */ preprocessTilesFile,
-	/* LOADTYPE_LANG  */ preprocessLangFile,
-	/* LOADTYPE_SETUP */ preprocessSetupFile,
-	/* LOADTYPE_PADS  */ preprocessPadsFile,
-	/* LOADTYPE_MODEL */ preprocessModelFile,
-	/* LOADTYPE_GUN   */ preprocessGunFile,
+	/* LOADTYPE_TILES */ F(preprocessTilesFile),
+	/* LOADTYPE_LANG  */ F(preprocessLangFile),
+	/* LOADTYPE_SETUP */ F(preprocessSetupFile),
+	/* LOADTYPE_PADS  */ F(preprocessPadsFile),
+	/* LOADTYPE_MODEL */ F(preprocessModelFile),
+	/* LOADTYPE_GUN   */ F(preprocessModelFile),
 };
 
+#undef F
+
 static inline void romdataWrongRomError(const char *fmt, ...)
 {
 	char reason[1024];
@@ -236,6 +244,17 @@ static inline void romdataLoadRom(void)
 	romDataSegSize = dataSegLen;
 }
 
+static inline void romdataUpdateSegStartEnd(struct romfile* seg)
+{
+	if (seg->segstart) {
+		*seg->segstart = seg->data;
+	}
+
+	if (seg->segend) {
+		*seg->segend = seg->data + seg->size;
+	}
+}
+
 static inline void romdataInitSegment(struct romfile *seg)
 {
 	if (!seg->data) {
@@ -269,7 +288,7 @@ static inline void romdataInitSegment(struct romfile *seg)
 		if (g_RomFile) {
 			newData = g_RomFile + (uintptr_t)seg->data;
 			seg->source = SRC_ROM;
-			sysLogPrintf(LOG_NOTE, "loading segment %s from ROM (offset %08x pointer %p)", seg->name, (u32)seg->data, newData);
+			sysLogPrintf(LOG_NOTE, "loading segment %s from ROM (offset %08x pointer %p)", seg->name, (uintptr_t)seg->data, newData);
 		} else {
 			sysFatalError("No ROM or external file for segment:\n%s", seg->name);
 		}
@@ -281,17 +300,19 @@ static inline void romdataInitSegment(struct romfile *seg)
 
 	seg->data = newData;
 
-	if (seg->segstart) {
-		*seg->segstart = seg->data;
-	}
-
-	if (seg->segend) {
-		*seg->segend = seg->data + seg->size;
-	}
+	romdataUpdateSegStartEnd(seg);
 
 	// call the post load function if any
 	if (seg->preprocess && !seg->preprocessed) {
-		seg->preprocess(seg->data, seg->size);
+		newData = seg->preprocess(seg->data, seg->size, &seg->size);
+
+		if (newData) {
+			if (seg->source == SRC_EXTERNAL)
+				sysMemFree(seg->data);
+			seg->data = newData;
+			romdataUpdateSegStartEnd(seg);
+		}
+		
 		seg->preprocessed = 1;
 	}
 }
@@ -447,7 +468,7 @@ u8 *romdataFileLoad(s32 fileNum, u32 *outSize)
 	return out;
 }
 
-void romdataFilePreprocess(s32 fileNum, s32 loadType, u8 *data, u32 size)
+void romdataFilePreprocess(s32 fileNum, s32 loadType, u8 *data, u32 size, u32 *outSize)
 {
 	if (fileNum < 1 || fileNum >= ROMDATA_MAX_FILES) {
 		sysLogPrintf(LOG_ERROR, "romdataFilePreprocess: invalid file num %d", fileNum);
@@ -465,7 +486,7 @@ void romdataFilePreprocess(s32 fileNum, s32 loadType, u8 *data, u32 size)
 				}
 			}
 			// then preprocess
-			filePreprocFuncs[loadType](data, size);
+			filePreprocFuncs[loadType](data, size, outSize);
 			// fileSlots[fileNum].preprocessed = 1;
 		}
 	}

src/game/file.c

@@ -4209,7 +4209,7 @@ void fileLoad(u8 *dst, u32 allocationlen, romptr_t *romaddrptr, struct fileinfo
 #ifndef PLATFORM_N64
 	// byteswap/preprocess file according to g_LoadType right after inflating it
 	const u32 dstsize = allocationlen ? info->loadedsize : romsize; 
-	romdataFilePreprocess(filenum, g_LoadType, dst, dstsize);
+	romdataFilePreprocess(filenum, g_LoadType, dst, dstsize, &info->loadedsize);
 	g_LoadType = LOADTYPE_NONE;
 #endif
 }

src/include/constants.h

@@ -48,7 +48,9 @@
 
 #define ABS(val)            ((val) > 0 ? (val) : -(val))
 #define ABSF(val)           ((val) > 0.0f ? (val) : -(val))
+#define ALIGN(val, size) ((val + (size - 1)) & ~(size - 1))
 #define ALIGN2(val)         (((val) | 1) ^ 0x1)
+#define ALIGN4(val)         (((val) | 3) ^ 0x3)
 #define ALIGN8(val)         ((((val) + 0x7) | 0x7) ^ 0x7)
 #define ALIGN16(val)        ((((val) + 0xf) | 0xf) ^ 0xf)
 #define ALIGN64(val)        (((((uintptr_t)(val)) + 0x3f) | 0x3f) ^ 0x3f)