Initial commit

tools/extract

@@ -0,0 +1,216 @@
+#!/usr/bin/python
+
+# Extract assets from a Perfect Dark ROM.
+#
+# Place ROM in the main project directory, named pd.<romid>.z64, then run this
+# with tools/extract <romid>. The directory extracted/<romid>/ will be created
+# and populated.
+#
+# Supported ROM IDs are:
+# ntsc-beta
+# ntsc-1.0
+# ntsc-final
+# pal-beta
+# pal-final
+# jap-final
+
+import os, sys, zlib
+
+class Extractor:
+    def extract(self, romid):
+        self.romid = romid
+
+        filename = 'pd.%s.z64' % romid
+
+        fd = open(filename, 'rb')
+        self.rom = fd.read()
+        fd.close()
+
+        self.globals = self.decompress(self.rom[self.val('globals'):])
+        self.extract_all()
+
+    def extract_all(self):
+        self.extract_audio()
+        self.extract_files()
+        self.extract_fonts()
+        self.extract_midis()
+        self.extract_globals()
+        self.extract_textures()
+        self.extract_ucodes()
+
+    #
+    # Audio
+    #
+
+    def extract_audio(self):
+        # Not implemented
+        pass
+
+    #
+    # Files
+    #
+
+    def extract_files(self):
+        offsets = self.get_file_offsets()
+        names = self.get_file_names(offsets[len(offsets) - 1])
+
+        for (index, offset) in enumerate(offsets):
+            if index == 0:
+                continue
+            try:
+                endoffset = offsets[index + 1]
+            except:
+                return
+            content = self.rom[offset:endoffset]
+            name = names[index]
+            self.write('files/%s' % name, content)
+
+            if name.endswith('Z') and len(content):
+                unzippedname = name[:-1]
+                try:
+                    self.write('files/%s' % unzippedname, self.decompress(content))
+                except:
+                    print(name)
+
+    def get_file_offsets(self):
+        i = self.val('files')
+        offsets = []
+        while True:
+            offset = int.from_bytes(self.globals[i:i+4], 'big')
+            if offset == 0 and len(offsets):
+                return offsets
+            offsets.append(offset)
+            i += 4
+
+    def get_file_names(self, tableaddr):
+        i = tableaddr
+        names = []
+        while True:
+            offset = int.from_bytes(self.rom[i:i+4], 'big')
+            if offset == 0 and len(names):
+                return names
+            names.append(self.read_name(tableaddr + offset))
+            i += 4
+
+    def read_name(self, address):
+        nullpos = self.rom[address:].index(0)
+        return str(self.rom[address:address + nullpos], 'utf-8')
+
+    #
+    # Fonts
+    #
+
+    def extract_fonts(self):
+        # Not implemented
+        pass
+
+    #
+    # Globals
+    #
+
+    def extract_globals(self):
+        self.write('Uglobals', self.globals)
+
+    #
+    # MIDIs
+    #
+
+    def extract_midis(self):
+        # Not implemented
+        pass
+
+    #
+    # Textures
+    #
+
+    def extract_textures(self):
+        # Not implemented
+        pass
+
+    #
+    # Ucodes
+    #
+
+    def extract_ucodes(self):
+        self.write('ucode/rspboot.bin', self.rom[0x40:0x1000])
+        self.write('ucode/boot.bin', self.rom[0x1000:0x3050])
+        self.write('ucode/library.bin', self.decompress(self.rom[0x3050:]))
+        self.write('ucode/decompressor.bin', self.rom[0x4e850:0x4fc30])
+        self.extract_ucode_game()
+
+    def extract_ucode_game(self):
+        binary = bytes()
+        start = i = self.val('game')
+
+        while True:
+            romoffset = start + int.from_bytes(self.rom[i:i+4], 'big') + 2
+            peek = int.from_bytes(self.rom[romoffset:romoffset+2], 'big')
+            if peek == 0:
+                break
+            part = self.decompress(self.rom[romoffset:romoffset+0x1000])
+            binary += part
+            if len(part) != 0x1000:
+                break
+            i += 4
+
+        self.write('ucode/game.bin', binary)
+
+    #
+    # Misc functions
+    #
+
+    def decompress(self, buffer):
+        header = int.from_bytes(buffer[0:2], 'big')
+        assert(header == 0x1173)
+        return zlib.decompress(buffer[5:], wbits=-15)
+
+    def write(self, filename, data):
+        filename = 'extracted/%s/%s' % (self.romid, filename)
+
+        dirname = os.path.dirname(filename)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+        fd = open(filename, 'wb')
+        fd.write(data)
+        fd.close()
+
+    def val(self, name):
+        return self.vals[self.romid][name]
+
+    vals = {
+        'ntsc-final': {
+            'game': 0x4fc40,
+            'files': 0x28080,
+            'globals': 0x39850,
+        },
+        'ntsc-1.0': {
+            'game': 0x4fc40,
+            'files': 0x28080,
+            'globals': 0x39850,
+        },
+        'ntsc-beta': {
+            'game': 0x43c40,
+            'files': 0x29160,
+            'globals': 0x30850,
+        },
+        'pal-final': {
+            'game': 0x4fc40,
+            'files': 0x28910,
+            'globals': 0x39850,
+        },
+        'pal-beta': {
+            'game': 0x4fc40,
+            'files': 0x29b90,
+            'globals': 0x39850,
+        },
+        'jap-final': {
+            'game': 0x4fc40,
+            'files': 0x28800,
+            'globals': 0x39850,
+        },
+    }
+
+extractor = Extractor()
+extractor.extract(sys.argv[1])
+

tools/inject

@@ -0,0 +1,237 @@
+#!/usr/bin/python
+
+import sys, zlib
+
+class Injector:
+
+    vacancies = [
+        (0x00ed83a0, 0x01d5ca00), # Normal files spot
+        (0x0002ea70, 0x00039850), # Unused space 1
+        (0x00157810, 0x001a15c0), # Unused space 2
+    ]
+
+    def rarezip(self, buffer):
+        length = len(buffer)
+
+        header = bytes([0x11, 0x73])
+        lendata = bytes([length >> 16])
+        lendata += bytes([(length >> 8) & 0xff])
+        lendata += bytes([length & 0xff])
+
+        obj = zlib.compressobj(level=9, wbits=-15)
+        return header + lendata + obj.compress(buffer) + obj.flush()
+
+    def rareunzip(self, compressed):
+        return zlib.decompress(compressed[5:], wbits=-15)
+
+    def load(self, romfile):
+        fp = open(romfile, 'rb')
+        rombuffer = fp.read()
+        fp.close()
+
+        self.rompart1 = rombuffer[0:0x2ea70]
+        self.rompart2 = rombuffer[0x0004e850:0x00157810]
+        self.rompart3 = rombuffer[0x001a15c0:0x00ed83a0]
+        self.rompart4 = rombuffer[0x01d5ca00:]
+
+        globals = self.rareunzip(rombuffer[0x39850:])
+
+        self.globaltop = globals[0:0x28080]
+        self.globalbot = globals[0x2a000:]
+
+        self.files = []
+        i = 0
+
+        while i <= 0x7dd:
+            romnameaddr = int.from_bytes(rombuffer[0x01d5ca00 + i * 4:0x01d5ca00 + i * 4 + 4], 'big') + 0x01d5ca00
+            romdataaddr = int.from_bytes(globals[0x28080 + i * 4:0x28080 + i * 4 + 4], 'big')
+
+            name = ''
+            while rombuffer[romnameaddr] != 0:
+                name = name + chr(rombuffer[romnameaddr])
+                romnameaddr += 1
+
+            self.files.append({
+                'name': name,
+                'romaddr': romdataaddr,
+                'data': None,
+            })
+            i += 1
+
+        self.files.sort(key=lambda file: file['romaddr'])
+
+        for index, file in enumerate(self.files):
+            start = file['romaddr']
+            end = self.files[index + 1]['romaddr'] if index < 0x7dd else 0x01d5ca00
+            file['data'] = rombuffer[start:end]
+
+    def transform(self):
+        # Replace file contents
+        for file in self.files:
+            if self.isFilePointless(file['name']):
+                file['data'] = None
+                continue
+
+            # Zipped file
+            try:
+                fp = open('build/ntsc-final/files/%s' % file['name'], 'rb')
+                contents = fp.read()
+                fp.close()
+            except:
+                continue
+
+            file['data'] = self.align(contents)
+
+        # Calculate new ROM addresses
+        vacancyid = 0
+        romaddr = self.vacancies[0][0]
+        vacend = self.vacancies[0][1]
+
+        for file in self.files:
+            if file['name'] == '':
+                continue
+
+            filelen = len(file['data'])
+            available = vacend - romaddr
+
+            if filelen > available:
+                vacancyid += 1
+                romaddr = self.vacancies[vacancyid][0]
+                vacend = self.vacancies[vacancyid][1]
+                available = vacend - romaddr
+
+            file['romaddr'] = romaddr
+            romaddr += filelen
+
+    def isFilePointless(self, name):
+        return False
+
+    def align(self, buffer):
+        length = len(buffer)
+
+        if length % 0x10 == 0:
+            return buffer
+
+        over = length % 0x10
+        pad = 0x10 - over
+
+        buffer += (0).to_bytes(1, 'big') * pad
+
+        return buffer
+
+    def compile(self):
+        buffer = self.rompart1
+        assert(len(buffer) == 0x2ea70)
+        buffer += self.compileVacancy(1)
+        assert(len(buffer) == 0x39850)
+        buffer += self.compileGlobals()
+        assert(len(buffer) == 0x4e850)
+        buffer += self.rompart2
+        assert(len(buffer) == 0x157810)
+        buffer += self.compileVacancy(2)
+        assert(len(buffer) == 0x1a15c0)
+        buffer += self.rompart3
+        assert(len(buffer) == 0xed83a0)
+        buffer += self.compileVacancy(0)
+        assert(len(buffer) == 0x01d5ca00)
+        buffer += self.rompart4
+        assert(len(buffer) == 0x2000000)
+        return buffer
+
+    def compileGlobals(self):
+        buffer = self.globaltop
+
+        for file in self.files:
+            buffer += file['romaddr'].to_bytes(4, 'big')
+
+        buffer += (0x01d5ca00).to_bytes(4, 'big')
+        buffer += (0).to_bytes(4, 'big')
+        buffer += self.globalbot
+
+        buffer = self.rarezip(buffer)
+
+        available = 0x15000 - len(buffer)
+        buffer += (0).to_bytes(1, 'big') * available
+        return buffer
+
+    def compileVacancy(self, vacid):
+        buffer = bytes()
+        vacstart = self.vacancies[vacid][0]
+        vacend = self.vacancies[vacid][1]
+
+        for file in self.files:
+            if file['romaddr'] >= vacstart and file['romaddr'] < vacend:
+                buffer += file['data']
+
+        available = vacend - vacstart - len(buffer)
+        buffer += (0).to_bytes(1, 'big') * available
+
+        return buffer
+
+    def ROL(self, i, b):
+        return ((i << b) | (i >> (32 - b))) & 0xffffffff
+
+    def R4(self, b):
+        return b[0]*0x1000000 + b[1]*0x10000 + b[2]*0x100 + b[3]
+
+    def crc(self, f):
+        seed = 0xdf26f436
+        t1 = t2 = t3 = t4 = t5 = t6 = seed
+
+        f.seek(0x0710 + 0x40)
+        lookup = f.read(0x100)
+
+        f.seek(0x1000)
+        for i in range(0x1000, 0x101000, 4):
+            d = self.R4(f.read(4))
+
+            if ((t6 + d) & 0xffffffff) < t6:
+                t4 += 1
+                t4 &= 0xffffffff
+
+            t6 += d
+            t6 &= 0xffffffff
+
+            t3 ^= d
+
+            r = self.ROL(d, d & 0x1F)
+
+            t5 += r
+            t5 &= 0xffffffff
+
+            if t2 > d:
+                t2 ^= r
+            else:
+                t2 ^= t6 ^ d
+
+            o = i & 0xFF
+            temp = self.R4(lookup[o:o + 4])
+            t1 += temp ^ d
+            t1 &= 0xffffffff
+
+        crc1 = t6 ^ t4 ^ t3
+        crc2 = t5 ^ t2 ^ t1
+
+        return crc1 & 0xffffffff, crc2 & 0xffffffff
+
+injector = Injector()
+injector.load(sys.argv[1])
+injector.transform()
+
+filename = 'build/ntsc-final/pd.z64'
+
+fp = open(filename, 'wb')
+fp.write(injector.compile())
+fp.close()
+
+fp = open(filename, 'rb')
+crcs = injector.crc(fp)
+fp.seek(0)
+buffer = fp.read()
+fp.close()
+
+fp = open(filename, 'r+b')
+fp.seek(0x10)
+fp.write(crcs[0].to_bytes(4, 'big'))
+fp.write(crcs[1].to_bytes(4, 'big'))
+fp.close()

tools/rareunzip

@@ -0,0 +1,22 @@
+#!/usr/bin/python
+
+import sys, zlib
+
+fp = open(sys.argv[1], 'rb')
+buffer = fp.read()
+fp.close()
+
+header = int.from_bytes(buffer[0:2], 'big')
+
+if header == 0x1172:
+    # GE
+    buffer = zlib.decompress(buffer[2:], wbits=-15)
+elif header == 0x1173:
+    # PD
+    buffer = zlib.decompress(buffer[5:], wbits=-15)
+else:
+    sys.stderr.write("Not an 0x1172 or 0x1173 compressed file\n");
+    sys.stderr.write("Got bytes 0x%04x\n" % header);
+    exit(1)
+
+sys.stdout.buffer.write(buffer)

tools/rarezip

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+size=$(stat --format="%s" $1)
+
+printf "0: 1173 %.6x" $size | xxd -r -g0 > $1.tmp
+
+cat $1 | tools/gzip --no-name --best | head --bytes=-8 | tail --bytes=+11 >> $1.tmp
+
+# Pad to 0x10 boundary
+compsize=$(stat --format="%s" $1.tmp)
+over=$((compsize % 0x10))
+
+if [ "$over" -gt 0 ]; then
+	while [ "$over" -lt 16 ]; do
+		echo -ne \\x00 >> $1.tmp
+		over=$((over + 1))
+	done
+fi
+
+cat $1.tmp
+
+rm -f $1.tmp
+