Build ROM with asset files

tools/buildrom

@@ -19,7 +19,7 @@ def main():
     write_binary(fd, 0xcfbf30, get_seqctl())
     write_binary(fd, 0xd05f90, get_seqtbl())
     write_binary(fd, 0xe82000, get_midi())
-    write_binary(fd, 0xed83a0, get_files())
+    write_files(fd)
     write_binary(fd, 0x1d65f40, get_textures())
     fd.close()
 
@@ -83,8 +83,11 @@ def get_seqtbl():
 def get_midi():
     return getfilecontents('extracted/ntsc-final/audio/midi.bin')
 
-def get_files():
-    return getfrombaserom(0xed83a0, 0xe8d3a0)
+def write_files(fd):
+    start = getlinkervariable('_filesSegmentRomStart')
+    end = getlinkervariable('_filesSegmentRomEnd')
+
+    write_binary(fd, 0xed83a0, getfromstage1rom(start, end - start))
 
 def get_textures():
     return getfrombaserom(0x01d65f40, 0x29a0c0)
@@ -102,7 +105,20 @@ def getfrombaserom(offset, len):
     fd.close()
     return binary
 
+def getfromstage1rom(offset, len):
+    fd = open('build/ntsc-final/stage1.bin', 'rb')
+    fd.seek(offset)
+    binary = fd.read(len)
+    fd.close()
+    return binary
+
 def zip(filename):
     return subprocess.check_output(['tools/rarezip', filename])
 
+def getlinkervariable(varname):
+    objdump = subprocess.check_output(['mips64-elf-objdump', 'build/ntsc-final/stage1.elf', '-t']).decode('utf-8')
+
+    matches = re.findall(r'^([0-9a-f]+) .*? %s$' % varname, objdump, re.MULTILINE)
+    return int(matches[0], 16)
+
 main()

tools/extract

@@ -87,27 +87,36 @@ class Extractor:
                 return
             content = self.rom[offset:endoffset]
             name = names[index]
+
+            # If the content is zipped then the last byte might be padding. So
+            # unzip it to see.
+            unzipped = None
+            if content[0:2] == b'\x11\x73':
+                (unzipped, unused) = self.decompressandgetunused(content)
+                if len(unused):
+                    content = content[:-len(unused)]
+
             self.write('files/%s' % name, content)
 
             unzippedname = name[:-1] if name.endswith('Z') else name
 
             if name.endswith('tilesZ'):
-                self.write('files/bgdata/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/bgdata/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('C'):
-                self.write('files/chrs/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/chrs/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('G'):
-                self.write('files/guns/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/guns/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('L'):
-                self.write('files/lang/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/lang/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('P') and len(content):
-                self.write('files/props/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/props/%s.bin' % unzippedname, unzipped)
 
             if name.startswith('U'):
-                self.write('files/setup/%s.bin' % unzippedname, self.decompress(content))
+                self.write('files/setup/%s.bin' % unzippedname, unzipped)
 
     def get_file_offsets(self):
         i = self.val('files')
@@ -193,6 +202,13 @@ class Extractor:
         assert(header == 0x1173)
         return zlib.decompress(buffer[5:], wbits=-15)
 
+    def decompressandgetunused(self, buffer):
+        header = int.from_bytes(buffer[0:2], 'big')
+        assert(header == 0x1173)
+        obj = zlib.decompressobj(wbits=-15)
+        bin = obj.decompress(buffer[5:])
+        return (bin, obj.unused_data)
+
     def write(self, filename, data):
         filename = 'extracted/%s/%s' % (self.romid, filename)