pal-final: Get ROM fully matching

tools/extract

@@ -461,7 +461,7 @@ class Extractor:
         'garbage1':      [0x0,       0x2ea22,   0x2ea6c,   0x0,       0x2eb21,   0x0,       ],
         'files':         [0x29160,   0x28080,   0x28080,   0x29b90,   0x28910,   0x28800,   ],
         'data':          [0x30850,   0x39850,   0x39850,   0x39850,   0x39850,   0x39850,   ],
-        'garbage2':      [0x0,       0x1574a0,  0x157800,  0x0,       0x157800,  0x0,       ],
+        'garbage2':      [0x0,       0x1574a0,  0x157800,  0x0,       0x158038,  0x0,       ],
         'animations':    [0x155dc0,  0x1a15c0,  0x1a15c0,  0x18cdc0,  0x18cdc0,  0x190c50,  ],
         'mpconfigs':     [0x785130,  0x7d0a40,  0x7d0a40,  0x7bc240,  0x7bc240,  0x7c00d0,  ],
         'firingrange':   [0x79e410,  0x7e9d20,  0x7e9d20,  0x7d5520,  0x7d5520,  0x7d93b0,  ],

tools/mkgamezips

@@ -75,7 +75,7 @@ def get_sums(bins):
     return [sum(part) for part in bins]
 
 def get_zips(bins):
-    return [zip(part) for part in bins]
+    return [zip(part, index) for part, index in enumerate(bins)]
 
 def sum(binary):
     sum = 0
@@ -83,9 +83,41 @@ def sum(binary):
         sum += int.from_bytes(binary[i:i+4], 'big')
     return sum & 0xffff
 
-def zip(binary):
+def zip(index, binary):
     obj = zlib.compressobj(level=9, wbits=-15)
-    return b'\x11\x73' + len(binary).to_bytes(3, 'big') + obj.compress(binary) + obj.flush()
+    stream = obj.compress(binary) + obj.flush()
+
+    for override in overrides[os.environ['ROMID']]:
+        if override[0] == index and zlib.crc32(stream) == override[1]:
+            stream = stream[0:override[2]]
+            stream += override[3]
+            break
+
+    return b'\x11\x73' + len(binary).to_bytes(3, 'big') + stream
+
+"""
+Rare's zip compression method is different to any known version of zlib.
+In a few cases (49 out of 2651) their algorithm produces differing files,
+where the final match directive chooses a seemingly random backtrack distance.
+To get matching zips, decomp patches the zips with this overrides list.
+Values are chunk index, crc32 checksum, write offset, write bytes
+"""
+overrides = {
+    'ntsc-beta': [],
+    'ntsc-1-0': [],
+    'ntsc-final': [],
+    'pal-final': [
+        (0x002e, 0x938a5928, 0x091a, b'\xc2\x9f\x7f\x03'),
+        (0x00e1, 0xae07cb57, 0x0b1b, b'\xe4\xd5\xff\x00'),
+        (0x00ea, 0xf80d6c37, 0x09ca, b'\xa9\xcb\xff\x02'),
+        (0x00f6, 0xdbd3f03a, 0x0002, b'\x6d\x6c\x1c\xd5\x15\x3d\x6f\x67\x6d\x6f\xec\xb5\x77\xe3\x6c\x94\x35\x8e\xda\x1d\x3c\x63\x2f\x09\x10\xdb\x81\x2a\x48\x9b\xb2\x80\xb1\x2a\xe2\x80\x0d\xa6\xb8\x52\x2b\x39\x35\x50\x23\x42\x30\xf0\xa3\x8e\x1a\x29\x43\x30\xb1\xd5\xec\x66\x36\x25\xb4\x06\x21\xb4\x24\x76\x92\xca\x6b\x8f\x93\x26\xad\x2d\x59\xaa\x91\x9d\x60\x68\x48\xd3\x12\x2a\x47\xca\x0f\x23\x8c\x40\xc2\x69\xa8\x14\xb5\xa9\x84\xb2\x3d\x77\x76\xed\xb8\x81\xf6\xc7\xe8\xcd\xbc\x77\xdf\xbd\xe7\xdd\x8f\x73\xdf\x38\x69\xac\x1d\xf1\x00\x88\x47\x4c\xcf\x64\xd4\xf4\xb4\xd7\x72\xdc\x64\x3a\x27\x11\x74\x4e\x20\xe4\x1c\x47\x38\x30\xd5\x3a\xe4\xdc\x2c\x67\x51\x2e\x9d\x97\x3b\x4e\xb9\x13\xdf\x22\x97\x5e\x26\xf7\xbb\x6f\xea\x33\xc2\x50\x22\xeb\x8c\x61\xed\x92\xfc\xa2\xfd\xbc\xbc\x2b\xcb\x3d\x86\x3f\x2f\x9b\xc9\xcb\xde\x8c\xf5\x78\x5e\x96\x38\x8c\x8a\xbc\xec\xf8\x32\xd9\xe5\x78\x97\xeb\x0d\xe4\x65\x9d\x65\x18\x96\x63\x5e\xa6\xd7\x1e\x51\x0b\x22\x9b\xfc\x90\xcf\x00\x82\xb5\xfb\x14\x54\x7b\xc4\x0c\x59\xf8\x34\xa6\x59\xc5\x89\x8e\xef\x3e\x69\x28\x68\xc1\x6d\xf0\x70\x2c\x09\x6d\x83\x3f\x16\xb6\x4a\xec\xf0\x5f\xee\xda\xe8\x87\xa7\xc5\xc2\x0a\xfb\xb0\x9a\xb3\x2f\x7a\x53\x5c\xf7\xd4\x0c\xa8\x89\x55\x3a\xb4\x9a\x41\x35\x16\x98\xfa\xf8\xaf\x35\x47\xd4\x49\x59\xef\x75\xd4\x07\xf6\x11\x75\xc9\x1e\x56\x13\xf6\x90\x1a\xa3\xdd\x93\xea\xd3\xa0\x6e\xb4\x50\x6f\x57\xf6\xba\xf1\x31\x4a\x9c\x53\x08\x7a\x40\xac\x11\xe2\x3c\x84\x3e\x7b\x40\x7d\xe1\x0c\xd3\xbf\x43\xe2\xdf\xbe\x7f\x38\x23\x58\x6b\x67\xb8\x9f\x7a\x8c\x52\xa8\xba\x37\x90\x45\x45\xf7\xfd\x46\x09\xa2\xc6\x2c\x14\x7d\x1e\xa4\xec\x5a\x9e\x3b\xc4\x7d\x61\xf7\xec\x79\x7d\x81\xa9\x44\x3b\xe3\x13\xa1\xdd\xdc\xfe\x0a\x44\xeb\x66\xb9\x3f\xdc\x7e\xbf\xf8\x8b\x7b\x22\x8c\x43\x68\xc9\xbf\x8b\x38\x88\x89\xfb\xc2\xb2\xdf\x38\x83\x62\xfb\x04\xcf\x3a\xaa\x2e\x89\x4c\xf5\xa3\xf0\x54\x07\xe1\x71\x46\xd5\xd9\xc0\xe9\xe6\x3f\x3b\x27\xd4\x79\x9e\xef\xaa\x73\x88\xb8\xd3\x6a\x74\x65\x1a\x5a\xac\xcc\x2a\x22\x3e\xe5\x0c\xa9\x51\xf3\xdc\x9a\x63\xc9\x77\x01\xfa\x63\xa1\xe6\xa4\x6a\x1d\x79\x99\x3e\xbf\x08\xef\xc8\x1b\xf0\x72\xde\xc7\x6f\x1f\xbf\xfd\xb1\xb2\xdd\x95\xe6\xb9\xfa\x34\xe7\xfd\xd3\x9d\x88\x37\x86\xe2\x31\x66\x1f\x1e\xf4\xa0\x24\xb5\x1d\x9b\x53\xcf\x23\x16\x2b\xd9\x5d\x89\xc0\x33\x54\xdc\x59\x85\xd2\x4e\x4b\x3d\xd7\xa6\x27\x9f\xad\xcf\x04\x81\x95\xb4\x35\xb1\x28\x97\x78\x1a\x0d\xf0\xd3\x28\x65\x35\xff\x64\xd5\xa2\x3c\xc2\x93\x96\x56\x36\xa3\xab\x11\xee\x7b\x0a\xfe\xc0\xe4\x2d\x31\xdb\x80\x9f\xd8\x27\x04\x5f\xea\xa7\xb4\xe1\xdd\x5d\x19\x2b\xa0\x1d\xcf\x36\xa8\x3d\xed\x55\xf0\x07\x2d\xec\x89\xe8\x38\x14\xd5\x13\x83\x77\x85\x02\xa7\x1f\xfd\x28\xd1\x73\xf7\x2e\x37\xfe\x83\xf4\x69\x07\x34\x63\x10\x9a\x33\xa8\xce\xfa\x71\xe7\x13\xce\x61\xfa\xc3\x51\x13\xb1\x95\x96\x2f\xb6\xca\xf2\x55\xbf\x84\xed\xd5\x75\xe8\xda\xbf\x09\xd8\xbf\x13\xe8\x1d\x51\x1f\xd4\x5d\x46\x56\x62\x25\xb1\x66\xce\x9c\x67\xbe\x9c\xf5\xcc\xb9\xf1\x8f\xd0\xef\x06\x9f\xa8\x1b\xd7\x71\xe6\xec\xe9\x1f\x37\x4b\x0e\xb8\xfe\xa5\xaf\x5d\x1b\xf4\xfd\xfe\xf5\xd4\xb7\x9d\x7e\x1d\x53\x13\xbd\xe3\xcc\xb3\xff\xa7\x6b\x88\xba\x24\xce\x63\x8c\xa9\xe8\x63\xbc\x97\xe7\xb9\xe8\x66\xae\x17\xf1\xdd\xcb\xb1\x80\x63\x01\x47\x2d\x9f\xfb\x95\x12\x07\xfb\x30\xfa\xc4\xae\xd1\x91\xfd\xd7\xe2\x39\x63\x01\xab\x28\x79\x65\x4d\x74\x11\x0b\x02\x9d\x96\x27\xd9\xa5\x27\xdf\xbb\x81\xcb\xb5\x2f\x38\x88\x67\x39\xc6\x6f\xc3\x24\x38\x02\xa7\x5b\x4c\xb7\x06\xf8\x6e\x1f\x52\x0b\xf6\x29\x75\x89\x76\x8a\xa7\xe3\xf8\x41\xf5\x29\x94\x34\x86\x90\xcb\x0b\x0d\x95\xa9\x2d\x68\x4f\x5e\x71\xcf\xe0\x6d\xf9\x04\xdf\xb1\x33\xe8\x49\x6d\x45\x3b\xb0\xc9\xbc\xed\x1d\x68\xad\x11\xac\x68\x28\x04\xe4\x49\x34\x20\x94\x78\x50\x6c\x3d\xba\xd5\x28\x82\x62\xce\x14\x1a\x65\x50\x8b\x6b\x32\x2f\xeb\x32\x17\x53\x96\x2f\x73\x27\xbe\xa6\x8c\x4f\x30\x88\x1d\xf1\xd3\xe3\xf4\x8d\xd8\xa0\xec\xd1\xe9\x26\x34\xbb\xf8\xf8\x2d\x79\x69\x3e\x5b\x9f\x56\xaf\xfb\xd6\x35\x47\xa0\x4d\x3f\x84\xb8\x73\x4a\x5d\x90\xb1\x31\xd4\x7c\x23\x8f\xb7\x32\x3f\xdb\x98\x63\x01\xe6\x57\xf8\x7d\x68\x9b\x66\xaa\x50\x31\x63\xa9\x7f\x77\xeb\xc9\x2b\xf5\xa3\x6e\x1e\x33\x77\x16\xe5\x12\x8d\xcc\xe3\xd2\x17\x20\xb2\xea\xc3\xae\xaa\x45\x79\x94\x75\x59\xea\x4f\xed\xba\xb6\x9e\xfb\x3e\x43\x50\xf2\x38\xd9\x43\x7f\x4a\x1e\x13\x53\x62\x2b\xfc\xf4\xe1\x04\x58\xcf\x2d\x69\x94\x4c\x37\x23\x2e\x32\x46\x21\x6e\x17\x2e\x31\x54\x16\xb8\xaf\xcb\xa8\xeb\x47\x56\xcd\xce\x98\xce\x78\x6e\x9f\xc8\xc5\xc2\xbb\x2b\x6b\xc2\xf5\xe9\xc6\x90\x75\x03\x77\x13\xf1\x6c\x21\xee\x52\xe2\x2e\xd9\x09\xf5\x65\x77\x15\x02\xdd\x16\xeb\x4a\x4f\x0e\xd7\x9f\x74\x71\x0f\xab\xb1\x45\x39\x17\xb7\x9f\xb8\x45\xf6\x6c\x57\xd5\xa2\xbc\x8b\x7b\x96\xf5\x77\x8d\xfb\x06\xe9\xf3\x5c\xfd\x85\x88\x7b\x2c\xe7\x4b\x75\xd5\xfe\xad\x9a\x7b\xdc\xc2\x2f\x25\x8e\xa9\x07\xd0\x2e\xf5\x42\xdf\x2b\xe6\x61\x48\xbd\x1f\xd7\x63\xea\x5e\xab\xe1\xe5\x14\xec\x71\x75\x81\x9c\x3a\xc1\x5a\x1b\x33\x6b\xa1\xcc\x95\xd9\xac\x70\x97\x70\x98\xe4\x8f\x27\x5d\x2b\x35\x20\xdc\x16\xfc\xfc\x04\x6b\xc1\xe5\xd3\x57\x6f\x95\xfe\x40\xbc\x35\x92\x5f\xc2\xdd\xd4\xbb\x4a\xe4\x99\xaf\x17\x5c\x9e\xce\xa8\x31\x77\x9f\xd4\x1f\x6b\x54\xfa\xd4Line truncated
+        (0x00f8, 0xb681689e, 0x0994, b'\xf8\xf7\xbf'),
+        (0x0117, 0x230a2b89, 0x068f, b'\x77\xc6\xfe\x0f'),
+        (0x0144, 0x7965231d, 0x09b8, b'\xce\xeb\xff\x02'),
+        (0x017c, 0x328ab6aa, 0x0a99, b'\x6a\xd4\x7f\x01'),
+        (0x0199, 0x98c16cb9, 0x0a57, b'\xd1\xff\x00'),
+    ],
+}
 
 padding = {
     'ntsc-beta': [],
@@ -147,7 +179,37 @@ padding = {
         0x96, 0xfa, 0xbe, 0x79, 0xe3, 0x1e, 0x7f, 0xff,
         0x67, 0x86, 0x86, 0x86, 0x15, 0x6e,
     ],
-    'pal-final': [],
+    'pal-final': [
+        0x00, 0x00, 0x00, 0x73, 0xe6, 0xc7, 0xd1, 0xe6,
+        0x7c, 0x76, 0xec, 0xdc, 0x9a, 0xf1, 0x6e, 0x54,
+        0x05, 0xd9, 0x72, 0x53, 0x7b, 0xb4, 0xbe, 0x5d,
+        0xaf, 0xf8, 0x3e, 0xcc, 0x53, 0x8a, 0x75, 0x17,
+        0x26, 0x40, 0x29, 0x29, 0x5f, 0xf3, 0x44, 0xc6,
+        0xf9, 0xfe, 0x57, 0x1b, 0xf6, 0x7e, 0x45, 0xfa,
+        0x9e, 0x75, 0x67, 0xb7, 0x71, 0xb2, 0x9f, 0x19,
+        0x02, 0xb2, 0xc5, 0x7c, 0xe2, 0x5b, 0xbb, 0xef,
+        0xb2, 0x0f, 0xbe, 0x9a, 0x9c, 0x6b, 0x0d, 0x9a,
+        0xfd, 0xed, 0x6d, 0xee, 0xfb, 0x7c, 0x1b, 0xf7,
+        0xcf, 0x53, 0xac, 0xf3, 0xeb, 0xcd, 0x8a, 0x4f,
+        0x18, 0x4c, 0x5d, 0xeb, 0x52, 0x28, 0x9b, 0xf4,
+        0x7d, 0xbf, 0x3f, 0xe9, 0x8f, 0x12, 0x6c, 0x67,
+        0xd7, 0x82, 0x6a, 0xdd, 0x4a, 0x9e, 0x5c, 0x23,
+        0x7c, 0x18, 0x8a, 0x7d, 0x87, 0x6e, 0x7b, 0x5f,
+        0xd5, 0x9d, 0xd4, 0x3f, 0x9f, 0xf9, 0xb6, 0xf6,
+        0xd4, 0x9f, 0x71, 0x21, 0x9d, 0xd1, 0xc0, 0x86,
+        0x63, 0x6b, 0xd8, 0x87, 0xac, 0x8c, 0x53, 0x6d,
+        0xac, 0xc9, 0xbc, 0x8e, 0xdf, 0x13, 0x73, 0x45,
+        0x5a, 0xf7, 0x3b, 0xbb, 0x72, 0x66, 0xa5, 0xc0,
+        0xb6, 0x9b, 0xe5, 0x98, 0x0b, 0x27, 0xed, 0xc2,
+        0x7f, 0x78, 0xb6, 0x39, 0x26, 0xb6, 0x19, 0x7a,
+        0x77, 0xea, 0x70, 0x79, 0xff, 0xd7, 0x0f, 0x5f,
+        0x53, 0xea, 0x78, 0x30, 0x9b, 0xd2, 0x51, 0x62,
+        0xd7, 0xb9, 0x7a, 0xfd, 0x9d, 0x10, 0x64, 0x72,
+        0x35, 0x3b, 0xa9, 0xeb, 0x65, 0x8e, 0x25, 0xe8,
+        0xe7, 0x51, 0xfa, 0x1f, 0xd1, 0xf7, 0x35, 0x82,
+        0x4d, 0xf5, 0xdd, 0x55, 0x23, 0x9b, 0x15, 0xc8,
+        0x5a, 0xa2, 0x63, 0x63, 0x63, 0x6f,
+    ],
 }
 
 main()

tools/packrom

@@ -109,6 +109,7 @@ def write_garbage_part(fd, addr, filename):
 def write_garbage(fd):
     if os.environ['ROMID'] == 'pal-final':
         write_garbage_part(fd, 0x2eb21, 'garbage1.bin')
+        write_garbage_part(fd, 0x158038, 'garbage2.bin')
     elif os.environ['ROMID'] == 'ntsc-final':
         write_garbage_part(fd, 0x2ea6c, 'garbage1.bin')
         write_garbage_part(fd, 0x157800, 'garbage2.bin')