Introduce JSON assets

This commit is contained in:
Ryan Dwyer committed 2021-06-11 14:30:20 +10:00
1 parent 6aec86ff83
commit c9b36ac9b6
5660 files changed
+5648483 -192061

No files matched your search

+131
View File
@@ -0,0 +1,131 @@
import os
import subprocess
def align4(value):
return (value + 3 | 3) ^ 3
def align16(value):
return (value + 15 | 15) ^ 15
def pad4(binary):
if len(binary) % 4:
amount = 4 - (len(binary) % 4)
binary += (0).to_bytes(amount, 'big')
return binary
def pad16(binary):
if len(binary) % 16:
amount = 16 - (len(binary) % 16)
binary += (0).to_bytes(amount, 'big')
return binary
def write_enums(typename, names, filename, terminator, start=0):
filename = 'src/generated/%s/%s' % (os.environ['ROMID'], filename)
mkpath(filename)
fd = open(filename, 'w')
fd.write('/**\n')
fd.write(' * This file was generated automatically. Changes may be overwritten.\n')
fd.write(' */\n')
fd.write('\n')
fd.write('enum %s {\n' % typename)
for index, name in enumerate(names):
if index == 0 and start != 0:
fd.write('\t%s = 0x%x,\n' % (name, start))
else:
fd.write('\t%s,\n' % name)
fd.write('\t%s\n' % terminator)
fd.write('};\n')
fd.close()
def write_object(data, filename):
filename = 'build/%s/assets/%s' % (os.environ['ROMID'], filename)
mkpath(filename)
fd = open(filename, 'wb')
# 0x00 - ELF header
fd.write(b'\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00')
fd.write(b'\x00\x01\x00\x08\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00')
fd.write(b'\x00\x00\x00\x40\x10\x00\x10\x00\x00\x34\x00\x00\x00\x00\x00\x28')
fd.write(b'\x00\x03\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
# 0x40 - Section table index 0 (blank)
fd.write(b'\x00\x00\x00\x00') # offset to section name
fd.write(b'\x00\x00\x00\x00') # type
fd.write(b'\x00\x00\x00\x00') # attributes (writeable and occupies memory during execution)
fd.write(b'\x00\x00\x00\x00') # virtual address
fd.write(b'\x00\x00\x00\x00') # offset to data
fd.write(b'\x00\x00\x00\x00') # length
fd.write(b'\x00\x00\x00\x00') # section index
fd.write(b'\x00\x00\x00\x00') # extra information
fd.write(b'\x00\x00\x00\x00') # alignment
fd.write(b'\x00\x00\x00\x00') # entry size
# 0x68 - Section table index 1 (.data)
fd.write(b'\x00\x00\x00\x01') # offset to section name
fd.write(b'\x00\x00\x00\x01') # type
fd.write(b'\x00\x00\x00\x03') # attributes (writeable and occupies memory during execution)
fd.write(b'\x00\x00\x00\x00') # virtual address
fd.write(b'\x00\x00\x00\xd0') # offset to data
fd.write(align16(len(data)).to_bytes(4, 'big'))
fd.write(b'\x00\x00\x00\x00') # section index
fd.write(b'\x00\x00\x00\x00') # extra information
fd.write(b'\x00\x00\x00\x10') # alignment
fd.write(b'\x00\x00\x00\x00') # entry size
# 0x90 - Section table index 2 (.shstrtab)
fd.write(b'\x00\x00\x00\x07') # offset to section name
fd.write(b'\x00\x00\x00\x03') # type
fd.write(b'\x00\x00\x00\x00') # attributes (writeable and occupies memory during execution)
fd.write(b'\x00\x00\x00\x00') # virtual address
fd.write(b'\x00\x00\x00\xb8') # offset to data
fd.write(b'\x00\x00\x00\x18') # length
fd.write(b'\x00\x00\x00\x00') # section index
fd.write(b'\x00\x00\x00\x00') # extra information
fd.write(b'\x00\x00\x00\x01') # alignment
fd.write(b'\x00\x00\x00\x00') # entry size
# 0xb8 = .shstrtab
fd.write(b'\x00')
fd.write('.data'.encode('utf8'))
fd.write(b'\x00')
fd.write('.shstrtab'.encode('utf8'))
fd.write((0).to_bytes(8, 'big'))
# 0xd0 = .data
fd.write(data)
# Pad to next 16-byte boundary
amount = 16 - (len(data) % 16)
fd.write((0).to_bytes(amount, 'big'))
fd.close()
def zip(data):
filename = '/tmp/pd-assetmgr-%d' % os.getpid()
fd = open(filename, 'wb')
fd.write(data)
fd.close()
stream = subprocess.check_output(['tools/gzip', '-c', '--no-name', '--best', filename])[10:-8]
os.remove(filename)
return b'\x11\x73' + len(data).to_bytes(3, 'big') + stream
def mkpath(filename):
dirname = os.path.dirname(filename)
if not os.path.isdir(dirname):
os.makedirs(dirname, exist_ok=True)
def writefile(filename, contents):
mkpath(filename)
fd = open(filename, 'wb')
fd.write(contents)
fd.close()
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
import assetmgr
import json
import os
def main():
fd = open('src/assets/%s/animations.json' % os.environ['ROMID'], 'r')
data = fd.read()
fd.close()
rows = json.loads(data)
make_object(rows)
make_header(rows)
def make_header(rows):
typename = 'animnum'
enums = [row['id'] for row in rows]
filename = 'animations.h'
terminator = 'ANIM_END'
assetmgr.write_enums(typename, enums, filename, terminator)
def make_object(rows):
binary = bytes()
table = len(rows).to_bytes(4, 'big')
pos = 0
for row in rows:
# Append data to the binary
data = getcontents('src/assets/%s/animations/%s' % (os.environ['ROMID'], row['file']))
binary += data
# Append the table to a different variable,
# which we'll append to binary at the end
flags = 0
if row['flag01']: flags |= 0x01
if row['flag02']: flags |= 0x02
if row['flag04']: flags |= 0x04
if row['flag08']: flags |= 0x08
table += row['numframes'].to_bytes(2, 'big')
table += row['bytesperframe'].to_bytes(2, 'big')
table += pos.to_bytes(4, 'big')
table += row['unk08'].to_bytes(2, 'big')
table += row['unk0a'].to_bytes(1, 'big')
table += flags.to_bytes(1, 'big')
pos += len(data)
binary = assetmgr.pad16(binary)
binary += table
binary = assetmgr.pad16(binary)
assetmgr.write_object(binary, 'animations.o')
def getcontents(filename):
with open(filename, 'rb') as fd:
return fd.read()
main()
+152
View File
@@ -0,0 +1,152 @@
#!/usr/bin/env python3
import assetmgr
import json
import os
import re
import sys
def main():
fd = open(sys.argv[1], 'r')
data = fd.read()
fd.close()
basename = os.path.basename(sys.argv[1]).replace('.json', '')
shortname = get_shortname(basename)
rows = json.loads(data)
if os.environ['ROMID'].startswith('ntsc') and basename.endswith('E'):
make_header(rows, shortname)
elif os.environ['ROMID'].startswith('pal') and basename.endswith('P'):
make_header(rows, shortname)
elif os.environ['ROMID'].startswith('jpn') and basename.endswith('J'):
make_header(rows, shortname)
make_object(rows, basename)
def get_shortname(basename):
return re.match(r'^[a-z0-9]+', basename)[0]
def make_header(rows, shortname):
typename = 'l_%s' % shortname
enums = [row['id'] for row in rows]
filename = 'lang/%s.h' % shortname
terminator = 'L_%s_END' % shortname.upper()
start = banks.index(shortname) * 512
assetmgr.write_enums(typename, enums, filename, terminator, start=start)
def make_object(rows, basename):
binary = make_binary(rows)
zipped = assetmgr.zip(binary)
# Write the zipped file, purely so `make test` can verify it
if '_str_' in basename:
tmpfilename = basename + 'Z'
else:
tmpfilename = basename
assetmgr.writefile('build/%s/assets/files/L%s' % (os.environ['ROMID'], tmpfilename), zipped)
filename = 'files/L%s.o' % basename
assetmgr.write_object(zipped, filename)
def make_binary(rows):
if len(rows) == 0:
return (0).to_bytes(16, 'big')
output = bytes()
pos = len(rows) * 4
for index, row in enumerate(rows):
if row['text'] is None:
output += (0).to_bytes(4, 'big')
else:
output += pos.to_bytes(4, 'big')
pos += assetmgr.align4(len(row['text']) + 1)
for index, row in enumerate(rows):
if row['text'] is not None:
output += row['text'].encode('latin_1')
amount = 4 - (len(row['text']) % 4)
output += (0).to_bytes(amount, 'big')
if len(output) % 16:
amount = 16 - (len(output) % 16)
output += (0).to_bytes(amount, 'big')
return output
banks = [
'',
'ame',
'arch',
'ark',
'ash',
'azt',
'cat',
'cave',
'arec',
'crad',
'cryp',
'dam',
'depo',
'dest',
'dish',
'ear',
'eld',
'imp',
'jun',
'lee',
'len',
'lip',
'lue',
'oat',
'pam',
'pete',
'ref',
'rit',
'run',
'sevb',
'sev',
'sevx',
'sevxb',
'sho',
'silo',
'stat',
'tra',
'wax',
'gun',
'title',
'mpmenu',
'propobj',
'mpweapons',
'options',
'misc',
'uff',
'old',
'ate',
'lam',
'mp1',
'mp2',
'mp3',
'mp4',
'mp5',
'mp6',
'mp7',
'mp8',
'mp9',
'mp10',
'mp11',
'mp12',
'mp13',
'mp14',
'mp15',
'mp16',
'mp17',
'mp18',
'mp19',
'mp20',
]
main()
+331
View File
@@ -0,0 +1,331 @@
#!/usr/bin/env python3
import assetmgr
import json
import os
import re
import struct
import sys
class App():
def run(self):
self.load_data()
self.pad_names = self.make_names(self.json['pads'])
self.waypoint_names = self.make_names(self.json['waypoints'])
self.waygroup_names = self.make_names(self.json['waygroups'])
self.populate_waygroup_waypoints()
self.make_header()
self.make_object()
def load_data(self):
fd = open(sys.argv[1], 'r')
data = fd.read()
fd.close()
self.json = json.loads(data)
self.shortname = os.path.basename(sys.argv[1]).replace('.json', '')
def make_names(self, items):
names = {}
for index, item in enumerate(items):
names[item['id']] = index
return names
def populate_waygroup_waypoints(self):
for index, waygroup in enumerate(self.json['waygroups']):
self.json['waygroups'][index]['waypoints'] = []
for waypoint in self.json['waypoints']:
index = self.waygroup_names[waypoint['waygroup']]
self.json['waygroups'][index]['waypoints'].append(waypoint['id'])
def make_header(self):
typename = 'pad_%s' % self.shortname
enums = [row['id'] for row in self.json['pads']]
filename = 'pads/%s.h' % self.shortname
terminator = 'PAD_%s_END' % self.shortname.upper()
assetmgr.write_enums(typename, enums, filename, terminator)
def make_object(self):
binary = self.make_binary()
zipped = assetmgr.zip(binary)
# Write the zipped file, purely so `make test` can verify it
assetmgr.writefile('build/%s/assets/files/bgdata/bg_%s_padsZ' % (os.environ['ROMID'], self.shortname), zipped)
filename = 'files/bgdata/bg_%s_padsZ.o' % self.shortname
assetmgr.write_object(zipped, filename)
# Order of stuff:
# - 0x00: header
# - 0x14: array of pad offsets (variable length)
# - Pads
# - Waypoint list
# - Waypoint neighbours
# - Waygroup list
# - Waygroup neighbours
# - Waygroup waypoints
# - Cover
def make_binary(self):
pads = self.make_pads()
waypoints_start = 0x14 + len(pads)
waypoints = self.make_waypoints(waypoints_start)
waygroups_start = waypoints_start + len(waypoints)
waygroups = self.make_waygroups(waygroups_start)
cover_start = waygroups_start + len(waygroups)
cover = self.make_cover()
output = bytes()
output += len(self.json['pads']).to_bytes(4, 'big')
output += len(self.json['cover']).to_bytes(4, 'big')
output += waypoints_start.to_bytes(4, 'big')
output += waygroups_start.to_bytes(4, 'big')
output += cover_start.to_bytes(4, 'big')
output += pads
output += waypoints
output += waygroups
output += cover
output = assetmgr.pad16(output)
return output
def make_pads(self):
output = bytes()
offsets = bytes()
start = pos = assetmgr.align4(0x14 + len(self.json['pads']) * 2)
for pad in self.json['pads']:
# flags, room and liftnum
# ffffffff ffffffff ffrrrrrr rrrrllll
flags = self.make_pad_flags(pad)
word = (flags << 14) | 0x3ff0 | (pad['liftnum'] & 0x0f)
output += word.to_bytes(4, 'big')
if flags & PADFLAG_INTPOS:
output += self.make_unsigned(pad['pos'][0]).to_bytes(2, 'big')
output += self.make_unsigned(pad['pos'][1]).to_bytes(2, 'big')
output += self.make_unsigned(pad['pos'][2]).to_bytes(2, 'big')
output += b'\x00\x00'
else:
output += self.make_float(pad['pos'][0])
output += self.make_float(pad['pos'][1])
output += self.make_float(pad['pos'][2])
if (flags & 0x0e) == 0:
output += self.make_float(pad['up'][0])
output += self.make_float(pad['up'][1])
output += self.make_float(pad['up'][2])
if (flags & 0xe0) == 0:
output += self.make_float(pad['dir'][0])
output += self.make_float(pad['dir'][1])
output += self.make_float(pad['dir'][2])
if flags & PADFLAG_HASBBOXDATA:
output += self.make_float(pad['xmin'])
output += self.make_float(pad['xmax'])
output += self.make_float(pad['ymin'])
output += self.make_float(pad['ymax'])
output += self.make_float(pad['zmin'])
output += self.make_float(pad['zmax'])
output = assetmgr.pad4(output)
offsets += pos.to_bytes(2, 'big')
pos = start + len(output)
offsets = assetmgr.pad4(offsets)
return offsets + output
# Some pads need to store 0x80000000 (negative 0), but this isn't supported
# in JSON, nor does it appear to be a thing in Python, hence this hack.
# For -0 values, the JSON uses a string which we check for here.
def make_float(self, value):
if value == '-0':
return b'\x80\x00\x00\x00'
return struct.pack('>f', value)
def make_pad_flags(self, pad):
flags = 0
if isinstance(pad['pos'][0], int) and isinstance(pad['pos'][1], int) and isinstance(pad['pos'][2], int):
flags |= PADFLAG_INTPOS
if abs(pad['up'][0]) == 1 and pad['up'][1] == 0 and pad['up'][2] == 0:
flags |= PADFLAG_UPALIGNTOX
if pad['up'][0] == 0 and abs(pad['up'][1]) == 1 and pad['up'][2] == 0:
flags |= PADFLAG_UPALIGNTOY
if pad['up'][0] == 0 and pad['up'][1] == 0 and abs(pad['up'][2]) == 1:
flags |= PADFLAG_UPALIGNTOZ
if (flags & 0x0e) and sum(pad['up']) == -1:
flags |= PADFLAG_UPALIGNINVERT
if abs(pad['dir'][0]) == 1 and pad['dir'][1] == 0 and pad['dir'][2] == 0:
flags |= PADFLAG_LOOKALIGNTOX
if pad['dir'][0] == 0 and abs(pad['dir'][1]) == 1 and pad['dir'][2] == 0:
flags |= PADFLAG_LOOKALIGNTOY
if pad['dir'][0] == 0 and pad['dir'][1] == 0 and abs(pad['dir'][2]) == 1:
flags |= PADFLAG_LOOKALIGNTOZ
if (flags & 0xe0) and sum(pad['dir']) == -1:
flags |= PADFLAG_LOOKALIGNINVERT
if pad['xmin'] != -100 or pad['xmax'] != 100 or pad['ymin'] != -100 or pad['ymax'] != 100 or pad['zmin'] != -100 or pad['zmax'] != 100:
flags |= PADFLAG_HASBBOXDATA
if pad['aiwaitlift']:
flags |= PADFLAG_AIWAITLIFT
if pad['aionlift']:
flags |= PADFLAG_AIONLIFT
if pad['aiwalkdirect']:
flags |= PADFLAG_AIWALKDIRECT
if pad['aidrop']:
flags |= PADFLAG_AIDROP
if pad['aiduck']:
flags |= PADFLAG_AIDUCK
if pad['flag00008000']:
flags |= PADFLAG_8000
if pad['flag00010000']:
flags |= PADFLAG_10000
return flags
def make_unsigned(self, value):
if value < 0:
value = 0x10000 - abs(value)
return value
def make_waypoints(self, start):
output = bytes()
neighbours = bytes()
pos = start + len(self.json['waypoints']) * 0x10 + 0x10
for row in self.json['waypoints']:
output += self.pad_names[row['pad']].to_bytes(4, 'big')
output += pos.to_bytes(4, 'big')
output += self.waygroup_names[row['waygroup']].to_bytes(4, 'big')
output += b'\x00\x00\x00\x00'
neighbours += self.make_waypoint_neighbours_list(row['neighbours'])
pos += len(row['neighbours']) * 4 + 4
# Terminator record
output += b'\xff\xff\xff\xff'
output += b'\x00\x00\x00\x00'
output += b'\x00\x00\x00\x00'
output += b'\x00\x00\x00\x00'
return output + neighbours
def make_waygroups(self, start):
neighbours = bytes()
waypoints = bytes()
for row in self.json['waygroups']:
neighbours += self.make_waygroup_neighbours_list(row['neighbours'])
waypoints += self.make_list(row['waypoints'])
output = bytes()
wpos = start + len(self.json['waygroups']) * 0x0c + 0x0c
npos = wpos + len(waypoints)
for row in self.json['waygroups']:
output += npos.to_bytes(4, 'big')
npos += len(row['neighbours']) * 4 + 4
output += wpos.to_bytes(4, 'big')
wpos += len(row['waypoints']) * 4 + 4
output += b'\x00\x00\x00\x00'
# Terminator record
output += b'\x00\x00\x00\x00'
output += b'\x00\x00\x00\x00'
output += b'\x00\x00\x00\x00'
return output + waypoints + neighbours
def make_cover(self):
output = bytes()
for row in self.json['cover']:
output += self.make_float(row['pos'][0])
output += self.make_float(row['pos'][1])
output += self.make_float(row['pos'][2])
output += self.make_float(row['dir'][0])
output += self.make_float(row['dir'][1])
output += self.make_float(row['dir'][2])
output += row['unk18'].to_bytes(4, 'big')
return output
def make_list(self, items):
output = bytes()
for value in items:
if 'WAYPOINT_' in value:
output += self.waypoint_names[value].to_bytes(4, 'big')
elif 'WAYGROUP_' in value:
output += self.waygroup_names[value].to_bytes(4, 'big')
output += b'\xff\xff\xff\xff'
return output
def make_waypoint_neighbours_list(self, items):
output = bytes()
for item in items:
value = self.waypoint_names[item['waypoint']]
if item['flag8000']: value |= 0x8000
if item['flag4000']: value |= 0x4000
output += value.to_bytes(4, 'big')
output += b'\xff\xff\xff\xff'
return output
def make_waygroup_neighbours_list(self, items):
output = bytes()
for item in items:
value = self.waygroup_names[item['waygroup']]
if item['flag8000']: value |= 0x8000
if item['flag4000']: value |= 0x4000
output += value.to_bytes(4, 'big')
output += b'\xff\xff\xff\xff'
return output
PADFLAG_INTPOS = 0x0001
PADFLAG_UPALIGNTOX = 0x0002
PADFLAG_UPALIGNTOY = 0x0004
PADFLAG_UPALIGNTOZ = 0x0008
PADFLAG_UPALIGNINVERT = 0x0010
PADFLAG_LOOKALIGNTOX = 0x0020
PADFLAG_LOOKALIGNTOY = 0x0040
PADFLAG_LOOKALIGNTOZ = 0x0080
PADFLAG_LOOKALIGNINVERT = 0x0100
PADFLAG_HASBBOXDATA = 0x0200
PADFLAG_AIWAITLIFT = 0x0400
PADFLAG_AIONLIFT = 0x0800
PADFLAG_AIWALKDIRECT = 0x1000
PADFLAG_AIDROP = 0x2000
PADFLAG_AIDUCK = 0x4000
PADFLAG_8000 = 0x8000
PADFLAG_10000 = 0x10000
app = App()
app.run()
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env python3
import assetmgr
import json
import os
def main():
fd = open('src/assets/%s/sequences.json' % os.environ['ROMID'], 'r')
data = fd.read()
fd.close()
rows = json.loads(data)
make_object(rows)
make_header(rows)
def make_header(rows):
typename = 'music'
enums = [row['id'] for row in rows]
filename = 'sequences.h'
terminator = 'MUSIC_END'
assetmgr.write_enums(typename, enums, filename, terminator)
def make_object(rows):
binary = len(rows).to_bytes(2, 'big') + b'\x00\x00'
datasection = bytes()
pos = 4 + len(rows) * 8
for row in rows:
contents = getcontents('src/assets/%s/sequences/%s' % (os.environ['ROMID'], row['file']))
zipped = assetmgr.zip(contents)
if len(zipped) % 2:
zipped += b'\x0a'
binary += pos.to_bytes(4, 'big')
binary += len(contents).to_bytes(2, 'big')
binary += len(zipped).to_bytes(2, 'big')
datasection += zipped
pos += len(zipped)
binary += datasection
assetmgr.write_object(binary, 'sequences.o')
def getcontents(filename):
with open(filename, 'rb') as fd:
return fd.read()
main()
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
import assetmgr
import json
import os
def main():
fd = open('src/assets/%s/textures.json' % os.environ['ROMID'], 'r')
data = fd.read()
fd.close()
rows = json.loads(data)
make_objects(rows)
make_header(rows)
def make_header(rows):
typename = 'texturenum'
enums = [row['id'] for row in rows]
filename = 'textures.h'
terminator = 'TEXTURE_END'
assetmgr.write_enums(typename, enums, filename, terminator)
def make_objects(rows):
databinary = bytes()
listbinary = bytes()
pos = 0
for row in rows:
contents = getcontents('src/assets/%s/textures/%s' % (os.environ['ROMID'], row['file']))
flags = ((row['flag00'] & 0x0f) << 4) | (row['surfacetype'] & 0x0f)
databinary += contents
listbinary += flags.to_bytes(1, 'big')
listbinary += pos.to_bytes(3, 'big')
listbinary += b'\x00\x00\x00\x00'
pos += len(contents)
listbinary += pos.to_bytes(4, 'big')
assetmgr.write_object(databinary, 'texturesdata.o')
assetmgr.write_object(listbinary, 'textureslist.o')
def getcontents(filename):
with open(filename, 'rb') as fd:
return fd.read()
main()
-45
View File
@@ -1,45 +0,0 @@
#!/usr/bin/env python3
import os
import subprocess
import sys
sys.path.append('src/assets/%s/sequences' % os.environ['ROMID'])
from sequences import sequences
def zip(filename):
return subprocess.check_output(['tools/rarezip', filename])
zips = []
fd = open('build/%s/assets/sequences.bin' % os.environ['ROMID'], 'wb')
fd.write(len(sequences).to_bytes(2, 'big'))
fd.seek(4)
offset = 4 + len(sequences) * 8
for name in sequences:
filename = 'src/assets/%s/sequences/%s' % (os.environ['ROMID'], name)
fd2 = open(filename, 'rb')
data = fd2.read()
fd2.close()
zipped = zip(filename)
ziplen = len(zipped)
if ziplen % 2 == 1:
zipped += (0x0a).to_bytes(1, 'big')
ziplen += 1
fd.write(offset.to_bytes(4, 'big'))
fd.write(len(data).to_bytes(2, 'big'))
fd.write(len(zipped).to_bytes(2, 'big'))
zips.append(zipped)
offset += len(zipped)
for zipdata in zips:
fd.write(zipdata)
fd.close()