summaryrefslogtreecommitdiff
path: root/tools/section2cpp.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/section2cpp.py')
-rw-r--r--tools/section2cpp.py612
1 files changed, 612 insertions, 0 deletions
diff --git a/tools/section2cpp.py b/tools/section2cpp.py
new file mode 100644
index 0000000000..1187abc8fa
--- /dev/null
+++ b/tools/section2cpp.py
@@ -0,0 +1,612 @@
+#!/usr/bin/env python3
+# PYTHON_ARGCOMPLETE_OK
+
+"""
+
+This script will extract literals and strings data
+from sections located in the baserom.dol.
+Useful when trying to match .rodata and .sdata2.
+
+Usage:
+./tools/section2cpp.py --section .rodata --string --object JKRSolidHeap.o
+
+"""
+
+import argparse
+import sys
+import os
+import struct
+from decimal import getcontext, Decimal
+from pathlib import Path, PurePath, PureWindowsPath
+from typing import (
+ Any,
+ Dict,
+ List,
+ Match,
+ NamedTuple,
+ NoReturn,
+ Optional,
+ Set,
+ Tuple,
+ Union,
+ Callable,
+ Pattern,
+)
+
+try:
+ import numpy
+except:
+ print("error: missing numpy")
+ sys.exit(1)
+
+
+try:
+ import argcomplete # type: ignore
+except ModuleNotFoundError:
+ argcomplete = None
+
+parser = argparse.ArgumentParser(description="Extract section data and generate C++ code.")
+
+parser.add_argument(
+ "--section",
+ dest="section",
+ type=str,
+ metavar="SECTION",
+ help="SECTION to extract data from.",
+ required=True
+)
+
+parser.add_argument(
+ "--file-offset",
+ dest="file_offset",
+ type=lambda x: int(x,0),
+ metavar="OFFSET",
+ help="OFFSET in the baserom for the SECTION."
+)
+
+parser.add_argument(
+ "--object",
+ dest="object_name",
+ type=str,
+ metavar="OBJECT",
+ help="OBJECT filename to extract data from. (e.g. JKRSolidHeap.o)"
+)
+
+parser.add_argument(
+ "--map",
+ dest="map_path",
+ type=str,
+ metavar="MAP",
+ help="frameworkF.map path",
+ default="frameworkF.map"
+)
+
+parser.add_argument(
+ "--baserom",
+ dest="baserom",
+ type=str,
+ metavar="DOL",
+ help="baserom.dol path",
+ default="baserom.dol"
+)
+
+parser.add_argument(
+ "--string",
+ dest="as_string",
+ action="store_true",
+ help="Print arrays as strings"
+)
+
+parser.add_argument(
+ "--array",
+ dest="as_array",
+ action="store_true",
+ help="Print everything as u8 arrays"
+)
+
+parser.add_argument(
+ "--shift-jis",
+ dest="shift_jis",
+ action="store_true",
+ help="Convert shift-jis to utf-8"
+)
+
+
+#
+#
+#
+
+def _itersplit(l, splitters):
+ current = []
+ for item in l:
+ if item in splitters:
+ yield current
+ current = []
+ else:
+ current.append(item)
+ yield current
+
+def magicsplit(l, *splitters):
+ return [subl for subl in _itersplit(l, splitters) ]
+
+def str_encoding(data):
+ if data[-1] != 0:
+ return None
+
+ try:
+ data.decode("utf-8")
+ return "utf-8"
+ except:
+ pass
+
+ try:
+ data.decode("shift_jisx0213")
+ return "shift-jis"
+ except:
+ pass
+
+ return None
+
+def encoding_char_list(encoding, data):
+ if args.shift_jis and encoding == "shift-jis":
+ try:
+ return escape(data.decode("shift_jisx0213"))
+ except:
+ pass
+ return [ str(bytes([x]))[2:-1].replace("\"", "\\\"") for x in data ]
+
+def raw_string(data):
+ return "".join(data)
+
+def raw_array(data):
+ return ",".join([hex(x) for x in list(data)])
+
+def escape_char(v):
+ if v == "\n":
+ return "\\n"
+ elif v == "\t":
+ return "\\t"
+ elif v == "\v":
+ return "\\v"
+ elif v == "\b":
+ return "\\b"
+ elif v == "\r":
+ return "\\r"
+ elif v == "\f":
+ return "\\f"
+ elif v == "\a":
+ return "\\a"
+ elif v == "\\":
+ return "\\\\"
+ elif v == "\"":
+ return "\\\""
+ elif ord(v) < 32 and ord(v) > 127:
+ return "\\x" + hex(v)[2:].upper().rjust(2, '0')
+ else:
+ return v
+
+def escape(v):
+ return "".join([ escape_char(x) for x in list(v) ])
+
+def bytes2float32(data):
+ if len(data) < 4:
+ return None
+ result = numpy.frombuffer(data[0:4][::-1], dtype='float32')
+ if result:
+ return result[0]
+ else:
+ return None
+
+def bytes2float64(data):
+ if len(data) < 8:
+ return None
+ result = numpy.frombuffer(data[0:8][::-1], dtype='float64')
+ if result:
+ return result[0]
+ else:
+ return None
+
+def is_nice_float32(f):
+ try:
+ if int(f*1000) == f*1000:
+ return True
+ if int(f*100) == f*100:
+ return True
+ if int(f*10) == f*10:
+ return True
+ if int(f) == f:
+ return True
+ except:
+ return False
+ return False
+
+def is_nice_float64(f):
+ try:
+ if int(f*1000) == f*1000:
+ return True
+ if int(f*100) == f*100:
+ return True
+ if int(f*10) == f*10:
+ return True
+ if int(f) == f:
+ return True
+ except:
+ return False
+ return False
+
+float32_exact: Dict[numpy.float32, Tuple[int,int]] = {}
+float64_exact: Dict[numpy.float64, Tuple[int,int]] = {}
+
+getcontext().prec = 64
+for i in range(1,32):
+ for j in range(1,32):
+ if i%j == 0:
+ continue
+ d = Decimal(i)/Decimal(j)
+ f = numpy.float32(d)
+ if str(f) != str(d):
+ if not f in float32_exact:
+ float32_exact[f] = (i,j)
+
+for i in range(1,32):
+ for j in range(1,32):
+ if i%j == 0:
+ continue
+ d = Decimal(i)/Decimal(j)
+ f = numpy.float64(d)
+ if str(f) != str(d):
+ if not f in float64_exact:
+ float64_exact[f] = (i,j)
+
+class Symbol:
+ def __init__(self, name, addr, size):
+ self.name = name
+ self.addr = addr
+ self.size = size
+ self.padding = 0
+
+ def __str__(self):
+ return " %s %s %s+%s %s" % (self.name.ljust(40, ' '), hex(self.addr), hex(self.addr + self.size), hex(self.padding), hex(self.size))
+
+class ObjectFile:
+ def __init__(self, path):
+ self.path = path
+ self.symbols = []
+ self.start = 0
+ self.end = 0
+ self.mk = False
+
+ def addSymbol(self, name, str_addr, str_size):
+ addr = int(str_addr, base=16)
+ size = int(str_size, base=16)
+
+ symbol = Symbol(name, addr, size)
+ if not self.symbols:
+ self.start = symbol.addr
+ else:
+ last_symbol = self.symbols[-1]
+ last_addr = last_symbol.addr + last_symbol.size
+ if last_addr != addr:
+ last_symbol.padding += addr - last_addr
+ self.symbols += [ symbol ]
+
+ def setEnd(self, end):
+ self.end = end
+ last_symbol = self.symbols[-1]
+ last_symbol.padding = self.end - (last_symbol.addr + last_symbol.size)
+
+def find_symbols():
+ file = map_path.open('r')
+ lines = file.readlines()
+
+ in_section = False
+ last_obj = None
+ for line in lines:
+ data = [ x.strip() for x in line.strip().split(" ") ]
+ data = [ x for x in data if len(x) > 0 ]
+
+ if len(data) == 3:
+ in_section = False
+ if data[0] == section:
+ in_section = True
+ continue
+
+ if not in_section:
+ continue
+ if len(data) < 6 or len(data) > 7:
+ continue
+
+ # get object filename
+ obj = data[5]
+ if len(data) > 6:
+ obj = data[6]
+
+ # remove path from object filename
+ obj = obj.split("\\")[-1]
+ if last_obj != obj:
+ assert obj not in object_map
+ object_map[obj] = ObjectFile(obj)
+ last_obj = obj
+
+ # add symbol
+ size = data[1]
+ addr = data[2]
+ name = data[4]
+ object_map[obj].addSymbol(name, addr, size)
+
+ keys = list(object_map.keys())
+ for i,_ in enumerate(keys[:-1]):
+ obj = object_map[keys[i]]
+ next_obj = object_map[keys[i + 1]]
+ obj.setEnd(next_obj.start)
+
+ # total size of rodata must be aligned to 0x20
+ obj = object_map[keys[-1]]
+ last_symbol = obj.symbols[-1]
+ last_addr = last_symbol.addr + last_symbol.size
+ last_symbol.padding = ((last_addr + 31) & ~31) - last_addr
+ file.close()
+
+def chunks(lst, n):
+ for i in range(0, len(lst), n):
+ yield lst[i:i + n]
+
+def data_as_string(data):
+ return ", ".join([ "0x" + hex(x)[2:].rjust(2, '0') for x in data ])
+
+
+class Literal:
+ def __init__(self, name, type, value, comment=None):
+ self.name = name
+ self.type = type
+ self.value = value
+ self.comment = comment
+
+ def format(self):
+ return str(self.value)
+
+ def lines(self):
+ line = "static const %s %s = %s;" % (self.type, self.name, self.format())
+ if self.comment:
+ line = line.ljust(90, ' ') + " // " + self.comment
+ return [ line ]
+
+ def __str__(self):
+ return "\n".join(self.lines())
+
+class Label(Literal):
+ def __init__(self, name):
+ super().__init__(name, "", None, None)
+
+ def lines(self):
+ return [ "", "", "// " + self.name ]
+
+class Float32Literal(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "float", value, comment)
+
+ def format(self):
+ return "%sf" % self.value
+
+class Float64Literal(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "double", value, comment)
+
+class FractionFloat32Literal(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "float", value, comment)
+
+ def format(self):
+ return "%i.0f / %i.0f" % self.value
+
+class FractionFloat64Literal(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "double", value, comment)
+
+ def format(self):
+ return "%i.0 / %i.0" % self.value
+
+class U32Literal(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "u32", value, comment)
+
+class S32Literal(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "s32", value, comment)
+
+class S64Literal(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "s64", value, comment)
+
+class U64Literal(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "u64", value, comment)
+
+class ArrayLiteral(Literal):
+ def __init__(self, name, value, comment=None):
+ super().__init__(name, "u8", value, comment)
+
+ def lines(self):
+ one_line = "static const %s %s[%i] = { %s };" % (self.type, self.name, len(self.value), data_as_string(self.value))
+
+ lines = []
+ if len(one_line) < 90:
+ lines += [ one_line ]
+ else:
+ lines += [ "static const %s %s[%i] = {" % (self.type, self.name, len(self.value)) ]
+ data_chunks = chunks(list(self.value), 16)
+ for chunk in data_chunks:
+ lines += [ " " + data_as_string(chunk) ]
+ lines += [ "};" ]
+
+ if lines and self.comment:
+ lines[0] = lines[0].ljust(90, ' ') + " // " + self.comment
+ return lines
+
+class StringLiteral(Literal):
+ def __init__(self, name, encoding, value, comment=None):
+ assert value[-1] == 0
+ super().__init__(name, "char", value[:-1], comment)
+ self.encoding = encoding
+
+ def lines(self):
+ char_list = encoding_char_list(self.encoding, self.value)
+ one_line = "static const %s %s = \"%s\";" % (self.type, self.name, raw_string(char_list))
+
+ lines = []
+ if len(one_line) < 90:
+ lines += [ one_line ]
+ else:
+ lines += [ "static const %s %s = " % (self.type, self.name) ]
+ data_chunks = chunks(char_list, 16)
+ for chunk in data_chunks:
+ lines += [ " \"%s\"" % raw_string(chunk) ]
+ lines[-1] += ";"
+
+ if lines and self.comment:
+ lines[0] = lines[0].ljust(90, ' ') + " // " + self.comment
+ return lines
+
+def output_cpp():
+ object_names = []
+ if object_name:
+ if not object_name in object_map:
+ print("error: %s object file not found!" % object_name)
+ sys.exit(1)
+ object_names += [ object_name ]
+ else:
+ object_names = [*object_map.keys()]
+
+ br = baserom.open("rb")
+ br.seek(0, os.SEEK_END)
+ br_size = br.tell()
+ br.seek(0, os.SEEK_SET)
+
+ literals = []
+ for obj_name in object_names:
+ literals += [ Label(obj_name) ]
+
+ obj = object_map[obj_name]
+ for symbol in obj.symbols:
+ label = "lbl_%s" % (hex(symbol.addr).upper()[2:])
+
+ symbol_file_offset = symbol.addr - file_offset
+ symbol_file_size = symbol.size + symbol.padding
+
+ if symbol_file_offset + symbol_file_size > br_size:
+ print("error: reading outside baserom file. (%i, %i)" % (symbol_file_offset + symbol_file_size, br_size))
+
+ br.seek(symbol_file_offset, os.SEEK_SET)
+ data = br.read(symbol.size)
+ padding = br.read(symbol.padding)
+
+ if args.as_string:
+ offset = 0
+ str_segments = [ x for x in magicsplit(data, 0) ]
+ for segment in str_segments[:-1]:
+ str_data = bytes(segment + [0])
+ encoding = str_encoding(str_data)
+
+ str_label = "lbl_%s" % (hex(symbol.addr + offset).upper()[2:])
+ if encoding == "shift-jis":
+ literals += [ StringLiteral(str_label, "shift-jis", str_data, "TODO: shift-jis strings in Metrowerks") ]
+ elif encoding == "utf-8":
+ literals += [ StringLiteral(str_label, "utf-8", str_data) ]
+ else:
+ literals += [ ArrayLiteral(str_label, str_data, "undecodable string") ]
+ offset += len(str_data)
+
+ if padding:
+ padding_label = "lbl_%s" % (hex(symbol.addr + symbol.size).upper()[2:])
+ literals += [ StringLiteral(padding_label, None, padding, "padding") ]
+ padding = None
+ elif args.as_array:
+ literals += [ ArrayLiteral(label, data) ]
+ else:
+ lit = None
+ if len(data) == 4:
+ u32_data = struct.unpack('>I', data)[0]
+ s32_data = struct.unpack('>i', data)[0]
+ float_data = bytes2float32(data)
+
+ if s32_data == 0 or (s32_data >= -4096 and s32_data <= 4096):
+ lit = S32Literal(label, s32_data)
+ elif u32_data == 0 or (u32_data < 4096):
+ lit = U32Literal(label, u32_data)
+ elif float_data in float32_exact:
+ lit = FractionFloat32Literal(label, float32_exact[float_data], "%sf %s" % (float_data, hex(u32_data)))
+ elif is_nice_float32(float_data):
+ lit = Float32Literal(label, float_data, hex(u32_data))
+
+ elif len(data) == 8:
+ u64_data = struct.unpack('>Q', data)[0]
+ s64_data = struct.unpack('>q', data)[0]
+ double_data = bytes2float64(data)
+
+ if u64_data == 0x4330000000000000:
+ lit = Float64Literal(label, double_data, "%s | u32 to float (compiler-generated)" % hex(u64_data))
+ elif u64_data == 0x4330000080000000:
+ lit = Float64Literal(label, double_data, "%s | s32 to float (compiler-generated)" % hex(u64_data))
+ elif s64_data == 0 or (s64_data >= -4096 and s64_data <= 4096):
+ lit = S64Literal(label, s64_data)
+ elif u64_data == 0 or (u64_data < 4096):
+ lit = U64Literal(label, u64_data)
+ elif double_data in float64_exact:
+ lit = FractionFloat64Literal(label, float64_exact[double_data], "%s %s" % (double_data, hex(u64_data)))
+ elif is_nice_float64(double_data):
+ lit = Float64Literal(label, double_data, hex(u64_data))
+
+ if not lit:
+ lit = ArrayLiteral(label, data)
+ literals += [ lit ]
+
+ if padding:
+ padding_label = "lbl_%s" % (hex(symbol.addr + symbol.size).upper()[2:])
+ literals += [ ArrayLiteral(padding_label, padding, "padding") ]
+
+ for lit in literals:
+ print(lit)
+
+ br.close()
+
+#
+#
+#
+
+try:
+ args = parser.parse_args()
+except:
+ parser.print_help()
+ sys.exit(0)
+
+section = args.section
+object_name = args.object_name
+file_offset: Optional[int] = args.file_offset
+baserom = Path(args.baserom)
+map_path = Path(args.map_path)
+
+file_offsets = {
+ ".rodata": 0x80003000,
+ ".sdata": 0x800802A0,
+ ".sdata2": 0x800811A0,
+}
+
+if not file_offset:
+ if not section in file_offsets:
+ print("error: missing --file-offset")
+ sys.exit(1)
+ else:
+ file_offset = file_offsets[section]
+
+if not baserom.exists():
+ print("error: baserom '%s' not found!" % args.baserom)
+ sys.exit(1)
+
+if not map_path.exists():
+ print("error: frameworkF.map '%s' not found!" % args.map_path)
+ sys.exit(1)
+
+object_map: Dict[str,ObjectFile] = {}
+
+find_symbols()
+output_cpp()
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-29 12:20:24 +0000