summaryrefslogtreecommitdiff
path: root/tools/section2cpp.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/section2cpp.py')
-rw-r--r--tools/section2cpp.py612
1 files changed, 0 insertions, 612 deletions
diff --git a/tools/section2cpp.py b/tools/section2cpp.py
deleted file mode 100644
index 1187abc8fa..0000000000
--- a/tools/section2cpp.py
+++ /dev/null
@@ -1,612 +0,0 @@
-#!/usr/bin/env python3
-# PYTHON_ARGCOMPLETE_OK
-
-"""
-
-This script will extract literals and strings data
-from sections located in the baserom.dol.
-Useful when trying to match .rodata and .sdata2.
-
-Usage:
-./tools/section2cpp.py --section .rodata --string --object JKRSolidHeap.o
-
-"""
-
-import argparse
-import sys
-import os
-import struct
-from decimal import getcontext, Decimal
-from pathlib import Path, PurePath, PureWindowsPath
-from typing import (
- Any,
- Dict,
- List,
- Match,
- NamedTuple,
- NoReturn,
- Optional,
- Set,
- Tuple,
- Union,
- Callable,
- Pattern,
-)
-
-try:
- import numpy
-except:
- print("error: missing numpy")
- sys.exit(1)
-
-
-try:
- import argcomplete # type: ignore
-except ModuleNotFoundError:
- argcomplete = None
-
-parser = argparse.ArgumentParser(description="Extract section data and generate C++ code.")
-
-parser.add_argument(
- "--section",
- dest="section",
- type=str,
- metavar="SECTION",
- help="SECTION to extract data from.",
- required=True
-)
-
-parser.add_argument(
- "--file-offset",
- dest="file_offset",
- type=lambda x: int(x,0),
- metavar="OFFSET",
- help="OFFSET in the baserom for the SECTION."
-)
-
-parser.add_argument(
- "--object",
- dest="object_name",
- type=str,
- metavar="OBJECT",
- help="OBJECT filename to extract data from. (e.g. JKRSolidHeap.o)"
-)
-
-parser.add_argument(
- "--map",
- dest="map_path",
- type=str,
- metavar="MAP",
- help="frameworkF.map path",
- default="frameworkF.map"
-)
-
-parser.add_argument(
- "--baserom",
- dest="baserom",
- type=str,
- metavar="DOL",
- help="baserom.dol path",
- default="baserom.dol"
-)
-
-parser.add_argument(
- "--string",
- dest="as_string",
- action="store_true",
- help="Print arrays as strings"
-)
-
-parser.add_argument(
- "--array",
- dest="as_array",
- action="store_true",
- help="Print everything as u8 arrays"
-)
-
-parser.add_argument(
- "--shift-jis",
- dest="shift_jis",
- action="store_true",
- help="Convert shift-jis to utf-8"
-)
-
-
-#
-#
-#
-
-def _itersplit(l, splitters):
- current = []
- for item in l:
- if item in splitters:
- yield current
- current = []
- else:
- current.append(item)
- yield current
-
-def magicsplit(l, *splitters):
- return [subl for subl in _itersplit(l, splitters) ]
-
-def str_encoding(data):
- if data[-1] != 0:
- return None
-
- try:
- data.decode("utf-8")
- return "utf-8"
- except:
- pass
-
- try:
- data.decode("shift_jisx0213")
- return "shift-jis"
- except:
- pass
-
- return None
-
-def encoding_char_list(encoding, data):
- if args.shift_jis and encoding == "shift-jis":
- try:
- return escape(data.decode("shift_jisx0213"))
- except:
- pass
- return [ str(bytes([x]))[2:-1].replace("\"", "\\\"") for x in data ]
-
-def raw_string(data):
- return "".join(data)
-
-def raw_array(data):
- return ",".join([hex(x) for x in list(data)])
-
-def escape_char(v):
- if v == "\n":
- return "\\n"
- elif v == "\t":
- return "\\t"
- elif v == "\v":
- return "\\v"
- elif v == "\b":
- return "\\b"
- elif v == "\r":
- return "\\r"
- elif v == "\f":
- return "\\f"
- elif v == "\a":
- return "\\a"
- elif v == "\\":
- return "\\\\"
- elif v == "\"":
- return "\\\""
- elif ord(v) < 32 and ord(v) > 127:
- return "\\x" + hex(v)[2:].upper().rjust(2, '0')
- else:
- return v
-
-def escape(v):
- return "".join([ escape_char(x) for x in list(v) ])
-
-def bytes2float32(data):
- if len(data) < 4:
- return None
- result = numpy.frombuffer(data[0:4][::-1], dtype='float32')
- if result:
- return result[0]
- else:
- return None
-
-def bytes2float64(data):
- if len(data) < 8:
- return None
- result = numpy.frombuffer(data[0:8][::-1], dtype='float64')
- if result:
- return result[0]
- else:
- return None
-
-def is_nice_float32(f):
- try:
- if int(f*1000) == f*1000:
- return True
- if int(f*100) == f*100:
- return True
- if int(f*10) == f*10:
- return True
- if int(f) == f:
- return True
- except:
- return False
- return False
-
-def is_nice_float64(f):
- try:
- if int(f*1000) == f*1000:
- return True
- if int(f*100) == f*100:
- return True
- if int(f*10) == f*10:
- return True
- if int(f) == f:
- return True
- except:
- return False
- return False
-
-float32_exact: Dict[numpy.float32, Tuple[int,int]] = {}
-float64_exact: Dict[numpy.float64, Tuple[int,int]] = {}
-
-getcontext().prec = 64
-for i in range(1,32):
- for j in range(1,32):
- if i%j == 0:
- continue
- d = Decimal(i)/Decimal(j)
- f = numpy.float32(d)
- if str(f) != str(d):
- if not f in float32_exact:
- float32_exact[f] = (i,j)
-
-for i in range(1,32):
- for j in range(1,32):
- if i%j == 0:
- continue
- d = Decimal(i)/Decimal(j)
- f = numpy.float64(d)
- if str(f) != str(d):
- if not f in float64_exact:
- float64_exact[f] = (i,j)
-
-class Symbol:
- def __init__(self, name, addr, size):
- self.name = name
- self.addr = addr
- self.size = size
- self.padding = 0
-
- def __str__(self):
- return " %s %s %s+%s %s" % (self.name.ljust(40, ' '), hex(self.addr), hex(self.addr + self.size), hex(self.padding), hex(self.size))
-
-class ObjectFile:
- def __init__(self, path):
- self.path = path
- self.symbols = []
- self.start = 0
- self.end = 0
- self.mk = False
-
- def addSymbol(self, name, str_addr, str_size):
- addr = int(str_addr, base=16)
- size = int(str_size, base=16)
-
- symbol = Symbol(name, addr, size)
- if not self.symbols:
- self.start = symbol.addr
- else:
- last_symbol = self.symbols[-1]
- last_addr = last_symbol.addr + last_symbol.size
- if last_addr != addr:
- last_symbol.padding += addr - last_addr
- self.symbols += [ symbol ]
-
- def setEnd(self, end):
- self.end = end
- last_symbol = self.symbols[-1]
- last_symbol.padding = self.end - (last_symbol.addr + last_symbol.size)
-
-def find_symbols():
- file = map_path.open('r')
- lines = file.readlines()
-
- in_section = False
- last_obj = None
- for line in lines:
- data = [ x.strip() for x in line.strip().split(" ") ]
- data = [ x for x in data if len(x) > 0 ]
-
- if len(data) == 3:
- in_section = False
- if data[0] == section:
- in_section = True
- continue
-
- if not in_section:
- continue
- if len(data) < 6 or len(data) > 7:
- continue
-
- # get object filename
- obj = data[5]
- if len(data) > 6:
- obj = data[6]
-
- # remove path from object filename
- obj = obj.split("\\")[-1]
- if last_obj != obj:
- assert obj not in object_map
- object_map[obj] = ObjectFile(obj)
- last_obj = obj
-
- # add symbol
- size = data[1]
- addr = data[2]
- name = data[4]
- object_map[obj].addSymbol(name, addr, size)
-
- keys = list(object_map.keys())
- for i,_ in enumerate(keys[:-1]):
- obj = object_map[keys[i]]
- next_obj = object_map[keys[i + 1]]
- obj.setEnd(next_obj.start)
-
- # total size of rodata must be aligned to 0x20
- obj = object_map[keys[-1]]
- last_symbol = obj.symbols[-1]
- last_addr = last_symbol.addr + last_symbol.size
- last_symbol.padding = ((last_addr + 31) & ~31) - last_addr
- file.close()
-
-def chunks(lst, n):
- for i in range(0, len(lst), n):
- yield lst[i:i + n]
-
-def data_as_string(data):
- return ", ".join([ "0x" + hex(x)[2:].rjust(2, '0') for x in data ])
-
-
-class Literal:
- def __init__(self, name, type, value, comment=None):
- self.name = name
- self.type = type
- self.value = value
- self.comment = comment
-
- def format(self):
- return str(self.value)
-
- def lines(self):
- line = "static const %s %s = %s;" % (self.type, self.name, self.format())
- if self.comment:
- line = line.ljust(90, ' ') + " // " + self.comment
- return [ line ]
-
- def __str__(self):
- return "\n".join(self.lines())
-
-class Label(Literal):
- def __init__(self, name):
- super().__init__(name, "", None, None)
-
- def lines(self):
- return [ "", "", "// " + self.name ]
-
-class Float32Literal(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "float", value, comment)
-
- def format(self):
- return "%sf" % self.value
-
-class Float64Literal(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "double", value, comment)
-
-class FractionFloat32Literal(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "float", value, comment)
-
- def format(self):
- return "%i.0f / %i.0f" % self.value
-
-class FractionFloat64Literal(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "double", value, comment)
-
- def format(self):
- return "%i.0 / %i.0" % self.value
-
-class U32Literal(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "u32", value, comment)
-
-class S32Literal(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "s32", value, comment)
-
-class S64Literal(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "s64", value, comment)
-
-class U64Literal(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "u64", value, comment)
-
-class ArrayLiteral(Literal):
- def __init__(self, name, value, comment=None):
- super().__init__(name, "u8", value, comment)
-
- def lines(self):
- one_line = "static const %s %s[%i] = { %s };" % (self.type, self.name, len(self.value), data_as_string(self.value))
-
- lines = []
- if len(one_line) < 90:
- lines += [ one_line ]
- else:
- lines += [ "static const %s %s[%i] = {" % (self.type, self.name, len(self.value)) ]
- data_chunks = chunks(list(self.value), 16)
- for chunk in data_chunks:
- lines += [ " " + data_as_string(chunk) ]
- lines += [ "};" ]
-
- if lines and self.comment:
- lines[0] = lines[0].ljust(90, ' ') + " // " + self.comment
- return lines
-
-class StringLiteral(Literal):
- def __init__(self, name, encoding, value, comment=None):
- assert value[-1] == 0
- super().__init__(name, "char", value[:-1], comment)
- self.encoding = encoding
-
- def lines(self):
- char_list = encoding_char_list(self.encoding, self.value)
- one_line = "static const %s %s = \"%s\";" % (self.type, self.name, raw_string(char_list))
-
- lines = []
- if len(one_line) < 90:
- lines += [ one_line ]
- else:
- lines += [ "static const %s %s = " % (self.type, self.name) ]
- data_chunks = chunks(char_list, 16)
- for chunk in data_chunks:
- lines += [ " \"%s\"" % raw_string(chunk) ]
- lines[-1] += ";"
-
- if lines and self.comment:
- lines[0] = lines[0].ljust(90, ' ') + " // " + self.comment
- return lines
-
-def output_cpp():
- object_names = []
- if object_name:
- if not object_name in object_map:
- print("error: %s object file not found!" % object_name)
- sys.exit(1)
- object_names += [ object_name ]
- else:
- object_names = [*object_map.keys()]
-
- br = baserom.open("rb")
- br.seek(0, os.SEEK_END)
- br_size = br.tell()
- br.seek(0, os.SEEK_SET)
-
- literals = []
- for obj_name in object_names:
- literals += [ Label(obj_name) ]
-
- obj = object_map[obj_name]
- for symbol in obj.symbols:
- label = "lbl_%s" % (hex(symbol.addr).upper()[2:])
-
- symbol_file_offset = symbol.addr - file_offset
- symbol_file_size = symbol.size + symbol.padding
-
- if symbol_file_offset + symbol_file_size > br_size:
- print("error: reading outside baserom file. (%i, %i)" % (symbol_file_offset + symbol_file_size, br_size))
-
- br.seek(symbol_file_offset, os.SEEK_SET)
- data = br.read(symbol.size)
- padding = br.read(symbol.padding)
-
- if args.as_string:
- offset = 0
- str_segments = [ x for x in magicsplit(data, 0) ]
- for segment in str_segments[:-1]:
- str_data = bytes(segment + [0])
- encoding = str_encoding(str_data)
-
- str_label = "lbl_%s" % (hex(symbol.addr + offset).upper()[2:])
- if encoding == "shift-jis":
- literals += [ StringLiteral(str_label, "shift-jis", str_data, "TODO: shift-jis strings in Metrowerks") ]
- elif encoding == "utf-8":
- literals += [ StringLiteral(str_label, "utf-8", str_data) ]
- else:
- literals += [ ArrayLiteral(str_label, str_data, "undecodable string") ]
- offset += len(str_data)
-
- if padding:
- padding_label = "lbl_%s" % (hex(symbol.addr + symbol.size).upper()[2:])
- literals += [ StringLiteral(padding_label, None, padding, "padding") ]
- padding = None
- elif args.as_array:
- literals += [ ArrayLiteral(label, data) ]
- else:
- lit = None
- if len(data) == 4:
- u32_data = struct.unpack('>I', data)[0]
- s32_data = struct.unpack('>i', data)[0]
- float_data = bytes2float32(data)
-
- if s32_data == 0 or (s32_data >= -4096 and s32_data <= 4096):
- lit = S32Literal(label, s32_data)
- elif u32_data == 0 or (u32_data < 4096):
- lit = U32Literal(label, u32_data)
- elif float_data in float32_exact:
- lit = FractionFloat32Literal(label, float32_exact[float_data], "%sf %s" % (float_data, hex(u32_data)))
- elif is_nice_float32(float_data):
- lit = Float32Literal(label, float_data, hex(u32_data))
-
- elif len(data) == 8:
- u64_data = struct.unpack('>Q', data)[0]
- s64_data = struct.unpack('>q', data)[0]
- double_data = bytes2float64(data)
-
- if u64_data == 0x4330000000000000:
- lit = Float64Literal(label, double_data, "%s | u32 to float (compiler-generated)" % hex(u64_data))
- elif u64_data == 0x4330000080000000:
- lit = Float64Literal(label, double_data, "%s | s32 to float (compiler-generated)" % hex(u64_data))
- elif s64_data == 0 or (s64_data >= -4096 and s64_data <= 4096):
- lit = S64Literal(label, s64_data)
- elif u64_data == 0 or (u64_data < 4096):
- lit = U64Literal(label, u64_data)
- elif double_data in float64_exact:
- lit = FractionFloat64Literal(label, float64_exact[double_data], "%s %s" % (double_data, hex(u64_data)))
- elif is_nice_float64(double_data):
- lit = Float64Literal(label, double_data, hex(u64_data))
-
- if not lit:
- lit = ArrayLiteral(label, data)
- literals += [ lit ]
-
- if padding:
- padding_label = "lbl_%s" % (hex(symbol.addr + symbol.size).upper()[2:])
- literals += [ ArrayLiteral(padding_label, padding, "padding") ]
-
- for lit in literals:
- print(lit)
-
- br.close()
-
-#
-#
-#
-
-try:
- args = parser.parse_args()
-except:
- parser.print_help()
- sys.exit(0)
-
-section = args.section
-object_name = args.object_name
-file_offset: Optional[int] = args.file_offset
-baserom = Path(args.baserom)
-map_path = Path(args.map_path)
-
-file_offsets = {
- ".rodata": 0x80003000,
- ".sdata": 0x800802A0,
- ".sdata2": 0x800811A0,
-}
-
-if not file_offset:
- if not section in file_offsets:
- print("error: missing --file-offset")
- sys.exit(1)
- else:
- file_offset = file_offsets[section]
-
-if not baserom.exists():
- print("error: baserom '%s' not found!" % args.baserom)
- sys.exit(1)
-
-if not map_path.exists():
- print("error: frameworkF.map '%s' not found!" % args.map_path)
- sys.exit(1)
-
-object_map: Dict[str,ObjectFile] = {}
-
-find_symbols()
-output_cpp()
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-03 22:35:24 +0000