get exdschema implemented for some base name setting

luminapie/definitions.py

@@ -0,0 +1,46 @@
+def get_definition(schema: dict[str, str]) -> dict[str, str]:
+    if 'type' in schema:
+        if schema['type'] == 'array':
+            return RepeatDefinition(schema)
+    return Definition(schema)
+
+
+class Definition:
+    def __init__(self, obj: dict[str, str]) -> None:
+        self.name = obj['name']
+
+    def __repr__(self) -> str:
+        return self.name
+
+
+class RepeatDefinition(Definition):
+    def __init__(self, obj: dict[str, str]) -> None:
+        super().__init__(obj)
+        self.obj = obj
+        self.count = obj['count']
+        self.inner_defs = []
+        self.process_inner()
+
+    def process_inner(self):
+        if 'fields' in self.obj:
+            for field in self.obj['fields']:
+                if 'name' in field:
+                    self.inner_defs.append(get_definition(field))
+                else:
+                    self.inner_defs.append(Definition({'name': ""}))
+        if self.inner_defs == []:
+            self.inner_defs.append(Definition({'name': ""}))
+
+    def flatten(self, extern: str) -> list[Definition]:
+        defs = []
+        extern = extern + self.name
+        for i in range(0, int(self.count)):
+            for inner in self.inner_defs:
+                if isinstance(inner, RepeatDefinition):
+                    defs.extend(inner.flatten(extern + i.__str__()))
+                else:
+                    defs.append(Definition({'name': extern + i.__str__() + inner.name}))
+        return defs
+
+    def __repr__(self) -> str:
+        return f'{self.flatten("")}'

luminapie/excel.py

@@ -1,4 +1,5 @@
 from luminapie.enums import ExcelColumnDataType
+from luminapie.definitions import Definition
 
 
 class ExcelListFile:
@@ -19,6 +20,9 @@ def parse(self):
                 continue
             self.dict[int(linearr[1])] = linearr[0]
 
+    def __repr__(self):
+        return f'''ExcelListFile: {self.header}, {self.dict}'''
+
 
 class ExcelHeader:
     def __init__(self, data: bytes):
@@ -52,8 +56,11 @@ def parse(self):
         self.type = ExcelColumnDataType(int.from_bytes(self.data[0:2], 'big'))
         self.offset = int.from_bytes(self.data[2:4], 'big')
 
-    def __repr__(self):
-        return f'''[{self.type.name}, {self.offset:x}]'''
+    def __lt__(self, other: 'ExcelColumnDefinition') -> bool:
+        return self.offset <= other.offset
+
+    def __repr__(self) -> str:
+        return f'''Column: {self.type.name}, offset: {self.offset:x}'''
 
 
 class ExcelDataPagination:
@@ -66,7 +73,7 @@ def parse(self):
         self.row_count = int.from_bytes(self.data[2:4], 'big')
 
     def __repr__(self):
-        return f'''[{self.start_id:x}, {self.row_count}]'''
+        return f'''Pagination: {self.start_id:x}, count: {self.row_count}'''
 
 
 class ExcelHeaderFile:
@@ -85,6 +92,7 @@ def parse(self):
         self.column_definitions: list[ExcelColumnDefinition] = []
         for i in range(self.header.column_count):
             self.column_definitions.append(ExcelColumnDefinition(self.data[32 + (i * 4) : 32 + ((i + 1) * 4)]))
+        self.column_definitions = sorted(self.column_definitions)
         self.pagination: list[ExcelDataPagination] = []
         for i in range(self.header.page_count):
             self.pagination.append(
@@ -102,8 +110,7 @@ def parse(self):
         for i in range(self.header.language_count):
             self.languages.append(self.data[32 + (self.header.column_count * 4) + (self.header.page_count * 4) + i])
 
-    def map_names(self, names: dict[int, str] = {}) -> tuple[dict[int, tuple[str, str]], int]:
-        """Maps the header column definitions to names and c types."""
+    def map_names(self, names: list[Definition]) -> tuple[dict[int, tuple[str, str]], int]:
         mapped: dict[int, tuple[str, str]] = {}
         largest_offset_index: int = 0
         for i in range(self.header.column_count):
@@ -119,32 +126,21 @@ def map_names(self, names: dict[int, str] = {}) -> tuple[dict[int, tuple[str, st
                 [_, name] = mapped[self.column_definitions[i].offset]
                 if name.split('_')[0] == 'Unknown':
                     continue
-                if i not in names:
-                    continue
                 if column_data_type_to_c_type(self.column_definitions[i].type) != 'unsigned __int8':
                     continue
                 else:
                     mapped[self.column_definitions[i].offset] = (
                         column_data_type_to_c_type(self.column_definitions[i].type),
-                        f'{name}_{names[i]}',
+                        f'{name}_{names[i].name}',
                     )
             else:
-                if i not in names:
-                    mapped[self.column_definitions[i].offset] = (
-                        column_data_type_to_c_type(self.column_definitions[i].type),
-                        f'Unknown_{self.column_definitions[i].offset:X}',
-                    )
-                else:
-                    mapped[self.column_definitions[i].offset] = (
-                        column_data_type_to_c_type(self.column_definitions[i].type),
-                        names[i],
-                    )
+                mapped[self.column_definitions[i].offset] = (
+                    column_data_type_to_c_type(self.column_definitions[i].type),
+                    names[i].name,
+                )
         mapped = dict(sorted(mapped.items()))
         return [mapped, size]
 
-    def __repr__(self):
-        return f'''ExcelHeaderFile: {self.header} , {self.column_definitions} , {self.pagination} , {self.languages}'''
-
 
 def column_data_type_to_c_type(column_data_type: ExcelColumnDataType) -> str:
     if column_data_type == ExcelColumnDataType.Bool:
@@ -177,9 +173,9 @@ def column_data_type_to_c_type(column_data_type: ExcelColumnDataType) -> str:
         or column_data_type == ExcelColumnDataType.PackedBool6
         or column_data_type == ExcelColumnDataType.PackedBool7
     ):
-        return 'unsigned __int8'  # IDA doesn't support bitfields in decompilation, so we'll just use a byte. A different method would be to create an enum for each bitfield, but that's a lot of work that i cant be bothered doing.
+        return 'unsigned __int8'
     elif column_data_type == ExcelColumnDataType.String:
-        return '__unsigned __int32'  # strings are stored as a 4 byte offset to a string table, so we'll just use a 4 byte integer since another function handles reasign of strings.
+        return '__unsigned __int32'
 
 
 def column_data_type_to_size(column_data_type: ExcelColumnDataType) -> int:

luminapie/exdschema.py

@@ -0,0 +1,73 @@
+from urllib.request import urlopen, Request
+from urllib.error import HTTPError, URLError
+from json import loads
+from luminapie.game_data import SemanticVersion
+from luminapie.definitions import Definition, RepeatDefinition, get_definition
+from typing import Union
+from yaml import load, Loader
+from zipfile import ZipFile
+from tempfile import TemporaryFile
+
+EXDSchemaFields = dict[str, Union[str, list[str], 'EXDSchemaFields']]
+
+
+def get_url(url: str, supress: bool = False) -> bytes | None:
+    req = Request(url)
+    try:
+        resp = urlopen(req)
+        return resp.read()
+    except HTTPError as e:
+        if not supress:
+            print('HTTP Error code: ', e.code, ' for url: ', url)
+        return None
+    except URLError as e:
+        if not supress:
+            print('HTTP Reason: ', e.reason, ' for url: ', url)
+        return None
+
+
+def get_latest_schema() -> dict[SemanticVersion, str]:
+    json = loads(get_url('https://api.github.com/repos/xivdev/EXDSchema/releases/latest'))
+    assetsJson = json['assets']
+    assets = {}
+    for asset in assetsJson:
+        version = SemanticVersion(*(int(x) for x in asset['name'].split('.')[0:5]))
+        assets[version] = asset['browser_download_url']
+    assets = dict(sorted(assets.items()))
+    return assets
+
+
+def get_latest_schema_url(ver: SemanticVersion) -> str:
+    latest_release = get_latest_schema()
+    # check if the current version can be retrieved
+    if ver in latest_release:
+        return latest_release[ver]
+    # grab the version before the current version if it can't be retrieved
+    for version in latest_release:
+        if version < ver:
+            return latest_release[version]
+
+
+def get_definitions(schema: SemanticVersion) -> dict[str, list[Definition]]:
+    exd_schema_map = {}
+    with TemporaryFile() as f:
+        f.write(get_url(get_latest_schema_url(schema), True))
+        f.seek(0)
+        schema_zip = ZipFile(f)
+
+        for file in schema_zip.namelist():
+            if file.endswith('.yml'):
+                schema_yml = load(schema_zip.read(file), Loader=Loader)
+                exd_schema_map[file.rsplit('.', 1)[0].rsplit('/', 1)[1]] = schema_yml['fields']
+
+    defs_map = {}
+    for exd in exd_schema_map:
+        defs = []
+        for field in exd_schema_map[exd]:
+            defin = get_definition(field)
+            if isinstance(defin, RepeatDefinition):
+                defs.extend(defin.flatten(""))
+            else:
+                defs.append(defin)
+        defs_map[exd] = defs
+    return defs_map

luminapie/game_data.py

@@ -6,6 +6,49 @@
 crc = Crc32()
 
 
+class SemanticVersion:
+    """Represents a semantic version string that can compare versions"""
+
+    year: int
+    month: int
+    date: int
+    patch: int
+    build: int
+
+    def __init__(self, year: int, month: int, date: int, patch: int, build: int = 0) -> None:
+        self.year = year
+        self.month = month
+        self.date = date
+        self.patch = patch
+        self.build = build
+
+    def __lt__(self, other: 'SemanticVersion') -> bool:
+        return (
+            self.year < other.year
+            or self.month < other.month
+            or self.date < other.date
+            or self.patch < other.patch
+            or self.build < other.build
+        )
+
+    def __repr__(self) -> str:
+        return f'{self.year}.{self.month.__str__().rjust(2, "0")}.{self.date.__str__().rjust(2, "0")}.{self.patch.__str__().rjust(4, "0")}.{self.build.__str__().rjust(4, "0")}'
+
+    def __eq__(self, __value: object) -> bool:
+        if not isinstance(__value, SemanticVersion):
+            return False
+        return (
+            self.year == __value.year
+            and self.month == __value.month
+            and self.date == __value.date
+            and self.patch == __value.patch
+            and self.build == __value.build
+        )
+
+    def __hash__(self) -> int:
+        return hash(repr(self))
+
+
 class Repository:
     def __init__(self, name: str, root: str):
         self.root = root
@@ -27,9 +70,9 @@ def parse_version(self):
             versionPath = os.path.join(self.root, 'sqpack', self.name, self.name + '.ver')
         if os.path.exists(versionPath):
             with open(versionPath, 'r') as f:
-                self.version = f.read().strip()
+                self.version = SemanticVersion(*(int(v) for v in f.read().strip().split('.')))
         else:
-            self.version = "Unknown"
+            self.version = SemanticVersion(0, 0, 0, 0)
 
     def setup_indexes(self):
         for file in get_sqpack_index(self.root, self.name):

luminapie/se_crc.py

@@ -60,10 +60,10 @@ def calc_index(self, path: str):
         filename = path_parts[-1]
         folder = path.rstrip(filename).rstrip('/')
 
-        folder_crc = self.calc(folder.encode('utf-8'))
-        file_crc = self.calc(filename.encode('utf-8'))
+        foldercrc = self.calc(folder.encode('utf-8'))
+        filecrc = self.calc(filename.encode('utf-8'))
 
-        return folder_crc << 32 | file_crc
+        return foldercrc << 32 | filecrc
 
     def calc_index2(self, path: str):
         return self.calc(path.encode('utf-8'))

main.py

@@ -1,18 +1,28 @@
 import os
 import json
 from luminapie.game_data import GameData, ParsedFileName
-from luminapie.excel import ExcelListFile
+from luminapie.excel import ExcelListFile, ExcelHeaderFile
+from luminapie.exdschema import get_definitions
 
 
 def main():
     f = open(os.path.join(os.getenv('APPDATA'), 'XIVLauncher', 'launcherConfigV3.json'), 'r')
     config = json.load(f)
     f.close()
-    # game_data = GameData(os.path.join(config['GamePath'], 'game'))
-    game_data = GameData("C:\\Users\\magnu\\Downloads\\ffxiv-dawntrail-bench\\game")
+    game_data = GameData(os.path.join(config['GamePath'], 'game'))
     exd_map = ExcelListFile(game_data.get_file(ParsedFileName('exd/root.exl'))).dict
+
+    exd_headers: dict[int, tuple[dict[int, tuple[str, str]], int]] = {}
+
+    exd_schema = get_definitions(game_data.repositories[0].version)
+
     for key in exd_map:
-        print(key, exd_map[key])
+        print(f'Parsing schema for {exd_map[key]}')
+        exd_headers[key] = ExcelHeaderFile(game_data.get_file(ParsedFileName(f'exd/{exd_map[key]}.exh'))).map_names(
+            exd_schema[exd_map[key]]
+        )
+
+    print(exd_headers)
 
 
 main()