Skip to content

Commit

Permalink
Merge pull request #3 from Nenkai/pack-builder
Browse files Browse the repository at this point in the history 
feat: Initial packing support.
  • Loading branch information
@Nenkai
Nenkai committed Aug 21, 2024
2 parents 347aef4 + a64d861 commit 75d9c26
Show file tree
Hide file tree
Showing 10 changed files with 656 additions and 40 deletions.
4 changes: 2 additions & 2 deletions FF16PackLib.CLI/FF16PackLib.CLI.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<OutputType>Exe</OutputType>
<AssemblyVersion>1.0.1.0</AssemblyVersion>
<FileVersion>1.0.1.0</FileVersion>
<AssemblyVersion>1.0.1.0</AssemblyVersion>
<FileVersion>1.0.1.0</FileVersion>
</PropertyGroup>

<ItemGroup>
Expand Down
48 changes: 46 additions & 2 deletions FF16PackLib.CLI/Program.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using FF16PackLib;
using FF16PackLib.Packing;

namespace FF16PackLib.CLI;

Expand All @@ -18,10 +19,11 @@ static void Main(string[] args)
Console.WriteLine("-----------------------------------------");
Console.WriteLine("");

var p = Parser.Default.ParseArguments<UnpackFileVerbs, UnpackAllVerbs, ListFilesVerbs>(args)
var p = Parser.Default.ParseArguments<UnpackFileVerbs, UnpackAllVerbs, ListFilesVerbs, PackVerbs>(args)
.WithParsed<UnpackFileVerbs>(UnpackFile)
.WithParsed<UnpackAllVerbs>(UnpackAll)
.WithParsed<ListFilesVerbs>(ListFiles);
.WithParsed<ListFilesVerbs>(ListFiles)
.WithParsed<PackVerbs>(PackFiles);
}

static void UnpackFile(UnpackFileVerbs verbs)
Expand Down Expand Up @@ -109,10 +111,36 @@ static void ListFiles(ListFilesVerbs verbs)
}
}

static void PackFiles(PackVerbs verbs)
{
if (!Directory.Exists(verbs.InputFile))
{
Console.WriteLine($"ERROR: Directory '{verbs.InputFile}' does not exist.");
return;
}

var builder = new FF16PackBuilder(new PackBuildOptions()
{
Encrypt = verbs.Encrypt,
Name = verbs.Name,
});

if (string.IsNullOrEmpty(verbs.OutputFile))
{
string fileName = Path.GetFileNameWithoutExtension(verbs.InputFile);
verbs.OutputFile = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(verbs.InputFile)), $"{fileName}.pac");
}

builder.InitFromDirectory(verbs.InputFile);
builder.WriteTo(verbs.OutputFile);

Console.WriteLine("Done.");
}

public static void DumpPack(FF16Pack pack)
{
Console.WriteLine($"Pack Info:");
Console.WriteLine($"- Internal Archive Name/Dir: {(string.IsNullOrEmpty(pack.ArchiveDir) ? "(none)" : pack.ArchiveDir)}");
Console.WriteLine($"- Num Files: {pack.GetNumFiles()}");
Console.WriteLine($"- Chunks: {pack.GetNumChunks()}");
Console.WriteLine($"- Header Encryption: {pack.HeaderEncrypted}");
Expand Down Expand Up @@ -143,6 +171,22 @@ public class UnpackAllVerbs
public string OutputPath { get; set; }
}

[Verb("pack", HelpText = "Pack files from a directory.")]
public class PackVerbs
{
[Option('i', "input", Required = true, HelpText = "Input directory")]
public string InputFile { get; set; }

[Option('o', "output", HelpText = "Output '.pac' file.")]
public string OutputFile { get; set; }

[Option('n', "name", HelpText = "Name of the pack file.")]
public string Name { get; set; }

[Option('e', "encrypt", HelpText = "Whether to encrypt the header.")]
public bool Encrypt { get; set; }
}

[Verb("list-files", HelpText = "List files in a .pac file.")]
public class ListFilesVerbs
{
Expand Down
2 changes: 1 addition & 1 deletion FF16PackLib.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.10.35122.118
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FF16Pack.CLI", "FF16PackLib.CLI\FF16PackLib.CLI.csproj", "{2D5761A8-95D1-4C70-BE35-245412FA44F1}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FF16PackLib.CLI", "FF16PackLib.CLI\FF16PackLib.CLI.csproj", "{2D5761A8-95D1-4C70-BE35-245412FA44F1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FF16PackLib", "FF16PackLib\FF16PackLib.csproj", "{A4B77394-4F7D-4AC9-B475-6FCDAED32521}"
EndProject
Expand Down
39 changes: 39 additions & 0 deletions FF16PackLib/Crypto/XorEncrypt.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;

namespace FF16PackLib.Crypto;

public class XorEncrypt
{
public const ulong XOR_KEY = 0x49D18FC870F3824E;

public static void CryptHeaderPart(Span<byte> data)
{
Span<byte> cur = data;
while (cur.Length >= 8)
{
MemoryMarshal.Cast<byte, ulong>(cur)[0] ^= XOR_KEY;
cur = cur[8..];
}

if (cur.Length >= 4)
{
MemoryMarshal.Cast<byte, uint>(cur)[0] ^= (uint)(XOR_KEY & 0xFFFFFFFF);
cur = cur[4..];
}

if (cur.Length >= 2)
{
MemoryMarshal.Cast<byte, ushort>(cur)[0] ^= (ushort)(XOR_KEY & 0xFFFF);
cur = cur[2..];
}

if (cur.Length >= 1)
cur[0] ^= (byte)(XOR_KEY & 0xFF);
}
}
52 changes: 19 additions & 33 deletions FF16PackLib/FF16Pack.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using Syroot.BinaryData;

using FF16PackLib.Hashing;
using FF16PackLib.Crypto;

namespace FF16PackLib;

Expand All @@ -23,10 +24,14 @@ namespace FF16PackLib;
public class FF16Pack : IDisposable
{
public const uint MAGIC = 0x4B434150;
public const int MAX_DECOMPRESSED_CHUNK_SIZE = 0x80000;
public const uint HEADER_SIZE = 0x400;

public const ulong XOR_KEY = 0x49D18FC870F3824E;
public const int MIN_FILE_SIZE_FOR_MULTIPLE_CHUNKS = 0x2000000;
public const int MAX_FILE_SIZE_FOR_SHARED_CHUNK = 0x100000;
public const int MAX_DECOMPRESSED_SHARED_CHUNK_SIZE = 0x400000;
public const int MAX_DECOMPRESSED_MULTI_CHUNK_SIZE = 0x80000;

public string ArchiveDir { get; private set; }
public bool HeaderEncrypted { get; set; }
public bool UseChunks { get; set; }

Expand Down Expand Up @@ -73,15 +78,17 @@ public static FF16Pack Open(string path)
bs.Position = 0x18;

if (pack.HeaderEncrypted)
DecryptHeaderPart(header.AsSpan(0x18, 0x100));
XorEncrypt.CryptHeaderPart(header.AsSpan(0x18, 0x100));

pack.ArchiveDir = Encoding.UTF8.GetString(header.AsSpan(0x18, 0x100)).TrimEnd('\0');

bs.Position = 0x118;
ulong chunksTableOffset = bs.ReadUInt64();
ulong stringsOffset = bs.ReadUInt64();
ulong stringTableSize = bs.ReadUInt64();

if (pack.HeaderEncrypted)
DecryptHeaderPart(header.AsSpan((int)stringsOffset, (int)stringTableSize));
XorEncrypt.CryptHeaderPart(header.AsSpan((int)stringsOffset, (int)stringTableSize));

for (int i = 0; i < numFiles; i++)
{
Expand Down Expand Up @@ -193,6 +200,10 @@ public void ExtractFile(string path, string outputDir)

public void ListFiles(string outputPath)
{
var exts = _files.Where(e => !e.Value.IsCompressed).Select(e => Path.GetExtension(e.Key)).Distinct();
foreach (var ext in exts)
Console.WriteLine(ext);

using var sw = new StreamWriter(outputPath);
foreach (var file in _files)
{
Expand All @@ -208,8 +219,8 @@ private void ExtractFileFromMultipleChunks(FF16PackFile packFile, string outputP
uint size = _stream.ReadUInt32();
uint[] chunkOffsets = _stream.ReadUInt32s((int)numChunks);

byte[] compBuffer = ArrayPool<byte>.Shared.Rent(MAX_DECOMPRESSED_CHUNK_SIZE);
byte[] decompBuffer = ArrayPool<byte>.Shared.Rent(MAX_DECOMPRESSED_CHUNK_SIZE);
byte[] compBuffer = ArrayPool<byte>.Shared.Rent(0x100000);
byte[] decompBuffer = ArrayPool<byte>.Shared.Rent(MAX_DECOMPRESSED_MULTI_CHUNK_SIZE);

var crc = new Crc32();
long remSize = (long)packFile.DecompressedFileSize;
Expand All @@ -219,7 +230,7 @@ private void ExtractFileFromMultipleChunks(FF16PackFile packFile, string outputP
int chunkCompSize = i < numChunks - 1 ?
(int)(chunkOffsets[i + 1] - chunkOffsets[i])
: (int)packFile.CompressedFileSize - (int)chunkOffsets[i];
int chunkDecompSize = (int)Math.Min(remSize, MAX_DECOMPRESSED_CHUNK_SIZE);
int chunkDecompSize = (int)Math.Min(remSize, MAX_DECOMPRESSED_MULTI_CHUNK_SIZE);

_stream.Position = (long)(packFile.DataOffset + chunkOffsets[i]);
_stream.Read(compBuffer, 0, chunkCompSize);
Expand Down Expand Up @@ -258,7 +269,7 @@ private void ExtractFileFromSpecificChunk(FF16PackFile packFile, string outputPa
fixed (byte* buffer = compBuffer)
fixed (byte* buffer2 = decompBuffer)
{
_codec.DecompressBuffer((nint)buffer, (int)packFile.CompressedFileSize, (nint)buffer2, (int)MAX_DECOMPRESSED_CHUNK_SIZE, (int)packFile.DecompressedFileSize);
_codec.DecompressBuffer((nint)buffer, (int)packFile.CompressedFileSize, (nint)buffer2, (int)MAX_DECOMPRESSED_MULTI_CHUNK_SIZE, (int)packFile.DecompressedFileSize);
}
}

Expand Down Expand Up @@ -320,31 +331,6 @@ private void ExtractFileFromSharedChunk(FF16PackFile packFile, string outputPath
_cachedChunks.Add(chunk);
}

private static void DecryptHeaderPart(Span<byte> data)
{
Span<byte> cur = data;
while (cur.Length >= 8)
{
MemoryMarshal.Cast<byte, ulong>(cur)[0] ^= XOR_KEY;
cur = cur[8..];
}

if (cur.Length >= 4)
{
MemoryMarshal.Cast<byte, uint>(cur)[0] ^= (uint)(XOR_KEY & 0xFFFFFFFF);
cur = cur[4..];
}

if (cur.Length >= 2)
{
MemoryMarshal.Cast<byte, ushort>(cur)[0] ^= (ushort)(XOR_KEY & 0xFFFF);
cur = cur[2..];
}

if (cur.Length >= 1)
cur[0] ^= (byte)(XOR_KEY & 0xFF);
}

public static void ThrowHashException(string path)
{
throw new InvalidDataException($"Hash for file '{path}' did not match.");
Expand Down
12 changes: 11 additions & 1 deletion FF16PackLib/FF16PackDStorageChunk.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,17 @@ public void FromStream(BinaryStream bs)
NumFilesInChunk = bs.ReadUInt16();
}

public static int GetSize()
public void Write(BinaryStream bs)
{
bs.WriteUInt64(DataOffset);
bs.WriteUInt32(CompressedChunkSize);
bs.WriteUInt32(DecompressedSize);
bs.WriteUInt32(0);
bs.WriteUInt16(ChunkIndex);
bs.WriteUInt16(NumFilesInChunk);
}

public static uint GetSize()
{
return 0x18;
}
Expand Down
21 changes: 20 additions & 1 deletion FF16PackLib/FF16PackFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,31 @@ public void FromStream(BinaryStream bs)
DataOffset = bs.ReadUInt64();
ChunkDefOffset = bs.ReadUInt64();
FileNameOffset = bs.ReadUInt64();

FileNameHash = bs.ReadUInt32();
CRC32Checksum = bs.ReadUInt32();
bs.ReadUInt32(); // Empty
ChunkHeaderSize = bs.ReadUInt32();
}

public void Write(BinaryStream bs)
{
bs.WriteUInt32(CompressedFileSize);
bs.WriteBoolean(IsCompressed);
bs.WriteByte((byte)ChunkedCompressionFlags);
bs.WriteInt16(0);
bs.WriteUInt64(DecompressedFileSize);
bs.WriteUInt64(DataOffset);
bs.WriteUInt64(ChunkDefOffset);
bs.WriteUInt64(FileNameOffset);
bs.WriteUInt32(FileNameHash);
bs.WriteUInt32(CRC32Checksum);
bs.WriteUInt32(0);
bs.WriteUInt32(ChunkHeaderSize);
}

public static uint GetSize()
{
return 0x38;
}
}

Expand Down
1 change: 1 addition & 0 deletions FF16PackLib/FF16PackLib.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

<ItemGroup>
<PackageReference Include="Syroot.BinaryData" Version="5.2.2" />
<PackageReference Include="Syroot.BinaryData.Memory" Version="5.2.2" />
<PackageReference Include="System.IO.Hashing" Version="8.0.0" />
<PackageReference Include="Vortice.DirectStorage" Version="3.5.0" />
</ItemGroup>
Expand Down
Loading

0 comments on commit 75d9c26

Please sign in to comment.