Skip to content
This repository has been archived by the owner on May 11, 2024. It is now read-only.

Commit

Permalink
- [Core] Added Blb file parsing [GI]
Browse files Browse the repository at this point in the history
  • Loading branch information
@Razmoth
Razmoth committed Sep 19, 2023
1 parent a7e7cb7 commit dbec3e2
Show file tree
Hide file tree
Showing 7 changed files with 282 additions and 65 deletions.
72 changes: 52 additions & 20 deletions AssetStudio/AssetsManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,19 +432,22 @@ private void LoadBlockFile(FileReader reader)
try
{
using var stream = new OffsetStream(reader.BaseStream, 0);
if (AssetsHelper.TryGet(reader.FullPath, out var offsets))
foreach (var offset in stream.GetOffsets(reader.FullPath))
{
foreach (var offset in offsets)
var name = offset.ToString("X8");
Logger.Info($"Loading Block {name}");

var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), name);
var subReader = new FileReader(dummyPath, stream, true);
switch (subReader.FileType)
{
LoadBlockSubFile(reader.FullPath, stream, offset);
case FileType.BundleFile:
LoadBundleFile(subReader, reader.FullPath, offset, false);
break;
case FileType.BlbFile:
LoadBlbFile(subReader, reader.FullPath, offset, false);
break;
}
}
else
{
do
{
LoadBlockSubFile(reader.FullPath, stream, stream.AbsolutePosition);
} while (stream.Remaining > 0);
}
}
catch (Exception e)
Expand All @@ -456,16 +459,6 @@ private void LoadBlockFile(FileReader reader)
reader.Dispose();
}
}
private void LoadBlockSubFile(string path, OffsetStream stream, long offset)
{
var name = offset.ToString("X8");
Logger.Info($"Loading Block {name}");

stream.Offset = offset;
var dummyPath = Path.Combine(Path.GetDirectoryName(path), name);
var subReader = new FileReader(dummyPath, stream, true);
LoadBundleFile(subReader, path, offset, false);
}
private void LoadBlkFile(FileReader reader)
{
Logger.Info("Loading " + reader.FullPath);
Expand Down Expand Up @@ -546,6 +539,45 @@ private void LoadMhy0File(FileReader reader, string originalPath = null, long or
reader.Dispose();
}
}

private void LoadBlbFile(FileReader reader, string originalPath = null, long originalOffset = 0, bool log = true)
{
if (log)
{
Logger.Info("Loading " + reader.FullPath);
}
try
{
var blbFile = new BlbFile(reader, reader.FullPath);
foreach (var file in blbFile.fileList)
{
var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), file.fileName);
var cabReader = new FileReader(dummyPath, file.stream);
if (cabReader.FileType == FileType.AssetsFile)
{
LoadAssetsFromMemory(cabReader, originalPath ?? reader.FullPath, blbFile.m_Header.unityRevision, originalOffset);
}
else
{
Logger.Verbose("Caching resource stream");
resourceFileReaders[file.fileName] = cabReader; //TODO
}
}
}
catch (Exception e)
{
var str = $"Error while reading Blb file {reader.FullPath}";
if (originalPath != null)
{
str += $" from {Path.GetFileName(originalPath)}";
}
Logger.Error(str, e);
}
finally
{
reader.Dispose();
}
}

public void CheckStrippedVersion(SerializedFile assetsFile)
{
Expand Down
172 changes: 172 additions & 0 deletions AssetStudio/BlbFile.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
using System;
using System.IO;
using System.Linq;
using K4os.Compression.LZ4;

namespace AssetStudio
{
public class BlbFile
{
private const uint DefaultUncompressedSize = 0x20000;

private BundleFile.StorageBlock[] m_BlocksInfo;
private BundleFile.Node[] m_DirectoryInfo;

public BundleFile.Header m_Header;
public StreamFile[] fileList;
public long Offset;


public BlbFile(FileReader reader, string path)
{
Offset = reader.Position;
reader.Endian = EndianType.LittleEndian;

var signature = reader.ReadStringToNull(4);
Logger.Verbose($"Parsed signature {signature}");
if (signature != "Blb\x02")
throw new Exception("not a Blb file");

var size = reader.ReadUInt32();
m_Header = new BundleFile.Header
{
version = 6,
unityVersion = "5.x.x",
unityRevision = "2017.4.30f1",
flags = (ArchiveFlags)0x43
};
m_Header.compressedBlocksInfoSize = size;
m_Header.uncompressedBlocksInfoSize = size;

Logger.Verbose($"Header: {m_Header}");

var header = reader.ReadBytes((int)m_Header.compressedBlocksInfoSize);
ReadBlocksInfoAndDirectory(header);
using var blocksStream = CreateBlocksStream(path);
ReadBlocks(reader, blocksStream);
ReadFiles(blocksStream, path);
}

private void ReadBlocksInfoAndDirectory(byte[] header)
{
using var stream = new MemoryStream(header);
using var reader = new EndianBinaryReader(stream, EndianType.LittleEndian);

m_Header.size = reader.ReadUInt32();
var lastUncompressedSize = reader.ReadUInt32();

reader.Position += 4;
var offset = reader.ReadInt64();
var compressionType = (CompressionType)reader.ReadByte();
var serializedFileVersion = (SerializedFileFormatVersion)reader.ReadByte();
reader.AlignStream();

var blocksInfoCount = reader.ReadInt32();
var nodesCount = reader.ReadInt32();

var blocksInfoOffset = reader.Position + reader.ReadInt64();
var nodesInfoOffset = reader.Position + reader.ReadInt64();
var bundleInfoOffset = reader.Position + reader.ReadInt64();

reader.Position = blocksInfoOffset;
m_BlocksInfo = new BundleFile.StorageBlock[blocksInfoCount];
Logger.Verbose($"Blocks count: {blocksInfoCount}");
for (int i = 0; i < blocksInfoCount; i++)
{
m_BlocksInfo[i] = new BundleFile.StorageBlock
{
compressedSize = reader.ReadUInt32(),
uncompressedSize = i == blocksInfoCount - 1 ? lastUncompressedSize : DefaultUncompressedSize,
flags = (StorageBlockFlags)0x43
};

Logger.Verbose($"Block {i} Info: {m_BlocksInfo[i]}");
}

reader.Position = nodesInfoOffset;
m_DirectoryInfo = new BundleFile.Node[nodesCount];
Logger.Verbose($"Directory count: {nodesCount}");
for (int i = 0; i < nodesCount; i++)
{
m_DirectoryInfo[i] = new BundleFile.Node
{
offset = reader.ReadInt32(),
size = reader.ReadInt32()
};

var pathOffset = reader.Position + reader.ReadInt64();

var pos = reader.Position;
reader.Position = pathOffset;
m_DirectoryInfo[i].path = reader.ReadStringToNull();
reader.Position = pos;

Logger.Verbose($"Directory {i} Info: {m_DirectoryInfo[i]}");
}
}

private Stream CreateBlocksStream(string path)
{
Stream blocksStream;
var uncompressedSizeSum = (int)m_BlocksInfo.Sum(x => x.uncompressedSize);
Logger.Verbose($"Total size of decompressed blocks: 0x{uncompressedSizeSum:X8}");
if (uncompressedSizeSum >= int.MaxValue)
blocksStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
else
blocksStream = new MemoryStream(uncompressedSizeSum);
return blocksStream;
}

private void ReadBlocks(EndianBinaryReader reader, Stream blocksStream)
{
foreach (var blockInfo in m_BlocksInfo)
{
var compressedSize = (int)blockInfo.compressedSize;
var uncompressedSize = (int)blockInfo.uncompressedSize;

var compressedBytes = BigArrayPool<byte>.Shared.Rent(compressedSize);
var uncompressedBytes = BigArrayPool<byte>.Shared.Rent(uncompressedSize);
reader.Read(compressedBytes, 0, compressedSize);

var compressedBytesSpan = compressedBytes.AsSpan(0, compressedSize);
var uncompressedBytesSpan = uncompressedBytes.AsSpan(0, uncompressedSize);

var numWrite = LZ4Codec.Decode(compressedBytesSpan, uncompressedBytesSpan);
if (numWrite != uncompressedSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
}

blocksStream.Write(uncompressedBytes, 0, uncompressedSize);
BigArrayPool<byte>.Shared.Return(compressedBytes);
BigArrayPool<byte>.Shared.Return(uncompressedBytes);
}
}

private void ReadFiles(Stream blocksStream, string path)
{
Logger.Verbose($"Writing files from blocks stream...");

fileList = new StreamFile[m_DirectoryInfo.Length];
for (int i = 0; i < m_DirectoryInfo.Length; i++)
{
var node = m_DirectoryInfo[i];
var file = new StreamFile();
fileList[i] = file;
file.path = node.path;
file.fileName = Path.GetFileName(node.path);
if (node.size >= int.MaxValue)
{
var extractPath = path + "_unpacked" + Path.DirectorySeparatorChar;
Directory.CreateDirectory(extractPath);
file.stream = new FileStream(extractPath + file.fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
}
else
file.stream = new MemoryStream((int)node.size);
blocksStream.Position = node.offset;
blocksStream.CopyTo(file.stream, node.size);
file.stream.Position = 0;
}
}
}
}
3 changes: 3 additions & 0 deletions AssetStudio/BundleFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,9 @@ private Header ReadBundleHeader(FileReader reader)
header.signature = "UnityFS";
goto case "UnityFS";
}
header.version = reader.ReadUInt32();
header.unityVersion = reader.ReadStringToNull();
header.unityRevision = reader.ReadStringToNull();
break;

}
Expand Down
44 changes: 0 additions & 44 deletions AssetStudio/Crypto/BlkUtils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ public static class BlkUtils
private const int DataOffset = 0x2A;
private const int KeySize = 0x1000;
private const int SeedBlockSize = 0x800;
private const int BufferSize = 0x10000;

public static XORStream Decrypt(FileReader reader, Blk blk)
{
Expand Down Expand Up @@ -64,48 +63,5 @@ public static XORStream Decrypt(FileReader reader, Blk blk)

return new XORStream(reader.BaseStream, DataOffset, xorpad);
}

public static IEnumerable<long> GetOffsets(this XORStream stream, string path)
{
if (AssetsHelper.TryGet(path, out var offsets))
{
foreach(var offset in offsets)
{
stream.Offset = offset;
yield return offset;
}
}
else
{
using var reader = new FileReader(path, stream, true);
var signature = reader.FileType switch
{
FileType.BundleFile => "UnityFS\x00",
FileType.Mhy0File => "mhy0",
_ => throw new InvalidOperationException()
};

Logger.Verbose($"Prased signature: {signature}");

var signatureBytes = Encoding.UTF8.GetBytes(signature);
var buffer = BigArrayPool<byte>.Shared.Rent(BufferSize);
while (stream.Remaining > 0)
{
var index = 0;
var absOffset = stream.AbsolutePosition;
var read = stream.Read(buffer);
while (index < read)
{
index = buffer.AsSpan(0, read).Search(signatureBytes, index);
if (index == -1) break;
var offset = absOffset + index;
stream.Offset = offset;
yield return offset;
index++;
}
}
BigArrayPool<byte>.Shared.Return(buffer);
}
}
}
}
8 changes: 7 additions & 1 deletion AssetStudio/FileReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class FileReader : EndianBinaryReader
private static readonly byte[] zipMagic = { 0x50, 0x4B, 0x03, 0x04 };
private static readonly byte[] zipSpannedMagic = { 0x50, 0x4B, 0x07, 0x08 };
private static readonly byte[] mhy0Magic = { 0x6D, 0x68, 0x79, 0x30 };
private static readonly byte[] blbMagic = { 0x42, 0x6C, 0x62, 0x02 };
private static readonly byte[] narakaMagic = { 0x15, 0x1E, 0x1C, 0x0D, 0x0D, 0x23, 0x21 };
private static readonly byte[] gunfireMagic = { 0x7C, 0x6D, 0x79, 0x72, 0x27, 0x7A, 0x73, 0x78, 0x3F };

Expand Down Expand Up @@ -84,6 +85,11 @@ private FileType CheckFileType()
return FileType.Mhy0File;
}
Logger.Verbose($"Parsed signature does not match with expected signature {Convert.ToHexString(mhy0Magic)}");
if (blbMagic.SequenceEqual(magic))
{
return FileType.BlbFile;
}
Logger.Verbose($"Parsed signature does not match with expected signature {Convert.ToHexString(mhy0Magic)}");
magic = ReadBytes(7);
Position = 0;
Logger.Verbose($"Parsed signature is {Convert.ToHexString(magic)}");
Expand Down Expand Up @@ -195,7 +201,7 @@ public static FileReader PreProcessing(this FileReader reader, Game game)
break;
}
}
if (reader.FileType == FileType.BundleFile && game.Type.IsBlockFile())
if (reader.FileType == FileType.BundleFile && game.Type.IsBlockFile() || reader.FileType == FileType.BlbFile)
{
Logger.Verbose("File might have multiple bundles !!");
try
Expand Down
1 change: 1 addition & 0 deletions AssetStudio/FileType.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public enum FileType
ZipFile,
BlkFile,
Mhy0File,
BlbFile,
BlockFile
}
}
Loading

0 comments on commit dbec3e2

Please sign in to comment.