MyouProject/myou-engine
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add zstd support for blend files.

Browse source
This commit is contained in:
Alberto Torres 2024-08-22 01:47:49 +02:00
parent 6b029cb83c
commit 808898eaef
3 changed files with 24 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitmodules vendored
View file

@ -4,3 +4,6 @@
[submodule "libs/pixie"] [submodule "libs/pixie"]
path = libs/pixie path = libs/pixie
url = https://git.myou.dev/MyouProject/pixie url = https://git.myou.dev/MyouProject/pixie
[submodule "libs/nim_zstd"]
path = libs/nim_zstd
url = https://github.com/DiThi/nim_zstd

1
libs/nim_zstd Submodule

@ -0,0 +1 @@
Subproject commit 75ee00aa3da80903dc509f1ff1605304cc77c173

38
src/loaders/blend_format.nim
View file

@ -32,10 +32,11 @@
{.experimental: "dotOperators".} {.experimental: "dotOperators".}
import strutils, tables, algorithm, sequtils, strformat, bitops import std/[strutils, tables, algorithm, sequtils, strformat, bitops]
import memfiles import std/memfiles
# import sugar # import sugar
import hashes import std/hashes
import zstd/decompress
# TODO!! ADD BOUND CHECKS TO ALL FNODES AND DNA1 PARSING! # TODO!! ADD BOUND CHECKS TO ALL FNODES AND DNA1 PARSING!
# TODO: implement big endian, test 32 bit # TODO: implement big endian, test 32 bit
@ -56,6 +57,8 @@ type
old_mem_ptr: pointer old_mem_ptr: pointer
# end pointer for bounds checking # end pointer for bounds checking
endp: pointer endp: pointer
# byte seq in case we used zstd
byte_seq: seq[byte]
BlendFile* = ref BlendFileVal BlendFile* = ref BlendFileVal
@ -142,13 +145,16 @@ template `-`(p, q: pointer): int = cast[int](cast[int](p) -% cast[int](q))
template `+=`(p: pointer, i: Natural) = p = cast[pointer](cast[int](p) +% cast[int](i)) template `+=`(p: pointer, i: Natural) = p = cast[pointer](cast[int](p) +% cast[int](i))
# this is for getting a string that may not be null terminated # this is for getting a string that may not be null terminated
# otherwise you can just do $cast[cstring](x.addr) # otherwise you could just do $cast[cstring](x.addr)
proc getString(data: pointer, max_size: Natural): string = proc getString(data: pointer, max_size: Natural): string =
let u8 = cast[ptr UncheckedArray[char]](data) let u8 = cast[ptr UncheckedArray[char]](data)
for i in 0 ..< max_size: var i = 0
while i < max_size:
if u8[i] == '\0': if u8[i] == '\0':
break break
result &= u8[i] inc(i)
result.setLen i
copyMem(result.cstring, data, i)
proc build_dna_table(dna_table: ptr FBlock64): (TypeAttrsRef, StructToTypeMap, seq[cstring], TypeLengths) = proc build_dna_table(dna_table: ptr FBlock64): (TypeAttrsRef, StructToTypeMap, seq[cstring], TypeLengths) =
var type_attrs = new TypeAttrsRef var type_attrs = new TypeAttrsRef
@ -230,25 +236,21 @@ proc build_dna_table(dna_table: ptr FBlock64): (TypeAttrsRef, StructToTypeMap, s
) )
aoffset += asize aoffset += asize
offset += 4 offset += 4
# if true:
# # if lengths[type_id] != aoffset:
# echo &"{types[type_id]}: {lengths[type_id]}, {aoffset}"
# var offsets = toSeq(type_attrs[type_id].pairs)
# for _,(k,v) in offsets.sortedByIt(it[1].offset):
# echo &"{types[type_id]}.{k}:\t\t{v.offset}"
assert lengths[type_id] == aoffset assert lengths[type_id] == aoffset
# type_attrs[type_id]["(size)"] = AttrOffset(offset: lengths[type_id]) # type_attrs[type_id]["(size)"] = AttrOffset(offset: lengths[type_id])
return (type_attrs, struct_to_type, types, lengths) return (type_attrs, struct_to_type, types, lengths)
proc openBlendFile*(path: string, data: pointer, len: int): BlendFile = proc openBlendFile*(path: string, data: pointer, len: int): BlendFile =
result = new BlendFile result = new BlendFile
# result.mem_file = memfiles.open(path, mode=fmRead)
result.file_path = path result.file_path = path
# let mem = result.mem_file.mem let (mem, file_length) = if cast[ptr uint32](data)[] == 0xFD2FB528'u32:
# result.mem = mem # Zstandard compressed file
# let file_length = result.mem_file.size # TODO: store offsets of each zstd frame so we can semi-random access
let mem = data # big files later
let file_length = len result.byte_seq = decompress cast[ptr UncheckedArray[byte]](data).toOpenArray(0, len-1)
(result.byte_seq[0].addr.pointer, result.byte_seq.len)
else:
(data, len)
assert file_length > 32, "Invalid file size" assert file_length > 32, "Invalid file size"
result.endp = mem + file_length result.endp = mem + file_length
let u8 = cast[ptr UncheckedArray[uint8]](mem) let u8 = cast[ptr UncheckedArray[uint8]](mem)