Add some documentation and fix fixdocs.nim.

doc/fixdocs.nim

@@ -55,16 +55,18 @@ for f in os.commandLineParams():
     var types: seq[string]
     var lines = readFile(f).split("\n")
     var menu_line = 0
+    var last_menu_line = 0
     var types_line = ""
     for i,line in lines:
         # if "reference reference-toplevel" in line and line.endswith ">Types</a>":
         if line.startswith "    title=\"TYPES = ":
             types_line = line
+            menu_line = i - 1
         elif types_line != "":
             types_line &= line
         if "</a>" in types_line:
             types = types_line.split({'"','='})[3].split({'|',' '}).filterIt it != ""
-            menu_line = i - 1
+            last_menu_line = i
             break
     if types.len == 0:
         continue
@@ -76,15 +78,18 @@ for f in os.commandLineParams():
 
     assert menu_line != 0
 
-    lines.delete menu_line
+    # types_id is usually "TYPES" but for some reason it's "TYPES_2" sometimes,
-    lines.delete menu_line
+    # so we'll extract the id from the href before we delete it
+    let types_id = lines[menu_line].rsplit("#",1)[1].split("\"")[0]
+    for i in menu_line .. last_menu_line:
+        lines.delete menu_line
 
     for tname in types:
         var parts = types_menu[tname].split('"')
         parts[5] = parts[5].strip_privates
         lines.insert(parts.join("\""), menu_line)
 
-    var body_line = lines.find "<div id=\"TYPES\">"
+    var body_line = lines.find &"<div id=\"{types_id}\">"
     while lines[body_line] != "</div>":
         lines.delete body_line
     lines.delete body_line

doc/modules.rst

@@ -28,12 +28,23 @@ Objects
 Graphics modules
 ----------------
 
+Anything that touches the graphics API (MeshData should be moved here)
+
 * `Render <graphics/render.html>`_
 * `Material <graphics/material.html>`_
 * `Texture <graphics/texture.html>`_
 * `Framebuffer <graphics/framebuffer.html>`_
 * `Ubo <graphics/ubo.html>`_
 
+GPU formats
+-----------
+
+Modules that handle mesh and texture formats and optimize them for the GPU while
+being completely agnostic to the graphics API.
+
+* `Texture decode <gpu_formats/texture_decode.html>`_
+* `Texture optimize <gpu_formats/texture_decode.html>`_
+
 Effect system
 -------------
 
@@ -60,6 +71,7 @@ Libraries
 ---------
 
 * `ArrRef <_._/libs/arr_ref/arr_ref.html>`_
+* `SliceMem <_._/libs/arr_ref/slice_mem.html>`_
 * `Loadable <_._/libs/loadable/loadable.html>`_
 * `DdxKtx <_._/libs/ddx_ktx/ddx_ktx.html>`_
 * `Float16 <_._/libs/float16/float16.html>`_

libs/arr_ref/slice_mem.nim

@@ -1,13 +1,15 @@
 
-## SliceMem represents a slice of memory, but it includes a reference to the
+## SliceMem is a general purpose array that can be casted to any type, and does
-## original container (of any type) or even a custom destructor, so memory is
+## not create copies when assigned multiple times, like a reference type, unless
-## properly freed when it's no longer in use.
+## you use `copy`. It can be created by itself, from another container such as
+## a seq (without copying when possible), or from a pointer and a custom
+## destructor.
 ## 
 ## It's called "SliceMem" because "MemSlice" is taken by std/memfiles.
 
 import std/strformat
 import std/hashes
-import std/macros
+import std/macros # for noMove()
 
 type
     SliceMem*[T] = object
@@ -31,6 +33,10 @@ template toPointer*(s: SliceMem): pointer =
 
 proc newSliceMem*[T, U](container: sink U; p: pointer, byte_len: int): SliceMem[T] =
     ## Create a SliceMem from a container, a pointer, and a length in bytes.
+    runnableExamples:
+        let x = @[1,2,3,4,5]
+        let s = newSliceMem(x, x[0].addr, x.len * sizeof(x[0]))
+    
     result = SliceMem[T](
         data: cast[ptr UncheckedArray[T]](p),
         byte_len: byte_len,
@@ -45,6 +51,11 @@ proc newSliceMem*[T, U](container: sink U; p: pointer, byte_len: int): SliceMem[
 proc newSliceMem*[T](p: ptr T, byte_len: int, destructor: proc() {.closure, raises: [].}): SliceMem[T] =
     ## Create a SliceMem from a pointer to a type, a length in bytes, and a
     ## destructor closure.
+    runnableExamples:
+        let x = createShared(int, 5)
+        proc destroy() = deallocShared(x)
+        let s = newSliceMem(x, 5, destroy)
+    
     result = SliceMem[T](
         data: cast[ptr UncheckedArray[T]](p),
         byte_len: byte_len,
@@ -52,6 +63,8 @@ proc newSliceMem*[T](p: ptr T, byte_len: int, destructor: proc() {.closure, rais
     )
 
 proc newSliceMem*(p: pointer, byte_len: int, destructor: proc() {.closure, raises: [].}): SliceMem[byte] {.inline.} =
+    ## Create a SliceMem from a pointer without type, a length in bytes, and a
+    ## destructor closure. Same as newSliceMem[T](...) but assumes type is byte.
     newSliceMem(cast[ptr byte](p), byte_len, destructor)
 
 template newSliceMem*(container: not pointer, p, byte_len: untyped): untyped =
@@ -83,6 +96,7 @@ proc newSliceMem*[T](size: int): SliceMem[T] =
     )
 
 macro noMove(e: untyped): untyped =
+    # remove the "move" from an expression
     if e.kind == nnkCommand and e[0].repr == "move": e[1]
     else: e
 
@@ -90,20 +104,43 @@ template toSliceMem*(container, slice: untyped): untyped =
     ## Create a SliceMem from a container and a slice that indicates the range.
     ## The container needs to have `[]` and `items()` (like `seq`). If it doesn't
     ## you need to pass pointers directly to `newSliceMem` instead.
-    if slice.len != 0:
+    runnableExamples:
-        # TODO: check that with boundChecks:off it doesn't try to read the value
+        var x = @[1,2,3,4,5]
-        let first = noMove(container)[slice.a].addr
+        let a = x.toSliceMem(1..3)
-        let last = noMove(container)[slice.b].addr
+        let b = toSliceMem(move x, 1..3) # This also works, and ensures that
-        newSliceMem[typeof(noMove(container).items)](container, first, last)
+                                         # the contents of x are not copied
-    else:
+    {.line.}:
-        SliceMem[typeof(container.items)]()
+        if slice.len != 0:
+            # TODO: check that with boundChecks:off it doesn't try to read the value
+            let first = noMove(container)[slice.a].addr
+            let last = noMove(container)[slice.b].addr
+            newSliceMem[typeof(noMove(container).items)](container, first, last)
+        else:
+            SliceMem[typeof(container.items)]()
 
 template toSliceMem*(container): untyped =
     ## Create a SliceMem from a container, with all its contents. The container
     ## needs to have `[]` and `items()` (like `seq`). If it doesn't you need to
     ## pass pointers directly to `newSliceMem` instead.
-    let s = noMove(container).low .. noMove(container).high
+    runnableExamples:
-    toSliceMem(container, s)
+        var x = @[1,2,3,4,5]
+        let a = x.toSliceMem
+        let b = toSliceMem(move x) # This also works, and ensures that
+                                   # the contents of x are not copied
+    {.line.}:
+        let s = noMove(container).low .. noMove(container).high
+        when not compiles(container[s.a].addr):
+            # expression container[x] has no address because it's a literal,
+            # let's make it real
+            let c = container
+            toSliceMem(c, s)
+        else:
+            toSliceMem(container, s)
+
+proc copy*[T](s: SliceMem[T]): SliceMem[T] =
+    ## Creates a copy of a SliceMem that doesn't reference the original one.
+    result = newSliceMem[byte](s.byte_len).to(T)
+    copyMem(result.toPointer, s.toPointer, s.byte_len)
 
 template len*[T](s: SliceMem[T]): Natural =
     s.byte_len div sizeof(T)
@@ -175,15 +212,24 @@ template toOpenArrayByte*(s: SliceMem): untyped =
     cast[ptr UncheckedArray[byte]](s.data).toOpenArray(0, s.byte_len - 1)
 
 template to*[T](s: SliceMem, typ: typedesc[T]): SliceMem[T] =
+    ## Cast the SliceMem from one type to another
     cast[SliceMem[T]](s)
 
-template to*[T](s: seq[SliceMem], typ: typedesc[T]): seq[SliceMem[T]] =
+template to*[T](s: openArray[SliceMem], typ: typedesc[T]): seq[SliceMem[T]] =
+    ## Cast a seq of SliceMems from one type to another
     var s2 = newSeqOfCap[SliceMem[T]](s.len)
     for slice in s:
         s2.add cast[SliceMem[T]](slice)
     s2
 
-template concatImpl(slices: untyped) =
+proc concat*[T](slices: varargs[SliceMem[T]]): SliceMem[T] =
+    ## Concatenates a list of SliceMems into a single one. Contents are copied.
+    runnableExamples:
+        let a = @[1,2,3,4,5].toSliceMem
+        let b = @[6,7,8,9,0].toSliceMem
+        var x = concat(a,b)
+        x = concat(a,b,a,a,b)  # You can pass any amount of slices as arguments
+        x = concat(@[b,b,a,a]) # or you can pass a seq
     var total = 0
     for s in slices:
         total += s.len
@@ -193,11 +239,71 @@ template concatImpl(slices: untyped) =
         copyMem(result[offset].addr, s.data, s.len * sizeof(T))
         offset += s.len
 
-proc concat*[T](slices: varargs[SliceMem[T]]): SliceMem[T] =
-    concatImpl(slices)
-
-proc concat*[T](slices: seq[SliceMem[T]]): SliceMem[T] =
-    concatImpl(slices)
-
 template `&`*[T,U](a: SliceMem[T], b: SliceMem[U]): SliceMem[T] =
+    ## Concatenates two SliceMems (of any type) into a new one with the type of
+    ## the first one.
     concat(a, b.to(T))
+
+proc serializeToSeq*[T](slices: openArray[SliceMem[T]], align = sizeof(int)): seq[SliceMem[byte]] =
+    ## Converts a list of SliceMems into a format that can be saved to file and
+    ## deserialized later. See `serialize` and `deserialize`. This proc returns a
+    ## seq that is ready to concatenate or to write to a file.
+    let mask = align - 1
+    assert((align and mask) == 0, "Align must be a power of two")
+    var offset = 0
+    let align_count = newSliceMem[int32](2)
+    align_count[0] = align.int32
+    align_count[1] = slices.len.int32
+    result.add align_count.to(byte)
+    offset += align_count.byte_len
+    let lengths = newSliceMem[int64](slices.len)
+    result.add lengths.to(byte)
+    offset += lengths.byte_len
+    let pad_bytes = newSliceMem[byte](align)
+    for i,s in slices:
+        let pad = ((align-(offset and mask)) and mask)
+        if pad != 0:
+            result.add pad_bytes[0 ..< pad]
+            offset += pad
+        lengths[i] = s.byte_len
+        result.add s.to(byte)
+        offset += s.byte_len
+
+proc serialize*[T](slices: openArray[SliceMem[T]], align = sizeof(int)): SliceMem[byte] =
+    ## Converts a list of SliceMems into a single segment of memory that can be
+    ## later deserialized into the separate SliceMems again. Use `deserialize`
+    ## for the inverse process. Use `serializeToSeq` to obtain the separate
+    ## formatted slices before they're concatenated and avoid copies.
+    runnableExamples:
+        let a = @[1,2,3,4,5].toSliceMem
+        let b = @[6,7,8,9,0].toSliceMem
+        let serialized = serialize(@[a,b])
+        let slices = serialized.to(int).deserialize
+        doAssert $slices == "@[SliceMem([1, 2, 3, 4, 5]), SliceMem([6, 7, 8, 9, 0])]"
+    concat(serializeToSeq(slices, align))
+
+proc deserialize*[T](data: SliceMem[T]): seq[SliceMem[T]] =
+    ## Reverts the process done by `serialize` except the conversion to byte.
+    ## You can convert to the appropriate type before or after deserialization
+    ## with `to` (see example in `serialize`)
+    let bytes = data.to(byte)
+    let i32 = data.to(int32)
+    let align = i32[0]
+    let count = i32[1]
+    let mask = align - 1
+    assert((align and mask) == 0, "Wrong format, align is invalid")
+    let lengths = bytes[8 ..< 8 + count*sizeof(int64)].to(int64)
+    var offset = 8 + lengths.byte_len
+    for i in 0 ..< count:
+        offset += ((align-(offset and mask)) and mask)
+        let len = lengths[i].int
+        result.add bytes[offset ..< offset+len].to(T)
+        offset += len
+
+template toString*(s: SliceMem): string =
+    ## Copies the contents of the SliceMem to a new string of the same length.
+    var str = newString(s.byte_len)
+    if s.byte_len != 0:
+        copyMem(str[0].addr, s.toPointer, s.byte_len)
+    str
+

libs/dds_ktx/dds_ktx.nim

@@ -17,6 +17,10 @@ type KtxPart* = object
     len*: int
     row_len*: int
 
+type KtxInfoParts* = object
+    info*: KtxInfo
+    parts*: seq[KtxPart]
+
 {.compile:("impl.c","-I.").}
 
 type ddsktx_format = enum
@@ -252,7 +256,7 @@ proc ParseDdsKtx*(p: pointer, len: int): seq[KtxPart] =
 
 func get_ASTC_internal_format*(blk_size: (SomeInteger,SomeInteger), is_sRGB: bool): int32 =
     result = if is_sRGB:
-        case blk_size[0]*100 or blk_size[1]:
+        case (blk_size[0].int*100 or blk_size[1].int):
             of 04_04: GL_COMPRESSED_SRGB8_ALPHA8_ASTC_4x4_KHR
             of 05_04: GL_COMPRESSED_SRGB8_ALPHA8_ASTC_5x4_KHR
             of 05_05: GL_COMPRESSED_SRGB8_ALPHA8_ASTC_5x5_KHR
@@ -269,7 +273,7 @@ func get_ASTC_internal_format*(blk_size: (SomeInteger,SomeInteger), is_sRGB: boo
             of 12_12: GL_COMPRESSED_SRGB8_ALPHA8_ASTC_12x12_KHR
             else: 0'i32
     else:
-        case blk_size[0]*100 or blk_size[1]:
+        case (blk_size[0].int*100 or blk_size[1].int):
             of 04_04: GL_COMPRESSED_RGBA_ASTC_4x4_KHR
             of 05_04: GL_COMPRESSED_RGBA_ASTC_5x4_KHR
             of 05_05: GL_COMPRESSED_RGBA_ASTC_5x5_KHR

libs/loadable/loadable.nim

@@ -231,11 +231,14 @@ func escapeUTF8*(s: string): string =
         else:
             result &= '%' & c.byte.toHex
 
+# TODO: automatically disable threads when not in main thread
+
 proc loadUri*(
         uri: string, 
         onload_func: proc(ok: bool, err: string, data: SliceMem[byte]) = nil,
         range = (-1,-1),
         auto_start = true,
+        use_threads = true,
     ): Fetch {.discardable.} =
 
     echo "fetching ", uri
@@ -243,7 +246,7 @@ proc loadUri*(
     var start_func: proc(self: LoadableResource)
     var self: Fetch
     var uri = uri
-    var use_threads = false
+    var use_threads = use_threads
     when not defined(onlyLocalFiles):
         when defined(emscripten):
             const is_remote = true
@@ -253,6 +256,7 @@ proc loadUri*(
         if is_remote:
             uri = uri.escapeUTF8
             when defined(emscripten):
+                use_threads = false # API is already threaded
                 start_func = proc(self: LoadableResource) =
                     var attr: emscripten_fetch_attr_t
                     emscripten_fetch_attr_init(attr)
@@ -273,7 +277,6 @@ proc loadUri*(
                         self.onload(false, &"Error fetching {fetch.url}: {err_msg}")
                     discard emscripten_fetch(attr, uri.cstring)
             else:
-                use_threads = true
                 var client = newHttpClient()
                 var response: string
                 start_func = proc(self: LoadableResource) =
@@ -290,6 +293,8 @@ proc loadUri*(
                         ))
 
     if not is_remote:
+        use_threads = false # TODO: detect networked file system?
+                            # TODO: also test if there's better perf in local
         start_func = proc(self: LoadableResource) =
             try:
                 var memfile = memfiles.open(uri, mode=fmRead)

nim.cfg

@@ -0,0 +1,2 @@
+path:"./libs/packages"
+threads:on

src/gpu_formats/texture_optimize.nim

@@ -1,11 +1,59 @@
+# The contents of this file are subject to the Common Public Attribution License
+# Version 1.0 (the “License”); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+# https://myou.dev/licenses/LICENSE-CPAL. The License is based on the Mozilla
+# Public License Version 1.1 but Sections 14 and 15 have been added to cover use
+# of software over a computer network and provide for limited attribution for
+# the Original Developer. In addition, Exhibit A has been modified to be
+# consistent with Exhibit B.
+#
+# Software distributed under the License is distributed on an “AS IS” basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
+# the specific language governing rights and limitations under the License.
+#
+# The Original Code is Myou Engine.
+#
+# the Original Developer is the Initial Developer.
+#
+# The Initial Developer of the Original Code is the Myou Engine developers.
+# All portions of the code written by the Myou Engine developers are Copyright
+# (c) 2024. All Rights Reserved.
+#
+# Alternatively, the contents of this file may be used under the terms of the
+# GNU Affero General Public License version 3 (the [AGPL-3] License), in which
+# case the provisions of [AGPL-3] License are applicable instead of those above.
+#
+# If you wish to allow use of your version of this file only under the terms of
+# the [AGPL-3] License and not to allow others to use your version of this file
+# under the CPAL, indicate your decision by deleting the provisions above and
+# replace them with the notice and other provisions required by the [AGPL-3]
+# License. If you do not delete the provisions above, a recipient may use your
+# version of this file under either the CPAL or the [AGPL-3] License.
+
+
+
+## This module takes textures decoded from JPEG, PNG, etc. and encodes them in
+## compressed GPU formats like BCn and ASTC. In order to use it:
+##
+##  * Configure `Engine.CacheSettings`
+##  * Add one or both of these defines: `myouUseBC7Encoder`, `myouUseAstcEncoder`
+##
+## The resulting cache can be used in builds without encoders.
+
 
 import ../types
 import ./texture_decode
-from dds_ktx import KtxInfo, KtxPart, get_ASTC_internal_format
+from dds_ktx import KtxInfo, KtxPart, KtxInfoParts, get_ASTC_internal_format
 import arr_ref
+import loadable
+import zstd/decompress
 import stb_image_resize
 import std/monotimes
+import std/marshal
 import std/bitops
+import std/json
+import std/uri
+import std/os
 
 # TODO: don't import it here
 from ../platform/gl import nil
@@ -14,6 +62,12 @@ when defined(myouUseBC7Encoder):
     import bc7enc
 when defined(myouUseAstcEncoder):
     import astc
+when defined(myouUseBC7Encoder) or defined(myouUseAstcEncoder):
+    import zstd/compress
+
+when defined(nimdoc):
+    type TYPES* = CacheSettings | EncodingSpeed | RgbBcFmt | BlockSize
+
 when defined(android) or defined(ios) or defined(emscripten):
     template has_bptc_support: bool = gl.GLAD_GL_EXT_texture_compression_bptc
 else:
@@ -22,14 +76,13 @@ template has_astc_support: bool = gl.GLAD_GL_OES_texture_compression_astc or
                                   gl.GLAD_GL_KHR_texture_compression_astc_ldr
 
 const myouEngineNumTextureThreads {.intdefine.} = 4
-const myouEngineCompressTextures {.booldefine.} = true
+const myouBC7VerboseMode {.booldefine.} = false
-const myouBC7VerboseMode {.booldefine.} = true
 
 template u32(x: untyped): uint32 = cast[uint32](x)
 
 type CallbackUncompressed = proc(tex: Texture, data: SliceMem[byte]) {.gcsafe.}
-type CallbackCompressed = proc(tex: Texture, info: KtxInfo, data: seq[KtxPart], refdata: seq[ArrRef[byte]]) {.gcsafe.}
+type CallbackCompressed = proc(tex: Texture, data: KtxInfoParts, refdata: seq[SliceMem[byte]]) {.gcsafe.}
-type CompressMipmapResult = tuple[data: ArrRef[byte], row_len: int]
+type CompressMipmapResult = tuple[data: SliceMem[byte], row_len: int]
 type CompressMipmap = proc(width, height: int32, p: pointer, len: int32): CompressMipmapResult {.closure.}
 
 template block_file_size(width, height, depth, block_x, block_y, block_z, block_byte_size: untyped): int = 
@@ -38,7 +91,7 @@ template block_file_size(width, height, depth, block_x, block_y, block_z, block_
     let blocks_z = (depth.int + block_z.int - 1) div block_z.int
     blocks_x * blocks_y * blocks_z * block_byte_size.int
 
-proc make_mipmaps(tex: Texture, pixels: SliceMem[byte], compress: CompressMipmap): (seq[KtxPart], seq[ArrRef[byte]]) =
+proc make_mipmaps(tex: Texture, pixels: SliceMem[byte], compress: CompressMipmap): (seq[KtxPart], seq[SliceMem[byte]]) =
     var width = tex.width.int32
     var height = tex.height.int32
     let depth = tex.format_depth
@@ -56,7 +109,7 @@ proc make_mipmaps(tex: Texture, pixels: SliceMem[byte], compress: CompressMipmap
     let pixel_stride = 4 * (tex.format.stride div tex.format.channel_count)
     let time = getmonotime().ticks.float/1000000000
     var parts: seq[KtxPart]
-    var data_refs: seq[ArrRef[byte]]
+    var data_refs: seq[SliceMem[byte]]
     var mip_level = 0'i32
     var w, prev_w = width
     var h, prev_h = height
@@ -111,7 +164,7 @@ when defined(myouUseBC7Encoder):
                 )
                 let out_len = block_file_size(w,h,1, 4,4,1, block_size_bytes)
                 let row_len = block_file_size(w,1,1, 4,4,1, block_size_bytes)
-                var data = newArrRef[byte](out_len)
+                var data = newSliceMem[byte](out_len)
                 var output = EncodeBcOutput(
                     data: data.toPointer,
                     len: out_len.int32,
@@ -128,10 +181,16 @@ when defined(myouUseBC7Encoder):
                 is_sRGB: tex.is_sRGB, is_bc: true,
                 internal_format: internal_format,
             )
-            callback(tex, info, parts, data_refs)
+            callback(tex, KtxInfoParts(info: info, parts: parts), data_refs)
+
+template first(bs: BlockSize): uint8 =
+    bs.uint8 shr 4'u8
+
+template second(bs: BlockSize): uint8 =
+    bs.uint8 and 0xf'u8
 
 when defined(myouUseAstcEncoder):
-    proc atsc_compress*(tex: Texture, pixels: SliceMem[byte], callback: CallbackCompressed, blk_size = (6,6), quality = 10.0) =
+    proc atsc_compress*(tex: Texture, pixels: SliceMem[byte], callback: CallbackCompressed, blk_size: BlockSize, speed: EncodingSpeed) =
         let profile = case tex.format.component_type:
             of UByte:
                 if tex.is_sRGB: ASTCENC_PRF_LDR_SRGB
@@ -152,10 +211,18 @@ when defined(myouUseAstcEncoder):
             of 3: AstcencSwizzle(r: SWZ_R, g: SWZ_G, b: SWZ_B, a: SWZ_1)
             else: AstcencSwizzle(r: SWZ_R, g: SWZ_G, b: SWZ_B, a: SWZ_A)
         let flags = 0'u32
+        let quality = case speed:
+            of UltraFast: 0.0
+            of VeryFast: 10.0
+            of Fast: 30.0
+            of Basic: 60.0
+            of Slow: 98.0
+            of VerySlow: 99.0
+            of Slowest: 100.0
         var config: AstcencConfig
         # TODO: 3D texture block size
         astc_assert astcenc_config_init(profile,
-            blk_size[0].uint32, blk_size[1].uint32, 1'u32,
+            blk_size.first.uint32, blk_size.second.uint32, 1'u32,
             quality, flags, config.addr)
         # config.progress_callback = proc(p: float32) {.cdecl.} =
         #     echo "progress ", p
@@ -167,7 +234,7 @@ when defined(myouUseAstcEncoder):
                 config.block_x, config.block_y, config.block_z, 16)
             let row_len = block_file_size(w, 1, 1,
                 config.block_x, config.block_y, config.block_z, 16)
-            let data = newArrRef[byte](buffer_size)
+            let data = newSliceMem[byte](buffer_size)
             let pointers = [p]
             let img = AstcencImage(
                 dim_x: w.u32, dim_y: h.u32, dim_z: 1.u32,
@@ -182,41 +249,67 @@ when defined(myouUseAstcEncoder):
 
         let (parts, data_refs) = make_mipmaps(tex, pixels, compress)
         
+        let blk = (blk_size.first, blk_size.second)
         let info = KtxInfo(
             width: parts[0].width, height: parts[0].height, depth: tex.format_depth.int32,
             num_layers: 1, num_mipmaps: parts.len.int32, has_alpha: tex.format.channel_count == 4,
             is_sRGB: tex.is_sRGB, is_astc: true,
-            internal_format: get_ASTC_internal_format(blk_size, tex.is_sRGB)
+            internal_format: get_ASTC_internal_format(blk, tex.is_sRGB)
         )
-        callback(tex, info, parts, data_refs)
+        callback(tex, KtxInfoParts(info: info, parts: parts), data_refs)
         astc_assert astcenc_compress_reset(ctx)
         astcenc_context_free(ctx)
 
+func `%`(p: pointer): JsonNode = %0
+
 proc loadOptimized*(tex: Texture, slices: seq[SliceMem[byte]],
         callback_uncompressed: CallbackUncompressed = nil,
         callback_compressed: CallbackCompressed = nil,
         flip = true, min_channels = 0) {.gcsafe.} =
     
+    let settings = tex.engine.cache_settings
     var min_channels = min_channels
-    var will_compress = myouEngineCompressTextures
+    var will_compress = settings.compress_textures
     var will_load_uncompressed_first = false
-    var will_encode_all = false
+    var will_encode_all = settings.compress_all_formats
 
     will_compress = will_compress and
         callback_compressed != nil and
         tex.format.component_type != UShort
     
+    var native_bc, native_astc = false
     if has_bptc_support:
+        native_bc = true
         # TODO: BC6H or RGBM
         will_compress = will_compress and
             tex.format.component_type == UByte
     elif has_astc_support:
-        # TODO: detect HDR support
+        native_astc = true
-        discard
+        # TODO: detect HDR support, use a fallback if unavailable
-    elif will_encode_all:
+    
-        will_load_uncompressed_first = true
+    # TODO: USE A BETTER KEY, INCLUDE SETTINGS
-    else:
+    let cache_key = tex.name
-        will_compress = false
+    let cache_file_name = cache_key.encodeUrl & ".zst"
+    if settings.use_cache and callback_compressed != nil:
+        let cache_file = settings.cache_dir & "/" & cache_file_name
+        # TODO: allow networked requests
+        if fileExists cache_file:
+            try:
+                let slices = readFile(cache_file).decompress.toSliceMem.to(byte).deserialize
+                var data = to[KtxInfoParts](slices[^1].toString)
+                for i,p in data.parts.mpairs:
+                    p.data = slices[i].toPointer
+                tex.callback_compressed(data, slices)
+                return
+            except:
+                # TODO: proper error handling and logging
+                echo "ERROR: could not load cache file for " & tex.name
+
+    if not (native_bc or native_astc):
+        if will_encode_all:
+            will_load_uncompressed_first = true
+        else:
+            will_compress = false
     
     if will_compress:
         min_channels = 4
@@ -226,26 +319,41 @@ proc loadOptimized*(tex: Texture, slices: seq[SliceMem[byte]],
                 (will_load_uncompressed_first or not will_compress):
             callback_uncompressed(tex, pixels)
         if will_compress:
+            when defined(myouUseBC7Encoder) or defined(myouUseAstcEncoder):
+                proc cb(tex: Texture, data: KtxInfoParts, 
+                        refdata: seq[SliceMem[byte]]) {.gcsafe.} =
+                    let info = data.info
+                    if (info.is_bc and native_bc) or
+                        (info.is_astc and native_astc):
+                            callback_compressed(tex, data, refdata)
+                    if settings.save_cache:
+                        let dir = if info.is_bc: settings.cache_dir_bc
+                        else: settings.cache_dir_astc
+                        let cache_file = dir & "/" & cache_file_name
+                        let outdata = refdata & @[($(%data)).toSliceMem.to(byte)]
+                        writeFile cache_file, outdata.serialize.toOpenArray.compress
+
+                let channels = tex.format.channel_count
+
             when defined(myouUseBC7Encoder):
-                let bc_format = if tex.format.channel_count == 1:
+                if has_bptc_support or will_encode_all:
-                    4.int8 # BC4 is 0.5 bytes per pixel grayscale
+                    let bc_format = if channels == 1:
-                else:
+                        4.int8 # BC4
-                    7.int8 # BC7 is just the best at anything else
+                    else:
-                if has_bptc_support:
+                        settings.bc_format_for_RGB.int8 # BC1 or BC7
-                    bcn_compress(tex, pixels, callback_compressed, bc_format)
+                    bcn_compress(tex, pixels, cb, bc_format)
                     if not will_encode_all: return
-                elif will_encode_all:
-                    # TODO: callback to store result
-                    discard
-                    # bcn_compress(tex, pixels, ..., bc_format)
             when defined(myouUseAstcEncoder):
                 if has_astc_support or will_encode_all:
-                    atsc_compress(tex, pixels, callback_compressed)
+                    let blk_size = if channels == 1:
+                        settings.astc_block_1ch
+                    elif channels == 2:
+                        settings.astc_block_2ch
+                    else:
+                        settings.astc_block_size
+                    atsc_compress(tex, pixels, cb,
+                                blk_size, settings.quality_speed)
                     if not will_encode_all: return
-                elif will_encode_all:
-                    # TODO: callback to store result
-                    discard
-                    # atsc_compress(tex, pixels, ...)
 
     , flip = flip, min_channels = min_channels)
 
@@ -266,7 +374,7 @@ type DecodeReturnChanMsg = tuple[
 var decode_return_chan: Channel[DecodeReturnChanMsg]
 type CompressedReturnChanMsg = tuple[
     callback: CallbackCompressed,
-    tex: Texture, info: KtxInfo, data: seq[KtxPart], refdata: seq[ArrRef[byte]],
+    tex: Texture, data: KtxInfoParts, refdata: seq[SliceMem[byte]],
 ]
 var compressed_return_chan: Channel[CompressedReturnChanMsg]
 
@@ -292,9 +400,8 @@ proc workerThreadProc() {.thread.} =
             break
         proc cb(tex: Texture, data: SliceMem[byte]) =
             decode_return_chan.send((callback: to_decode.callback_uncompressed, tex: tex, data: data))
-        proc cbc(tex: Texture, info: KtxInfo, data: seq[KtxPart], refdata: seq[ArrRef[byte]]) =
+        proc cbc(tex: Texture, data: KtxInfoParts, refdata: seq[SliceMem[byte]]) =
-            compressed_return_chan.send((callback: to_decode.callback_compressed, tex: tex, 
+            compressed_return_chan.send((callback: to_decode.callback_compressed, tex: tex, data: data, refdata: refdata))
-            info: info, data: data, refdata: refdata))
         let cb_out = if to_decode.callback_uncompressed != nil: cb else: nil
         let cbc_out = if to_decode.callback_compressed != nil: cbc else: nil
         loadOptimized(to_decode.tex, to_decode.slices, cb_out, cbc_out,
@@ -318,8 +425,8 @@ proc updateTextureWorkerThreads*() =
         let tried = compressed_return_chan.tryRecv()
         if not tried.dataAvailable:
             break
-        let (cb, tex, info, data, refdata) = tried.msg
+        let (cb, tex, data, refdata) = tried.msg
-        cb(tex, info, data, refdata)
+        cb(tex, data, refdata)
 
 proc terminateTextureWorkerThreads*() =
     for worker in workers:

src/graphics/texture.nim

@@ -362,19 +362,19 @@ proc loadCubeSideFromPixels*(self: Texture, pixels: pointer, side: int32 = 0) =
         self.width.GLsizei, self.height.GLsizei,
         0, ts.format, ts.gltype, pixels)
 
-proc loadCompressedData*(self: Texture, info: KtxInfo, data: seq[KtxPart], refdata: seq[ArrRef[byte]]) {.gcsafe.} =
+proc loadCompressedData*(self: Texture, data: KtxInfoParts, refdata: seq[SliceMem[byte]]) {.gcsafe.} =
-    assert info.depth == 1 and info.num_layers == 1,
+    assert data.info.depth == 1 and data.info.num_layers == 1,
         "Compressed array and 3D textures not supported yet"
     let ts = self.storage
     self.loaded = true
     self.bind_it(needs_active_texture=true)
     let target = if self.tex_type == TexCube: GL_TEXTURE_CUBE_MAP_POSITIVE_X.GLuint.int32
     else: ts.target.GLuint.int32
-    for part in data:
+    for part in data.parts:
         glCompressedTexImage2D(cast[GLenum](target+part.face), part.mip_level,
-            info.internal_format.GLenum, part.width.GLsizei, part.height.GLsizei,
+            data.info.internal_format.GLenum, part.width.GLsizei, part.height.GLsizei,
             0, part.len.GLsizei, part.data)
-    self.setMipmapRange(0, info.num_mipmaps - 1)
+    self.setMipmapRange(0, data.info.num_mipmaps - 1)
 
 proc setFilter*(self: Texture, filter: TextureFilter) =
     self.filter = filter
@@ -485,7 +485,10 @@ proc newTexture*(engine: MyouEngine, name: string, file_name: string, is_sRGB: b
             engine.renderer.enqueue proc()=
                 try:
                     self.ensure_storage()
-                    self.loadCompressedData(ktx_info, ParseDdsKtx(data.data, data.byte_len), @[])
+                    let info_parts = KtxInfoParts(
+                        info: ktx_info,
+                        parts: ParseDdsKtx(data.data, data.byte_len))
+                    self.loadCompressedData(info_parts, @[data])
                     self.loaded = true
                 # except Exception as e:
                 except:

src/loaders/blend.nim

@@ -92,8 +92,8 @@ proc registerBlendLoader*(engine: MyouEngine) =
 method close*(self: BlendLoader) =
     if self.resource != nil:
         self.resource = nil
-        self.blend_file = nil
+    self.blend_file = nil
-        self.cached_materials.clear()
+    self.cached_materials.clear()
 
 method openAssetFile*(self: BlendLoader, path: string) =
     if self.resource != nil:

src/types.nim

@@ -84,6 +84,7 @@ type
         use_glsl_tone_mapping*: bool
         loaders_by_ext*: Table[string, seq[proc(e: MyouEngine): Loader]]
         glsl_version*: string
+        cache_settings*: CacheSettings
         all_framebuffers*: seq[Framebuffer] ## private
         new_scenes*: Table[string, Scene] ## private
     
@@ -762,6 +763,61 @@ type
         # on_destroy*: OnDestroy
         path_handler*: proc(path: string): string
         override_textures_sampler_type*: Table[string, string]
+    
+    # texture_optimize.nim
+
+    EncodingSpeed* = enum
+        UltraFast
+        VeryFast
+        Fast
+        Basic
+        Slow
+        VerySlow
+        Slowest
+    
+    RgbBcFmt* = enum
+        BC1 = 1 ## 4 BPP (0.5 bytes per pixel), very small but low quality.
+        BC7 = 7 ## 8 BPP (1 byte per pixel), larger but very good quality.
+
+    BlockSize* = enum
+        Bs4x4 = 0x4_4 ## 8.00 BPP
+        Bs5x4 = 0x5_4 ## 6.40 BPP
+        Bs5x5 = 0x5_5 ## 5.12 BPP
+        Bs6x5 = 0x6_5 ## 4.27 BPP
+        Bs6x6 = 0x6_6 ## 3.56 BPP
+        Bs8x5 = 0x8_5 ## 3.20 BPP
+        Bs8x6 = 0x8_6 ## 2.67 BPP
+        Bs8x8 = 0x8_8 ## 2.00 BPP (below 10x6)
+        Bs10x5 = 0xA_5 ## 2.56 BPP
+        Bs10x6 = 0xA_6 ## 2.13 BPP
+        Bs10x8 = 0xA_8 ## 1.60 BPP
+        Bs10x10 = 0xA_A ## 1.28 BPP
+        Bs12x10 = 0xC_A ## 1.07 BPP
+        Bs12x12 = 0xC_C ## 0.89 BPP
+        # TODO: 3D block sizes
+
+    CacheSettings* = object
+        compress_textures*: bool        ## Whether to compress textures on load
+                                        ## when not loaded from cache cache
+        use_cache*: bool                ## Whether to try to load from cache
+        save_cache*: bool               ## Whether to save compressed textures
+        cache_dir*: string              ## Cache directory
+        quality_speed*: EncodingSpeed   ## Whether you need them fast or good
+        bc_format_for_RGB*: RgbBcFmt    ## Which BCn to use for RGB images.
+                                        ## BC7 is the best but BC1 is half the
+                                        ## size and encodes very fast
+        astc_block_size*: BlockSize     ## Defines quality and size of ASTC.
+                                        ## 4x4 is the best but biggest,
+                                        ## 6x6 is a good balance,
+                                        ## 8x8 is bad but very small.
+        astc_block_1ch*: BlockSize      ## Block size for 1 channel textures,
+                                        ## since less data has to be encoded
+        astc_block_2ch*: BlockSize      ## Block size for 2 channel textures
+
+        compress_all_formats*: bool     ## Encode textures for all platforms
+        cache_dir_bc*: string           ## Cache directory for writing bc
+        cache_dir_astc*: string         ## Cache directory for writing astc
+
 
     # INCOMPLETE
 

src/util.nim

@@ -92,7 +92,7 @@ func align4*[T: SomeInteger](n: T): T = n + ((4-(n and 3)) and 3)
 func align*[T: SomeInteger](n, align: T): T =
     let mask = align - 1
     assert((align and mask) == 0, "align must be a power of two")
-    return n + ((4-(n and mask)) and mask)
+    return n + ((align-(n and mask)) and mask)
 
 func bytelen*[T](s: seq[T]): int = s.len * sizeof(T)