ArrRef: Remove unnecessary copy of murmur3 hash code.

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "libs/pixie"]
 	path = libs/pixie
 	url = https://git.myou.dev/MyouProject/pixie
+[submodule "libs/nim_zstd"]
+	path = libs/nim_zstd
+	url = https://github.com/DiThi/nim_zstd

libs/arr_ref/arr_ref.nim

@@ -131,78 +131,9 @@ when defined(isNimSkull):
         if a.len != 0:
             copyMem(result[0].addr, a.arr.addr, result.len * sizeof(T))
 
-# The following is just copied straight from hashes.nim, just replacing openArray by ArrRef...
-
-when defined(js):
-  proc imul(a, b: uint32): uint32 =
-    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/imul
-    let mask = 0xffff'u32
-    var
-      aHi = (a shr 16) and mask
-      aLo = a and mask
-      bHi = (b shr 16) and mask
-      bLo = b and mask
-    result = (aLo * bLo) + (aHi * bLo + aLo * bHi) shl 16
-else:
-  template imul(a, b: uint32): untyped = a * b
-
-proc rotl32(x: uint32, r: int): uint32 {.inline.} =
-  (x shl r) or (x shr (32 - r))
-
 proc hash*[T](arr: ArrRef[T]): Hash =
-  # https://github.com/PeterScott/murmur3/blob/master/murmur3.c
-  const
-    c1 = 0xcc9e2d51'u32
-    c2 = 0x1b873593'u32
-    n1 = 0xe6546b64'u32
-    m1 = 0x85ebca6b'u32
-    m2 = 0xc2b2ae35'u32
-  let
-    x = arr.to byte
-    size = len(x)
-    stepSize = 4 # 32-bit
-    n = size div stepSize
-  var
-    h1: uint32
-    i = 0
-
-  # body
-  while i < n * stepSize:
-    var k1: uint32
-    when defined(js) or defined(sparc) or defined(sparc64):
-      var j = stepSize
-      while j > 0:
-        dec j
-        k1 = (k1 shl 8) or (ord(x[i+j])).uint32
-    else:
-      k1 = cast[ptr uint32](unsafeAddr x[i])[]
-    inc i, stepSize
-
-    k1 = imul(k1, c1)
-    k1 = rotl32(k1, 15)
-    k1 = imul(k1, c2)
-
-    h1 = h1 xor k1
-    h1 = rotl32(h1, 13)
-    h1 = h1*5 + n1
-
-  # tail
-  var k1: uint32
-  var rem = size mod stepSize
-  while rem > 0:
-    dec rem
-    k1 = (k1 shl 8) or (ord(x[i+rem])).uint32
-  k1 = imul(k1, c1)
-  k1 = rotl32(k1, 15)
-  k1 = imul(k1, c2)
-  h1 = h1 xor k1
-
-  # finalization
-  h1 = h1 xor size.uint32
-  h1 = h1 xor (h1 shr 16)
-  h1 = imul(h1, m1)
-  h1 = h1 xor (h1 shr 13)
-  h1 = imul(h1, m2)
-  h1 = h1 xor (h1 shr 16)
-  return cast[Hash](h1)
-
+    # Make use of stdlib's murmur3
+    # NOTE: We use the size of elements in bytes instead of the actual size
+    # just in case the actual size is bigger than that.
+    hash(cast[ptr UncheckedArray[byte]](arr.arr.addr).toOpenArray(0, arr.len * sizeof(T) - 1))
+    # TODO: for bigger elements, would a different algorithm be faster?

libs/nim_zstd

@@ -0,0 +1 @@
+Subproject commit 75ee00aa3da80903dc509f1ff1605304cc77c173

src/graphics/render.nim

@@ -192,7 +192,24 @@ proc draw_all*(self: RenderManager) =
                 continue
             scene.update_all_matrices()
             scene.update_lights()
-    # TODO: render probes required by cameras of enabled screens
+            # TODO: move this
+            var update_probe_ubo = false
+            for probe in scene.cubemap_probes:
+                let upd = case probe.update_strategy:
+                    of UpdateNever: false
+                    of UpdateOnFirstUse:
+                        # TODO: detect when relevant objects are fully loaded
+                        # and also when anything that affects it is visible and
+                        # nearby
+                        not probe.used
+                    of UpdateAlways: true
+                if upd:
+                    probe.render_cubemap(true)
+                    probe.used = true
+                    update_probe_ubo = true
+            if update_probe_ubo:
+                scene.cubemap_UBO.update()
+
     # TODO: render shadows required by cameras of enabled screens
     for screen in self.engine.screens:
         if not screen.enabled:

src/loaders/blend.nim

@@ -33,8 +33,11 @@
 {.warning[UseBase]: off.} # can't seem to satisy this...
 {.warning[UnusedImport]: off.} # for ./loader_base
 
+const myouUseBlendLoader {.booldefine.} = true
+
 import ../types
 import ./loader_base
+export loader_base
 import std/tables
 import std/strutils
 import std/strformat
@@ -66,8 +69,9 @@ import float16
 when defined(nimdoc):
     type TYPES* = BlendLoader
 
-proc newBlendLoader*(engine: MyouEngine, use_indices:bool = true,
-        shader_library: string = "", shader_textures = initTable[string, Texture]()): BlendLoader =
+proc newBlendLoader*(engine: MyouEngine,
+        shader_library: string = "",
+        shader_textures = initTable[string, Texture]()): BlendLoader =
     result = new BlendLoader
     result.engine = engine
     if shader_library != "":
@@ -76,8 +80,13 @@ proc newBlendLoader*(engine: MyouEngine, use_indices:bool = true,
     else:
         result.shader_library = get_builtin_shader_library()
         result.shader_textures = get_builtin_shader_textures()
-    result.file_extensions = @["blend"]
-    result.use_indices = use_indices
+    result.use_indices = true
+
+proc registerBlendLoader*(engine: MyouEngine) =
+    when myouUseBlendLoader:
+        engine.registerLoader(@["blend"], proc(e: MyouEngine): Loader =
+            e.newBlendLoader()
+        )
 
 var used_resources: seq[LoadableResource]
 
@@ -575,10 +584,16 @@ method loadScene*(self: BlendLoader, name: string="", scene: Scene=nil, callback
             if scn["id"]["name"].cstr == idname.cstring:
                 let node = scn
                 var scene = scene
+                let was_first_scene = self.engine.scenes.len == 0
                 if scene == nil:
                     scene = self.engine.new_scene(name=name)
                 self.loadSceneImpl(node, scene)
                 callback(scene)
+                # TODO: when loading is async, move this stuff after loading has
+                # finished
+                if was_first_scene and not scene.enabled:
+                    echo "Warning: Your scene is not enabled, use 'scene.enable_render()' or 'scene.enable_all()'"
+                scene.render_all_cubemaps(true)
                 return
         assert false, &"Scene {name} not found"
 

src/loaders/blend_format.nim

@@ -32,10 +32,11 @@
 
 {.experimental: "dotOperators".}
 
-import strutils, tables, algorithm, sequtils, strformat, bitops
-import memfiles
+import std/[strutils, tables, algorithm, sequtils, strformat, bitops]
+import std/memfiles
 # import sugar
-import hashes
+import std/hashes
+import zstd/decompress
 
 # TODO!! ADD BOUND CHECKS TO ALL FNODES AND DNA1 PARSING!
 # TODO: implement big endian, test 32 bit
@@ -56,6 +57,8 @@ type
         old_mem_ptr: pointer
         # end pointer for bounds checking
         endp: pointer
+        # byte seq in case we used zstd
+        byte_seq: seq[byte]
     
     BlendFile* = ref BlendFileVal
 
@@ -142,13 +145,16 @@ template `-`(p, q: pointer): int = cast[int](cast[int](p) -% cast[int](q))
 template `+=`(p: pointer, i: Natural) = p = cast[pointer](cast[int](p) +% cast[int](i))
 
 # this is for getting a string that may not be null terminated
-# otherwise you can just do $cast[cstring](x.addr)
+# otherwise you could just do $cast[cstring](x.addr)
 proc getString(data: pointer, max_size: Natural): string =
     let u8 = cast[ptr UncheckedArray[char]](data)
-    for i in 0 ..< max_size:
+    var i = 0
+    while i < max_size:
         if u8[i] == '\0':
             break
-        result &= u8[i]
+        inc(i)
+    result.setLen i
+    copyMem(result.cstring, data, i)
 
 proc build_dna_table(dna_table: ptr FBlock64): (TypeAttrsRef, StructToTypeMap, seq[cstring], TypeLengths)  =
     var type_attrs = new TypeAttrsRef
@@ -230,25 +236,21 @@ proc build_dna_table(dna_table: ptr FBlock64): (TypeAttrsRef, StructToTypeMap, s
             )
             aoffset += asize
             offset += 4
-        # if true:
-        # # if lengths[type_id] != aoffset:
-        #     echo &"{types[type_id]}: {lengths[type_id]}, {aoffset}"
-        #     var offsets = toSeq(type_attrs[type_id].pairs)
-        #     for _,(k,v) in offsets.sortedByIt(it[1].offset):
-        #         echo &"{types[type_id]}.{k}:\t\t{v.offset}"
         assert lengths[type_id] == aoffset
         # type_attrs[type_id]["(size)"] = AttrOffset(offset: lengths[type_id])
     return (type_attrs, struct_to_type, types, lengths)
 
 proc openBlendFile*(path: string, data: pointer, len: int): BlendFile =
     result = new BlendFile
-    # result.mem_file = memfiles.open(path, mode=fmRead)
     result.file_path = path
-    # let mem = result.mem_file.mem
-    # result.mem = mem
-    # let file_length = result.mem_file.size
-    let mem = data
-    let file_length = len
+    let (mem, file_length) = if cast[ptr uint32](data)[] == 0xFD2FB528'u32:
+        # Zstandard compressed file
+        # TODO: store offsets of each zstd frame so we can semi-random access
+        # big files later
+        result.byte_seq = decompress cast[ptr UncheckedArray[byte]](data).toOpenArray(0, len-1)
+        (result.byte_seq[0].addr.pointer, result.byte_seq.len)
+    else:
+        (data, len)
     assert file_length > 32, "Invalid file size"
     result.endp = mem + file_length
     let u8 = cast[ptr UncheckedArray[uint8]](mem)

src/loaders/loader_base.nim

@@ -31,13 +31,22 @@
 # version of this file under either the CPAL or the [AGPL-3] License.
 
 import ../types
+import std/tables
 
 when defined(nimdoc):
     type TYPES* = Loader
 
-method openAssetFile*(self: Loader, path: string) {.base, locks: "unknown".} =
+method openAssetFile*(self: Loader, path: string) {.base.} =
     discard
-method loadScene*(self: Loader, name: string="", scene: Scene=nil): Scene {.base, locks: "unknown".} =
+method loadScene*(self: Loader, name: string="", scene: Scene=nil, 
+    callback: proc(scene: Scene)) {.base.} =
     discard
-method loadImageImpl*(self: Loader) {.base, locks: "unknown".} =
-    discard
+method loadImageImpl*(self: Loader) {.base.} =
+    discard
+
+proc registerLoader*(engine: MyouEngine,
+        extensions: seq[string],
+        constructor: proc(e: MyouEngine): Loader) =
+    
+    for ext in extensions:
+        engine.loaders_by_ext.mgetOrPut(ext, @[]).add constructor

src/myou_engine.nim

@@ -74,6 +74,7 @@ import std/monotimes
 import ./graphics/render
 import ./screen
 import ./platform/platform
+import ./loaders/blend
 import ./util
 import arr_ref
 
@@ -129,6 +130,8 @@ proc newMyouEngine*(
     when not defined(nimdoc):
         init_graphics(result, width, height, title, opengl_version, opengl_es)
         discard result.newScreen(width, height, title)
+    
+    registerBlendLoader(result)
 
 proc get_builtin_shader_library*(use_cubemap_prefiltering = true): string =
     ## Returns a string with the code of the default shader library of the
@@ -179,3 +182,18 @@ proc run*(self: MyouEngine) =
     last_time = getmonotime().ticks.float/1000000000
     when not defined(nimdoc):
         start_platform_main_loop(self, myou_main_loop)
+
+proc loadScene*(self: MyouEngine, uri: string, callback: proc(scene: Scene), name = "", ext = "") =
+    let ext = if ext == "":
+        uri.rsplit('.', 1)[1]
+    else:
+        ext
+    
+    if ext notin self.loaders_by_ext:
+        raise ValueError.newException "File extension not supported: " & ext
+    
+    # TODO: use the next loader if the first one fails
+    let loaders = self.loaders_by_ext[ext]
+    let loader = loaders[0](self)
+    loader.openAssetFile(uri)
+    loader.loadScene(name, nil, callback)

src/objects/cubemap_probe.nim

@@ -38,6 +38,16 @@ when defined(nimdoc):
     type TYPES* = ProbeInfluenceType | ProbeParallaxType | CubemapProbe | CubemapProbeUniform | SH9Uniform
 
 # Forward declarations and ob type methods
+proc newCubemapProbe*(engine: MyouEngine, name: string="camera", scene: Scene = nil,
+        influence_type: ProbeInfluenceType = SphereInfluence,
+        influence_distance: float32 = 2.5,
+        falloff: float32 = 0.2,
+        intensity: float32 = 1,
+        clipping_start: float32 = 0.8,
+        clipping_end: float32 = 40,
+        parallax_type: ProbeParallaxType = NoParallax,
+        parallax_distance: float32 = 0.0, # 0 means auto
+    ): CubemapProbe
 func getCubemapSideMatrix*(side: int, position = vec3()): Mat4
 proc render_cubemap*(self: CubemapProbe, use_roughness_prefiltering = false, mipmap_shader: Material = nil)
 proc render_background_cubemap*(scene: Scene, use_roughness_prefiltering = false, mipmap_shader: Material = nil, world_to_cube_matrix: Mat4 = mat4(), upload_UBO = true)
@@ -86,6 +96,7 @@ proc newCubemapProbe*(engine: MyouEngine, name: string="camera", scene: Scene =
         parallax_distance
     else:
         influence_distance
+    self.update_strategy = UpdateOnFirstUse
     return self
 
 

src/scene.nim

@@ -351,6 +351,17 @@ proc destroy*(self: Scene) =
     self.engine.scenes.del(self.name)
     return
 
+proc enable_render*(self: Scene) =
+    self.enabled = true
+
+proc enable_physics*(self: Scene) =
+    if self.world != nil:
+        self.world.enabled = true
+
+proc enable_all*(self: Scene) =
+    self.enable_render()
+    self.enable_physics()
+
 proc new_gameobject*(self: Scene, name: string): GameObject =
     return self.engine.new_gameobject(name=name, scene=self)
 
@@ -528,10 +539,6 @@ proc get_lighting_code*(self: Scene): string =
     code.add staticOrDebugRead "shadows/simple_shadow.glsl"
     return code.join "\n"
 
-
-# template as_float_array(x: untyped): untyped =
-#     cast[array[sizeof(typeof(x)) div 4, float]](x)
-
 proc update_lights*(self: Scene) =
     if self.lighting_UBOs.len == 0:
         return

src/types.nim

@@ -81,6 +81,7 @@ type
         renderer*: RenderManager
         tone_mapping_library*: string
         tone_mapping_function*: string
+        loaders_by_ext*: Table[string, seq[proc(e: MyouEngine): Loader]]
         # TODO: remove this and query it when screen is created
         width*, height*: int
         glsl_version*: string
@@ -693,6 +694,11 @@ type
     
     # cubemap_probe.nim
 
+    ProbeUpdateStrategy* = enum
+        UpdateNever
+        UpdateOnFirstUse
+        UpdateAlways
+
     ProbeInfluenceType* = enum
         SphereInfluence
         BoxInfluence
@@ -714,6 +720,8 @@ type
         intensity*: float32
         clipping_start*: float32
         clipping_end*: float32
+        update_strategy*: ProbeUpdateStrategy
+        used*: bool
     
     # NOTE: layout must match struct CubemapInfo in scene.nim
     CubemapProbeUniform* = object
@@ -738,6 +746,16 @@ type
         influence_distance*: float32
         falloff*: float32
 
+    # loader_base.nim
+
+    Loader* = ref object of RootObj
+        engine*: MyouEngine
+        shader_library*: string
+        shader_textures*: Table[string, Texture]
+        # on_destroy*: OnDestroy
+        path_handler*: proc(path: string): string
+        override_textures_sampler_type*: Table[string, string]
+
     # INCOMPLETE
 
     Armature* = ref object of GameObject
@@ -750,6 +768,8 @@ type
         parent_object*: GameObject
 
     World* = ref object of RootObj
+        enabled*: bool
+    
     DebugDraw* = ref object of RootObj
     Body* = ref object
 
@@ -806,15 +826,6 @@ const HARDCODED_MAXUBOS* = 80
 
 # LOADERS
 
-type Loader* = ref object of RootObj
-    engine*: MyouEngine
-    file_extensions*: seq[string]
-    shader_library*: string
-    shader_textures*: Table[string, Texture]
-    # on_destroy*: OnDestroy
-    path_handler*: proc(path: string): string
-    override_textures_sampler_type*: Table[string, string]
-
 type BlendLoader* = ref object of Loader
     blend_file_path*: string ## private
     blend_file*: BlendFile ## private
@@ -822,16 +833,15 @@ type BlendLoader* = ref object of Loader
     cached_materials*: Table[(FNode, string, bool),
         (string, seq[Varying], OrderedTable[string, string], OrderedTable[string, TexturePixels])] ## private
     resource*: LoadableResource
-
-template AllLoaders*: untyped = @[
-    newBlendLoader,
-]
-
-template enqueue*(self: RenderManager, fun: proc()) =
-    if self.initialized:
+    
+template enqueue*(renderer: RenderManager, fun: proc()) =
+    ## Run a proc after the renderer has been initialized.
+    ## 
+    ## If it was already initialized, it runs it immediately.
+    if renderer.initialized:
         fun()
     else:
-        self.queue.add fun
+        renderer.queue.add fun
 
 when not defined(release):
     from sugar import dump