Add threading support to LoadableResource.

TODO: Use destructors to set all cursors of children objects to `nil`.

libs/loadable/loadable.nim

@@ -58,12 +58,14 @@ type LoadableResource* = ref object of RootObj
     done_func: proc()
     cancel_func: proc()
     str*: proc(): string
+    use_threads: bool
     result: ref Result # only to be used with loadAll
 
 proc newLoadableResource*[T: LoadableResource](
         start: proc(self: LoadableResource),
         done: proc() = nil,
         str: proc(): string = nil,
+        use_threads = false,
     ): T =
     new(result)
     result.start_func = start
@@ -71,11 +73,29 @@ proc newLoadableResource*[T: LoadableResource](
     result.str = str
     if str == nil:
         result.str = proc(): string = ""
+    result.use_threads = use_threads
+
+# main -> thread  channels
+var to_start: Channel[LoadableResource]
+# main <- thread  channels
+var to_return: Channel[(LoadableResource, bool, string, pointer, int)]
+var to_done: Channel[LoadableResource]
 
 proc start*[T: LoadableResource](self: T) =
     self.status = Started
+    if self.use_threads:
+        to_start.send self
+    else:
         self.start_func(self)
 
+proc doneImpl*[T: LoadableResource](self: T) =
+    if self.status == Started:
+        self.cancel()
+    else:
+        self.status = NotStarted
+        if self.done_func != nil:
+            self.done_func()
+
 proc `onload=`*[T: LoadableResource](self: T, onload_func: proc(ok: bool, err: string, p: pointer, len: int)) =
     self.onload_func = onload_func
 
@@ -85,25 +105,24 @@ proc onload*[T: LoadableResource](self: T, ok: bool, err: string, p: pointer, le
         if self.result != nil:
             self.result[] = (ok, err, p, len)
         if self.onload_func != nil:
+            if self.use_threads:
+                to_return.send((self.LoadableResource, ok, err, p, len))
+            else:
                 self.onload_func(ok, err, p, len)
     else: # cancelled
-        self.status = NotStarted
+        self.doneImpl()
-        if self.done_func != nil:
-            self.done_func()
 
 # TODO: check if we can always use destructors instead of calling this
 proc done*[T: LoadableResource](self: T) =
-    if self.status == Started:
+    if self.use_threads:
-        self.cancel()
+        to_done.send self
     else:
-        self.status = NotStarted
+        self.doneImpl()
-        if self.done_func != nil:
-            self.done_func()
 
 proc cancel*[T: LoadableResource](self: T) =
     if self.status != Started:
         return
-    if self.cancel_func != nil:
+    if self.cancel_func == nil:
         # self.`onload=`(proc(ok, err, p, len: auto) =
         #     self.done())
         self.onload_func = proc(ok: bool, err: string, p: pointer, len: int) =
@@ -128,6 +147,31 @@ proc cancel*[T: LoadableResource](self: T) =
 #             results.add res.result[]
 #             res.result = nil
 
+var worker: Thread[void]
+proc workerThreadProc() {.thread.} =
+    while true:
+        let res = to_start.recv()
+        if res == nil:
+            break
+        cast[proc(self: LoadableResource) {.gcsafe.}](res.start_func)(res)
+
+worker.createThread(workerThreadProc)
+to_start.open()
+to_return.open()
+to_done.open()
+
+proc updateLoadableWorkerThreads*() =
+    while true:
+        let tried = to_return.tryRecv()
+        if not tried.dataAvailable:
+            break
+        let (res, ok, err, p, len) = tried.msg
+        res.onload_func(ok, err, p, len)
+    while true:
+        let tried = to_done.tryRecv()
+        if not tried.dataAvailable:
+            break
+        tried.msg.done_func()
 
 type Fetch* = ref object of LoadableResource
     custom: pointer
@@ -226,6 +270,7 @@ proc loadUri*(
     var done_func: proc()
     var self: Fetch
     var uri = uri
+    var use_threads = false
     when not defined(onlyLocalFiles):
         when defined(emscripten):
             const is_remote = true
@@ -261,6 +306,7 @@ proc loadUri*(
                         discard emscripten_fetch_close(cast[ptr emscripten_fetch_t](self.custom))
                         self.custom = nil
             else:
+                use_threads = true
                 var client = newHttpClient()
                 var response: string
                 start_func = proc(self: LoadableResource) =
@@ -281,9 +327,14 @@ proc loadUri*(
 
     if not is_remote:
         start_func = proc(self: LoadableResource) =
+            when not defined(release):
+                var done_called = false
             try:
                 var memfile = memfiles.open(uri, mode=fmRead)
                 self.done_func = proc() =
+                    when not defined(release):
+                        assert not done_called, "Done is being called multiple times. Did you forget to set auto_done = false?"
+                        done_called = true
                     memfile.close()
                 self.onload(true, "", memfile.mem, memfile.size)
                 # TODO!!!! check whether these objects are freed
@@ -293,7 +344,7 @@ proc loadUri*(
             if cast[Fetch](self).auto_done:
                 self.done()
     proc str(): string = uri
-    self = newLoadableResource[Fetch](start_func, done_func, str)
+    self = newLoadableResource[Fetch](start_func, done_func, str, use_threads=use_threads)
     self.onload_func = onload_func
     self.auto_done = auto_done
     if auto_start:

src/bundle.nim

@@ -89,9 +89,11 @@ export ubo
 export util
 export vmath except Quat
 
-when defined(android):
 import platform/platform
+when defined(android):
     export platform.myouAndroidGetActivity
     export platform.myouAndroidGetJniEnv
     export platform.myouAndroidGetInternalDataPath
     export platform.myouAndroidGetExternalDataPath
+
+export platform.myouSetKeyboardVisible

src/graphics/framebuffer.nim

@@ -340,4 +340,4 @@ proc newMainFramebuffer*(engine: MyouEngine): Framebuffer =
     result.engine = engine
     result.framebuffer = 0
     result.is_complete = true
-    result.use_sRGB = true
+    result.use_sRGB = not engine.use_glsl_tone_mapping

src/loaders/blend.nim

@@ -88,14 +88,9 @@ proc registerBlendLoader*(engine: MyouEngine) =
             e.newBlendLoader()
         )
 
-var used_resources: seq[LoadableResource]
-
 method close*(self: BlendLoader) =
     if self.resource != nil:
         self.resource.done()
-        # TODO: investigate
-        # this is a hack to avoid a crasn on android
-        used_resources.add self.resource
         self.resource = nil
         self.blend_file = nil
         self.cached_materials.clear()
@@ -111,12 +106,16 @@ proc loadAsync(self: BlendLoader, callback: proc()) =
         callback()
     else:
         self.close()
-        self.resource = self.blend_file_path.loadUri proc(ok, err, data, len: auto) =
+        self.resource = loadUri(self.blend_file_path,
+            proc(ok, err, data, len: auto) =
                 if ok:
                     self.blend_file = openBlendFile(self.blend_file_path, data, len)
                     callback()
                 else:
                     raise IOError.newException err
+            ,
+            auto_done = false
+        )
 
 type BlenderObTypes = enum
     BEmpty = 0

src/myou_engine.nim

@@ -64,6 +64,7 @@ proc newMyouEngine*(
         opengl_version = default_gl_version,
         opengl_es = default_gl_es,
         glsl_version = "",
+        use_glsl_tone_mapping = true,
     ): MyouEngine
 proc get_builtin_shader_library*(use_cubemap_prefiltering = true): string
 proc get_builtin_shader_textures*(): Table[string, Texture]
@@ -76,6 +77,7 @@ import ./screen
 import ./platform/platform
 import ./loaders/blend
 import ./util
+from loadable import updateLoadableWorkerThreads
 import arr_ref
 
 export arr_ref
@@ -87,6 +89,7 @@ proc newMyouEngine*(
         opengl_version = default_gl_version,
         opengl_es = default_gl_es,
         glsl_version = "",
+        use_glsl_tone_mapping = true,
     ): MyouEngine =
     ## Creates a Myou Engine instance. You need to call this before you can use
     ## the engine. You also need to call `run <#run,MyouEngine>`_ at the end of
@@ -103,6 +106,10 @@ proc newMyouEngine*(
     # to override it with per-camera exposure settings
     if opengl_es:
         assert opengl_version >= 300, "Minimum supported OpenGL ES version is 3.0"
+    else:
+        assert opengl_version >= 330, "Minimum supported OpenGL version is 3.3"
+
+    if opengl_es or use_glsl_tone_mapping:
         result.tone_mapping_library = dedent """
             float linearrgb_to_srgb(float c){
                 if (c < 0.0031308) return (c < 0.0) ? 0.0 : c * 12.92;
@@ -119,8 +126,8 @@ proc newMyouEngine*(
             #define MYOU_TONE_MAP(x) linearrgb_to_srgb(x)
         """
         result.tone_mapping_function = "MYOU_TONE_MAP"
+        result.use_glsl_tone_mapping = true
     else:
-        assert opengl_version >= 330, "Minimum supported OpenGL version is 3.3"
         result.tone_mapping_library = "\n#define MYOU_TONE_MAP(x) x\n"
     
     echo "assigning renderer"
@@ -159,6 +166,13 @@ proc myou_main_loop*(self: MyouEngine) =
     ##
     ## You usually don't need to call this. Use `run <#run,MyouEngine>`_
     ## instead.
+    updateLoadableWorkerThreads()
+    # TODO: make a table object that can be iterated while changing, e.g. with a
+    # seq and a dirty flag to update the seq
+    if self.new_scenes.len != 0:
+        for name,scene in self.new_scenes.pairs:
+            self.scenes[name] = scene
+        self.new_scenes.clear()
     let time = getmonotime().ticks.float/1000000000
     let delta_seconds = time - last_time
     for _,scene in self.scenes.pairs:

src/platform/generic.nim

@@ -162,3 +162,7 @@ proc myouOnTouch(touch: int32, ending: char, x, y: float32) {.exportc,cdecl.} =
 
 proc platform_switch_screen*(screen: Screen): bool {.inline.} =
     discard
+
+proc myouSetKeyboardVisible*(show: bool) =
+    assert false, "TODO: myouSetKeyboardVisible in generic"
+

src/platform/glfm_wrap.nim

@@ -42,6 +42,7 @@ import std/math
 import std/sequtils
 import std/strformat
 import std/strutils
+import std/unicode
 import ./gl
 import ../screen
 import ../util
@@ -106,8 +107,8 @@ proc glfmMain(w: Window) {.cdecl,exportc,noreturn.} =
 proc make_window*(width, height: int32, title: string): Window =
     if window == nil:
         return
-    window.glfmSetKeyFunc proc(display, keyCode, action, mods: auto): auto {.cdecl.} =
+    window.glfmSetKeyFunc proc(display: ptr GLFMDisplay; keyCode: GLFMKey;
-        # dump (cast[KeyCode](keyCode), keyCode.int, action, mods)
+                    action: GLFMKeyAction; mods: cint): bool {.cdecl.} =
         let key = cast[KeyCode](keyCode)
         let shiftKey = (mods and 1).bool
         let ctrlKey = (mods and 2).bool
@@ -121,6 +122,11 @@ proc make_window*(width, height: int32, title: string): Window =
                 key: key,
             ))
         return true
+    window.glfmSetCharFunc proc(display: ptr GLFMDisplay; utf8: cstring; modifiers: cint) {.cdecl.} =
+        let codepoint = cast[uint32](($utf8).runeAt(0))
+        for cb in display.screen.char_callbacks:
+            cb(codepoint)
+
     window.glfmSetTouchFunc proc (display: ptr GLFMDisplay; touch: cint; phase: GLFMTouchPhase;
                       x: cdouble; y: cdouble): bool {.cdecl.} =
         let screen = window.screen
@@ -166,77 +172,6 @@ proc make_window*(width, height: int32, title: string): Window =
                     screen.last_buttons = screen.last_buttons and not (1'i8 shl btn)
                 screen.last_x = x
                 screen.last_y = y
-
-        # template send_move(x, y: untyped) =
-        #     let buttons = screen.last_buttons
-        #     let left = (buttons and 1).bool
-        #     let middle = (buttons and 2).bool
-        #     let right = (buttons and 4).bool
-        #     let position = vec2(x,y)
-        #     let movement = vec2(x-screen.last_x, y-screen.last_y)
-        #     for cb in screen.mouse_move_callbacks:
-        #         cb(MouseMoveEvent(
-        #             left: left, middle: middle, right: right,
-        #             # shiftKey: shiftKey, ctrlKey: ctrlKey, altKey: altKey, metaKey: metaKey,
-        #             position: position, movement: movement,
-        #         ))
-        #     screen.last_x = x
-        #     screen.last_y = y
-        # template send_btn(pressed1, btn, x, y: untyped) =
-        #     for cb in window.screen.mouse_button_callbacks:
-        #         cb(MouseButtonEvent(
-        #             pressed: pressed1,
-        #             button: cast[MouseButton](btn),
-        #             position: vec2(x,y)
-        #         ))
-        #     if pressed1:
-        #         screen.last_buttons = screen.last_buttons or (1'i8 shl btn)
-        #     else:
-        #         screen.last_buttons = screen.last_buttons and not (1'i8 shl btn)
-        #     screen.last_x = x
-        #     screen.last_y = y
-
-        # template swap_1_2(count: untyped): untyped =
-        #     case count:
-        #         of 1: 2
-        #         of 2: 1
-        #         else: count
-        # if screen.is_touch:
-        #     # mouse emulation
-        #     if phase in [GLFMTouchPhaseMoved, GLFMTouchPhaseHover]:
-        #         let last = screen.touches[0]
-        #         if last[0] == touch:
-        #             let dx = x - last[1]
-        #             let dy = y - last[2]
-        #             screen.touches[0][1] = x
-        #             screen.touches[0][2] = y
-        #             send_move(screen.last_x + dx, screen.last_y + dy)
-        #     elif phase == GLFMTouchPhaseBegan:
-        #         let count = screen.touches.len
-        #         screen.touches.add (touch.int32, x.float, y.float)
-        #         # in glfm buttons 1 and 2 are swapped so mouse emulation
-        #         # through number of touches behave as we expect (no swap_1_2 here)
-        #         let button = count
-        #         if screen.last_buttons == 0:
-        #             send_btn(true, button, x, y)
-        #         else:
-        #             let previous = screen.last_buttons.countTrailingZeroBits
-        #             if previous < count:
-        #                 send_btn(true, button, screen.last_x, screen.last_y)
-        #                 send_btn(false, previous, screen.last_x, screen.last_y)
-        #     else:
-        #         screen.touches.keepItIf it[0] != touch
-        #         if screen.touches.len == 0:
-        #             let previous = screen.last_buttons.countTrailingZeroBits
-        #             send_btn(false, previous, x, y)
-        # else:
-        #     if phase in [GLFMTouchPhaseMoved, GLFMTouchPhaseHover]:
-        #         send_move(x,y)
-        #     else:
-        #         let pressed = phase == GLFMTouchPhaseBegan
-        #         # in glfm buttons 1 and 2 are swapped compared to glfw
-        #         # (but they match DOM)
-        #         send_btn(pressed, touch.swap_1_2, x, y)
         return true
 
     window.glfmSetAppFocusFunc proc(window: Window, focused: bool) {.cdecl.} =
@@ -369,5 +304,5 @@ proc myouAndroidGetExternalDataPath*(): string =
         let activity = cast[ptr array[6, cstring]](window.glfmAndroidGetActivity())
         return $activity[5]
 
-
+proc myouSetKeyboardVisible*(show: bool) =
-
+    window.glfmSetKeyboardVisible(show)

src/platform/glfw_wrap.nim

@@ -88,6 +88,10 @@ proc make_window*(width, height: int32, title: string): Window =
         for cb in window.screen.key_callbacks:
             cb(e)
     )
+    discard glfw.setCharCallback(window, proc(window: Window, codepoint: uint32) {.cdecl.} =
+        for cb in window.screen.char_callbacks:
+            cb(codepoint)
+    )
     discard glfw.setCursorPosCallback(window, proc(window: Window, x, y: float) {.cdecl.} =
         # TODO: when a mouse button is pressed do we need to poll the position
         # with glfw.getCursorPos until released? not needed on linux
@@ -232,3 +236,7 @@ proc start_platform_main_loop*(engine: MyouEngine, main_loop: proc(self: MyouEng
 
 proc myouAndroidGetActivity*(): pointer =
     assert false, "Not using Android"
+
+proc myouSetKeyboardVisible*(show: bool) =
+    discard
+

src/scene.nim

@@ -95,7 +95,7 @@ proc initScene*(self: Scene, engine: MyouEngine, name: string = "Scene",
     while self.name in engine.scenes:
         collision_seq += 1
         self.name = name & "$" & $collision_seq
-    engine.scenes[self.name] = self
+    engine.new_scenes[self.name] = self
     self.mesh_passes.setLen 3
     self.world = newWorld(self)
     self.background_color = vec4(0, 0, 0, 1)

src/types.nim

@@ -81,11 +81,11 @@ type
         renderer*: RenderManager
         tone_mapping_library*: string
         tone_mapping_function*: string
+        use_glsl_tone_mapping*: bool
         loaders_by_ext*: Table[string, seq[proc(e: MyouEngine): Loader]]
-        # TODO: remove this and query it when screen is created
-        width*, height*: int
         glsl_version*: string
         all_framebuffers*: seq[Framebuffer] ## private
+        new_scenes*: Table[string, Scene] ## private
     
     # gameobject.nim
 
@@ -99,7 +99,7 @@ type
     GameObject* = ref object of RootObj
         # TODO: remove otype
         otype*: ObjectType ## private
-        engine*: MyouEngine ## private
+        engine* {.cursor.}: MyouEngine ## private
         debug*: bool ## private
         position*: Vec3
         rotation*: Quat
@@ -111,12 +111,12 @@ type
         object_color*: Vec4
         # alpha*: float
         matrix_parent_inverse*: Mat4
-        scene*: Scene
+        scene* {.cursor.}: Scene
         source_scene_name*: string
         # data_dir*: string
         # dupli_group*: unknown
         visible*: bool
-        parent*: GameObject
+        parent* {.cursor.}: GameObject
         children*: seq[GameObject]
         auto_update_matrix*: bool
         world_matrix*: Mat4
@@ -159,7 +159,8 @@ type
     
     # most fields should be private
     MeshData* = ref object
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
+        # TODO: seq of {.cursor.}s?
         users*: seq[GameObject] ## private
         hash*: string
         varrays*: seq[ArrRef[float32]]
@@ -288,8 +289,8 @@ type
     # shadows
 
     ShadowManager* = ref object of RootObj
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
-        light*: Light
+        light* {.cursor.}: Light
         framebuffer*: Framebuffer
         texture*: Texture
         sampler_type*: string
@@ -331,7 +332,7 @@ type
         padding3: float32
         
     Scene* = ref object
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
         name*: string
         enabled*: bool
         children*: seq[GameObject]
@@ -407,7 +408,7 @@ type
         viewport_size*, viewport_size_inv*: Vec2
 
     RenderManager* = ref object
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
         initialized*: bool
         # temporary_framebuffers*: Table[int, ByteFramebuffer]
         render_tick*: int
@@ -493,7 +494,7 @@ type
     # ubo.nim
 
     UBO* = ref object
-        renderer*: RenderManager
+        renderer* {.cursor.}: RenderManager
         name*: string
         size32*: int
         byte_storage*: ArrRef[byte]
@@ -530,7 +531,7 @@ type
         EsPrecisionHigh
 
     Material* = ref object
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
         name*: string
         textures*: OrderedTable[string, Texture]
         ubos*: seq[UBO]
@@ -561,7 +562,7 @@ type
     
     # TODO: choose better names
     Shader* = ref object
-        engine*: MyouEngine # do we need this?
+        engine* {.cursor.}: MyouEngine # do we need this?
         id*: int
         program*: GLuint
         material*: Material # do we need this?
@@ -640,7 +641,7 @@ type
         tile_size*: Vec2
 
     Texture* = ref object of RootObj
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
         name*: string
         storage*: TextureStorage ## private
         loaded*: bool
@@ -661,7 +662,7 @@ type
         DepthTexture
     
     Framebuffer* = ref object of RootObj
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
         width*, height*: int
         format*: TextureFormat
         depth_type*: FramebufferDepthType
@@ -753,7 +754,7 @@ type
     # loader_base.nim
 
     Loader* = ref object of RootObj
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
         shader_library*: string
         shader_textures*: Table[string, Texture]
         # on_destroy*: OnDestroy
@@ -778,7 +779,7 @@ type
     Body* = ref object
 
     Screen* = ref object of RootObj
-        engine*: MyouEngine
+        engine* {.cursor.}: MyouEngine
         width*, height*: int32
         orientation*: int8
         viewports*: seq[Viewport]
@@ -789,6 +790,7 @@ type
         last_x*, last_y*: float ## private
         last_buttons*, last_mods*: int8 ## private
         key_callbacks*: seq[proc(event: KeyEvent)] ## private
+        char_callbacks*: seq[proc(codepoint: uint32)] ## private
         mouse_move_callbacks*: seq[proc(event: MouseMoveEvent)] ## private
         mouse_button_callbacks*: seq[proc(event: MouseButtonEvent)] ## private
         mouse_wheel_callbacks*: seq[proc(event: MouseWheelEvent)] ## private