myou-engine/libs/loadable/loadable.nim

# The contents of this file are subject to the Common Public Attribution License
# Version 1.0 (the “License”); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# https://myou.dev/licenses/LICENSE-CPAL. The License is based on the Mozilla
# Public License Version 1.1 but Sections 14 and 15 have been added to cover use
# of software over a computer network and provide for limited attribution for
# the Original Developer. In addition, Exhibit A has been modified to be
# consistent with Exhibit B.
#
# Software distributed under the License is distributed on an “AS IS” basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
# the specific language governing rights and limitations under the License.
#
# The Original Code is Myou Engine.
#
# the Original Developer is the Initial Developer.
#
# The Initial Developer of the Original Code is the Myou Engine developers.
# All portions of the code written by the Myou Engine developers are Copyright
# (c) 2024. All Rights Reserved.
#
# Alternatively, the contents of this file may be used under the terms of the
# GNU Affero General Public License version 3 (the [AGPL-3] License), in which
# case the provisions of [AGPL-3] License are applicable instead of those above.
#
# If you wish to allow use of your version of this file only under the terms of
# the [AGPL-3] License and not to allow others to use your version of this file
# under the CPAL, indicate your decision by deleting the provisions above and
# replace them with the notice and other provisions required by the [AGPL-3]
# License. If you do not delete the provisions above, a recipient may use your
# version of this file under either the CPAL or the [AGPL-3] License.


## A LoadableResource is similar to a JS promise or a Future, except that you can
## store it to load it any number of times, and instead of providing an object,
## you get a pointer, a length, and a function to tell it you're done with it.
## You cannot have more than one callback, but you can obtain the results of many
## with `loadAll` or get a callback when many are done with `allDone`
## 
## TODO IMPORTANT: probably not thread safe, call all functions in the same thread
##                 if you use loadAll and allDone
## 
## TODO: investigate the use of signals instead of callbacks

{.hint[ConvFromXtoItselfNotNeeded]:off.}

import arr_ref # for SliceMem

type LoadableResourceStatus* = enum
    NotStarted
    Started
    Finished
    Error

# type Result = tuple[ok: bool, err: string, data: SliceMem[byte]]

type LoadableResource* = ref object of RootObj
    status: LoadableResourceStatus
    start_func: proc(self: LoadableResource) {.closure.}
    onload_func: proc(ok: bool, err: string, data: SliceMem[byte]) {.closure.}
    cancel_func: proc() {.closure.}
    str*: proc(): string
    use_threads: bool
    # result: ref Result # only to be used with loadAll

proc newLoadableResource*[T: LoadableResource](
        start: proc(self: LoadableResource),
        str: proc(): string = nil,
        use_threads = false,
    ): T =
    new(result)
    result.start_func = start
    result.str = str
    if str == nil:
        result.str = proc(): string = ""
    result.use_threads = use_threads

# main -> thread  channels
var to_start: Channel[LoadableResource]
# main <- thread  channels
var to_return: Channel[(LoadableResource, bool, string, SliceMem[byte])]

proc start*[T: LoadableResource](self: T) =
    self.status = Started
    if self.use_threads:
        to_start.send self
    else:
        self.start_func(self)

proc `onload=`*[T: LoadableResource](self: T, onload_func: proc(ok: bool, err: string, data: SliceMem[byte])) =
    self.onload_func = onload_func

proc onload*[T: LoadableResource](self: T, ok: bool, err: string, data = SliceMem[byte]()) =
    if self.status == Started:
        self.status = if ok: Finished else: Error
        # if self.result != nil:
        #     self.result[] = (ok, err, data)
        if self.onload_func != nil:
            if self.use_threads:
                to_return.send((self.LoadableResource, ok, err, data))
            else:
                self.onload_func(ok, err, data)

proc cancel*[T: LoadableResource](self: T) =
    if self.status != Started:
        return
    if self.cancel_func != nil:
        self.cancel_func()
        self.status = NotStarted

# proc allDone*(list: seq[LoadableResource], done: proc()) =


# proc loadAll*(list: seq[LoadableResource], onload: proc(all_ok: bool, result: seq[Result])) =
#     for i,res in list:
#         if not defined(release):
#             assert res.status == NotStarted, "All resources in the list must be not started"
#             assert res.onload_func == nil, "All resources in the list must not have onload"
#         new(res.result)
#         res.start()
#     list.allDone, proc() =
#         var results = newSeqOfCap[Result](list.len)
#         for res in list:
#             results.add res.result[]
#             res.result = nil

var worker: Thread[void]
proc workerThreadProc() {.thread.} =
    while true:
        let res = to_start.recv()
        if res == nil:
            break
        cast[proc(self: LoadableResource) {.gcsafe.}](res.start_func)(res)

worker.createThread(workerThreadProc)
to_start.open()
to_return.open()

proc updateLoadableWorkerThreads*() =
    while true:
        let tried = to_return.tryRecv()
        if not tried.dataAvailable:
            break
        let (res, ok, err, data) = tried.msg
        res.onload_func(ok, err, data)

proc terminateLoadableWorkerThreads*() =
    # TODO: test this
    to_start.send(nil.LoadableResource)
    worker.joinThread()

type Fetch* = ref object of LoadableResource
    custom: pointer

when not defined(onlyLocalFiles):
    when defined(emscripten):
        type
            FetchAttribute = enum
                LoadToMemory = 1
                StreamData = 2
                PersistFile = 4
                Append = 8
                Replace = 16
                NoDownload = 32
                Synchronous = 64
                Waitable = 128
            emscripten_fetch_attr_t {.importc,nodecl,header:"emscripten/fetch.h".} = object
                requestMethod: array[32, char]
                userData: pointer
                onsuccess: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
                onerror: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
                onprogress: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
                onreadystatechange: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
                attributes: FetchAttribute # uint32
                timeoutMSecs: uint32
                withCredentials: bool # actually int32
                destinationPath: cstring
                userName: cstring
                password: cstring
                requestHeaders: ptr UncheckedArray[cstring]
                overriddenMimeType: cstring
                requestData: cstring
                requestDataSize: uint
            emscripten_fetch_t {.importc,nodecl.} = object
                id: uint32
                userData: pointer
                url: cstring
                data: pointer
                numBytes: uint64
                dataOffset: uint64
                totalBytes: uint64
                readyState: uint16
                status: uint16
                statusText: array[64, char]
            EmscriptenResult = enum
                TimedOut = -8
                NoData = -7
                Failed = -6
                InvalidParam = -5
                UnknownTarget = -4
                InvalidTarget = -3
                FailedNotDeferred = -2
                NotSupported = -1
                Success = 0
                Deferred = 1
        const EMSCRIPTEN_RESULT_STRINGS = [
                "TimedOut","NoData","Failed","InvalidParam",
                "UnknownTarget","InvalidTarget","FailedNotDeferred",
                "NotSupported","Success","Deferred"]
        proc emscripten_fetch_attr_init(fetch_attr: var emscripten_fetch_attr_t)  {.importc,nodecl.}
        proc emscripten_fetch(fetch_attr: var emscripten_fetch_attr_t, url: cstring): ptr emscripten_fetch_t {.importc,nodecl.}
        proc emscripten_fetch_wait(fetch: ptr emscripten_fetch_t, timeoutMSecs: float): EmscriptenResult  {.importc,nodecl.}
        proc emscripten_fetch_close(fetch: ptr emscripten_fetch_t): EmscriptenResult  {.importc,nodecl.}
        proc emscripten_fetch_get_response_headers_length(fetch: ptr emscripten_fetch_t): uint  {.importc,nodecl.}
        proc emscripten_fetch_get_response_headers(fetch: ptr emscripten_fetch_t, dst: cstring , dstSizeBytes: uint): uint  {.importc,nodecl.}
        proc emscripten_fetch_unpack_response_headers(headersString: cstring): ptr UncheckedArray[cstring] {.importc,nodecl.}
        proc emscripten_fetch_free_unpacked_response_headers(unpackedHeaders: ptr UncheckedArray[cstring]) {.importc,nodecl.}

    else:
        import std/httpclient

import std/memfiles
import std/strutils
import std/strformat

func escapeUTF8*(s: string): string =
    result = newStringOfCap(s.len + 5)
    for i,c in s:
        if c > ' ' and c <= '~':
            result &= c
        else:
            result &= '%' & c.byte.toHex

# TODO: automatically disable threads when not in main thread

proc loadUri*(
        uri: string, 
        onload_func: proc(ok: bool, err: string, data: SliceMem[byte]) = nil,
        range = (-1,-1),
        auto_start = true,
        use_threads = true,
    ): Fetch {.discardable.} =

    echo "fetching ", uri

    var start_func: proc(self: LoadableResource)
    var self: Fetch
    var uri = uri
    var use_threads = use_threads
    when not defined(onlyLocalFiles):
        when defined(emscripten):
            const is_remote = true
        else:
            let is_remote = uri.startswith("http://") or uri.startswith("https://")
    
        if is_remote:
            uri = uri.escapeUTF8
            when defined(emscripten):
                use_threads = false # API is already threaded
                start_func = proc(self: LoadableResource) =
                    var attr: emscripten_fetch_attr_t
                    emscripten_fetch_attr_init(attr)
                    attr.userData = cast[pointer](self)
                    attr.requestMethod.addr.copyMem("GET".cstring, 4)
                    attr.attributes = LoadToMemory
                    attr.onsuccess = proc(fetch: ptr emscripten_fetch_t) {.cdecl.} =
                        var self = cast[Fetch](fetch.userData)
                        self.custom = fetch.pointer
                        self.onload(true, "", newSliceMem(fetch.data, fetch.numBytes.int, proc() =
                            emscripten_fetch_close(fetch)
                        ))
                    attr.onerror = proc(fetch: ptr emscripten_fetch_t) {.cdecl.} =
                        var self = cast[Fetch](fetch.userData)
                        let err_msg = $fetch.statusText.addr.cstring
                        let err = emscripten_fetch_close(fetch)
                        # self.onload(false, &"Error: {err_msg} ({EMSCRIPTEN_RESULT_STRINGS[err.int + 8]})", nil, 0)
                        self.onload(false, &"Error fetching {fetch.url}: {err_msg}")
                    discard emscripten_fetch(attr, uri.cstring)
            else:
                var client = newHttpClient()
                var response: string
                start_func = proc(self: LoadableResource) =
                    let self = cast[Fetch](self)
                    var ok = false
                    try:
                        response = client.getContent(uri)
                        ok = true
                    except:
                        self.onload(false, &"Error fetching {uri}: {getCurrentExceptionMsg()}")
                    if ok:
                        self.onload(true, "", newSliceMem(response[0].addr.pointer, response.len, proc() =
                            discard response
                        ))

    if not is_remote:
        use_threads = false # TODO: detect networked file system?
                            # TODO: also test if there's better perf in local
        start_func = proc(self: LoadableResource) =
            try:
                var memfile = memfiles.open(uri, mode=fmRead)
                self.onload(true, "", newSliceMem(memfile.mem, memfile.size, proc() =
                    try:
                        memfile.close()
                    # this should never happen but destructors require it
                    except OSError: discard
                ))
            except OSError:
                self.onload(false, "Could not open file: " & uri)
    proc str(): string = uri
    self = newLoadableResource[Fetch](start_func, str, use_threads=use_threads)
    self.onload_func = onload_func
    if auto_start:
        start(self)
    return self