# The contents of this file are subject to the Common Public Attribution License # Version 1.0 (the “License”); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # https://myou.dev/licenses/LICENSE-CPAL. The License is based on the Mozilla # Public License Version 1.1 but Sections 14 and 15 have been added to cover use # of software over a computer network and provide for limited attribution for # the Original Developer. In addition, Exhibit A has been modified to be # consistent with Exhibit B. # # Software distributed under the License is distributed on an “AS IS” basis, # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for # the specific language governing rights and limitations under the License. # # The Original Code is Myou Engine. # # the Original Developer is the Initial Developer. # # The Initial Developer of the Original Code is the Myou Engine developers. # All portions of the code written by the Myou Engine developers are Copyright # (c) 2024. All Rights Reserved. # # Alternatively, the contents of this file may be used under the terms of the # GNU Affero General Public License version 3 (the [AGPL-3] License), in which # case the provisions of [AGPL-3] License are applicable instead of those above. # # If you wish to allow use of your version of this file only under the terms of # the [AGPL-3] License and not to allow others to use your version of this file # under the CPAL, indicate your decision by deleting the provisions above and # replace them with the notice and other provisions required by the [AGPL-3] # License. If you do not delete the provisions above, a recipient may use your # version of this file under either the CPAL or the [AGPL-3] License. ## A LoadableResource is similar to a JS promise or a Future, except that you can ## store it to load it any number of times, and instead of providing an object, ## you get a pointer, a length, and a function to tell it you're done with it. ## You cannot have more than one callback, but you can obtain the results of many ## with `loadAll` or get a callback when many are done with `allDone` ## ## TODO IMPORTANT: probably not thread safe, call all functions in the same thread ## if you use loadAll and allDone ## ## TODO: investigate the use of signals instead of callbacks {.hint[ConvFromXtoItselfNotNeeded]:off.} import arr_ref # for SliceMem type LoadableResourceStatus* = enum NotStarted Started Finished Error # type Result = tuple[ok: bool, err: string, data: SliceMem[byte]] type LoadableResource* = ref object of RootObj status: LoadableResourceStatus start_func: proc(self: LoadableResource) {.closure.} onload_func: proc(ok: bool, err: string, data: SliceMem[byte]) {.closure.} cancel_func: proc() {.closure.} str*: proc(): string use_threads: bool # result: ref Result # only to be used with loadAll proc newLoadableResource*[T: LoadableResource]( start: proc(self: LoadableResource), str: proc(): string = nil, use_threads = false, ): T = new(result) result.start_func = start result.str = str if str == nil: result.str = proc(): string = "" result.use_threads = use_threads # main -> thread channels var to_start: Channel[LoadableResource] # main <- thread channels var to_return: Channel[(LoadableResource, bool, string, SliceMem[byte])] proc start*[T: LoadableResource](self: T) = self.status = Started if self.use_threads: to_start.send self else: self.start_func(self) proc `onload=`*[T: LoadableResource](self: T, onload_func: proc(ok: bool, err: string, data: SliceMem[byte])) = self.onload_func = onload_func proc onload*[T: LoadableResource](self: T, ok: bool, err: string, data = SliceMem[byte]()) = if self.status == Started: self.status = if ok: Finished else: Error # if self.result != nil: # self.result[] = (ok, err, data) if self.onload_func != nil: if self.use_threads: to_return.send((self.LoadableResource, ok, err, data)) else: self.onload_func(ok, err, data) proc cancel*[T: LoadableResource](self: T) = if self.status != Started: return if self.cancel_func != nil: self.cancel_func() self.status = NotStarted # proc allDone*(list: seq[LoadableResource], done: proc()) = # proc loadAll*(list: seq[LoadableResource], onload: proc(all_ok: bool, result: seq[Result])) = # for i,res in list: # if not defined(release): # assert res.status == NotStarted, "All resources in the list must be not started" # assert res.onload_func == nil, "All resources in the list must not have onload" # new(res.result) # res.start() # list.allDone, proc() = # var results = newSeqOfCap[Result](list.len) # for res in list: # results.add res.result[] # res.result = nil var worker: Thread[void] proc workerThreadProc() {.thread.} = while true: let res = to_start.recv() if res == nil: break cast[proc(self: LoadableResource) {.gcsafe.}](res.start_func)(res) worker.createThread(workerThreadProc) to_start.open() to_return.open() proc updateLoadableWorkerThreads*() = while true: let tried = to_return.tryRecv() if not tried.dataAvailable: break let (res, ok, err, data) = tried.msg res.onload_func(ok, err, data) proc terminateLoadableWorkerThreads*() = # TODO: test this to_start.send(nil.LoadableResource) worker.joinThread() type Fetch* = ref object of LoadableResource custom: pointer when not defined(onlyLocalFiles): when defined(emscripten): type FetchAttribute = enum LoadToMemory = 1 StreamData = 2 PersistFile = 4 Append = 8 Replace = 16 NoDownload = 32 Synchronous = 64 Waitable = 128 emscripten_fetch_attr_t {.importc,nodecl,header:"emscripten/fetch.h".} = object requestMethod: array[32, char] userData: pointer onsuccess: proc(fetch: ptr emscripten_fetch_t) {.cdecl.} onerror: proc(fetch: ptr emscripten_fetch_t) {.cdecl.} onprogress: proc(fetch: ptr emscripten_fetch_t) {.cdecl.} onreadystatechange: proc(fetch: ptr emscripten_fetch_t) {.cdecl.} attributes: FetchAttribute # uint32 timeoutMSecs: uint32 withCredentials: bool # actually int32 destinationPath: cstring userName: cstring password: cstring requestHeaders: ptr UncheckedArray[cstring] overriddenMimeType: cstring requestData: cstring requestDataSize: uint emscripten_fetch_t {.importc,nodecl.} = object id: uint32 userData: pointer url: cstring data: pointer numBytes: uint64 dataOffset: uint64 totalBytes: uint64 readyState: uint16 status: uint16 statusText: array[64, char] EmscriptenResult = enum TimedOut = -8 NoData = -7 Failed = -6 InvalidParam = -5 UnknownTarget = -4 InvalidTarget = -3 FailedNotDeferred = -2 NotSupported = -1 Success = 0 Deferred = 1 const EMSCRIPTEN_RESULT_STRINGS = [ "TimedOut","NoData","Failed","InvalidParam", "UnknownTarget","InvalidTarget","FailedNotDeferred", "NotSupported","Success","Deferred"] proc emscripten_fetch_attr_init(fetch_attr: var emscripten_fetch_attr_t) {.importc,nodecl.} proc emscripten_fetch(fetch_attr: var emscripten_fetch_attr_t, url: cstring): ptr emscripten_fetch_t {.importc,nodecl.} proc emscripten_fetch_wait(fetch: ptr emscripten_fetch_t, timeoutMSecs: float): EmscriptenResult {.importc,nodecl.} proc emscripten_fetch_close(fetch: ptr emscripten_fetch_t): EmscriptenResult {.importc,nodecl.} proc emscripten_fetch_get_response_headers_length(fetch: ptr emscripten_fetch_t): uint {.importc,nodecl.} proc emscripten_fetch_get_response_headers(fetch: ptr emscripten_fetch_t, dst: cstring , dstSizeBytes: uint): uint {.importc,nodecl.} proc emscripten_fetch_unpack_response_headers(headersString: cstring): ptr UncheckedArray[cstring] {.importc,nodecl.} proc emscripten_fetch_free_unpacked_response_headers(unpackedHeaders: ptr UncheckedArray[cstring]) {.importc,nodecl.} else: import std/httpclient import std/memfiles import std/strutils import std/strformat func escapeUTF8*(s: string): string = result = newStringOfCap(s.len + 5) for i,c in s: if c > ' ' and c <= '~': result &= c else: result &= '%' & c.byte.toHex # TODO: automatically disable threads when not in main thread proc loadUri*( uri: string, onload_func: proc(ok: bool, err: string, data: SliceMem[byte]) = nil, range = (-1,-1), auto_start = true, use_threads = true, ): Fetch {.discardable.} = echo "fetching ", uri var start_func: proc(self: LoadableResource) var self: Fetch var uri = uri var use_threads = use_threads when not defined(onlyLocalFiles): when defined(emscripten): const is_remote = true else: let is_remote = uri.startswith("http://") or uri.startswith("https://") if is_remote: uri = uri.escapeUTF8 when defined(emscripten): use_threads = false # API is already threaded start_func = proc(self: LoadableResource) = var attr: emscripten_fetch_attr_t emscripten_fetch_attr_init(attr) attr.userData = cast[pointer](self) attr.requestMethod.addr.copyMem("GET".cstring, 4) attr.attributes = LoadToMemory attr.onsuccess = proc(fetch: ptr emscripten_fetch_t) {.cdecl.} = var self = cast[Fetch](fetch.userData) self.custom = fetch.pointer self.onload(true, "", newSliceMem(fetch.data, fetch.numBytes.int, proc() = emscripten_fetch_close(fetch) )) attr.onerror = proc(fetch: ptr emscripten_fetch_t) {.cdecl.} = var self = cast[Fetch](fetch.userData) let err_msg = $fetch.statusText.addr.cstring let err = emscripten_fetch_close(fetch) # self.onload(false, &"Error: {err_msg} ({EMSCRIPTEN_RESULT_STRINGS[err.int + 8]})", nil, 0) self.onload(false, &"Error fetching {fetch.url}: {err_msg}") discard emscripten_fetch(attr, uri.cstring) else: var client = newHttpClient() var response: string start_func = proc(self: LoadableResource) = let self = cast[Fetch](self) var ok = false try: response = client.getContent(uri) ok = true except: self.onload(false, &"Error fetching {uri}: {getCurrentExceptionMsg()}") if ok: self.onload(true, "", newSliceMem(response[0].addr.pointer, response.len, proc() = discard response )) if not is_remote: use_threads = false # TODO: detect networked file system? # TODO: also test if there's better perf in local start_func = proc(self: LoadableResource) = try: var memfile = memfiles.open(uri, mode=fmRead) self.onload(true, "", newSliceMem(memfile.mem, memfile.size, proc() = try: memfile.close() # this should never happen but destructors require it except OSError: discard )) except OSError: self.onload(false, "Could not open file: " & uri) proc str(): string = uri self = newLoadableResource[Fetch](start_func, str, use_threads=use_threads) self.onload_func = onload_func if auto_start: start(self) return self