310 lines
12 KiB
Nim
310 lines
12 KiB
Nim
# The contents of this file are subject to the Common Public Attribution License
|
|
# Version 1.0 (the “License”); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
# https://myou.dev/licenses/LICENSE-CPAL. The License is based on the Mozilla
|
|
# Public License Version 1.1 but Sections 14 and 15 have been added to cover use
|
|
# of software over a computer network and provide for limited attribution for
|
|
# the Original Developer. In addition, Exhibit A has been modified to be
|
|
# consistent with Exhibit B.
|
|
#
|
|
# Software distributed under the License is distributed on an “AS IS” basis,
|
|
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
|
|
# the specific language governing rights and limitations under the License.
|
|
#
|
|
# The Original Code is Myou Engine.
|
|
#
|
|
# the Original Developer is the Initial Developer.
|
|
#
|
|
# The Initial Developer of the Original Code is the Myou Engine developers.
|
|
# All portions of the code written by the Myou Engine developers are Copyright
|
|
# (c) 2024. All Rights Reserved.
|
|
#
|
|
# Alternatively, the contents of this file may be used under the terms of the
|
|
# GNU Affero General Public License version 3 (the [AGPL-3] License), in which
|
|
# case the provisions of [AGPL-3] License are applicable instead of those above.
|
|
#
|
|
# If you wish to allow use of your version of this file only under the terms of
|
|
# the [AGPL-3] License and not to allow others to use your version of this file
|
|
# under the CPAL, indicate your decision by deleting the provisions above and
|
|
# replace them with the notice and other provisions required by the [AGPL-3]
|
|
# License. If you do not delete the provisions above, a recipient may use your
|
|
# version of this file under either the CPAL or the [AGPL-3] License.
|
|
|
|
|
|
## A LoadableResource is similar to a JS promise or a Future, except that you can
|
|
## store it to load it any number of times, and instead of providing an object,
|
|
## you get a pointer, a length, and a function to tell it you're done with it.
|
|
## You cannot have more than one callback, but you can obtain the results of many
|
|
## with `loadAll` or get a callback when many are done with `allDone`
|
|
##
|
|
## TODO IMPORTANT: probably not thread safe, call all functions in the same thread
|
|
## if you use loadAll and allDone
|
|
##
|
|
## TODO: investigate the use of signals instead of callbacks
|
|
|
|
{.hint[ConvFromXtoItselfNotNeeded]:off.}
|
|
|
|
import arr_ref # for SliceMem
|
|
|
|
type LoadableResourceStatus* = enum
|
|
NotStarted
|
|
Started
|
|
Finished
|
|
Error
|
|
|
|
# type Result = tuple[ok: bool, err: string, data: SliceMem[byte]]
|
|
|
|
type LoadableResource* = ref object of RootObj
|
|
status: LoadableResourceStatus
|
|
start_func: proc(self: LoadableResource) {.closure.}
|
|
onload_func: proc(ok: bool, err: string, data: SliceMem[byte]) {.closure.}
|
|
cancel_func: proc() {.closure.}
|
|
str*: proc(): string
|
|
use_threads: bool
|
|
# result: ref Result # only to be used with loadAll
|
|
|
|
proc newLoadableResource*[T: LoadableResource](
|
|
start: proc(self: LoadableResource),
|
|
str: proc(): string = nil,
|
|
use_threads = false,
|
|
): T =
|
|
new(result)
|
|
result.start_func = start
|
|
result.str = str
|
|
if str == nil:
|
|
result.str = proc(): string = ""
|
|
result.use_threads = use_threads
|
|
|
|
# main -> thread channels
|
|
var to_start: Channel[LoadableResource]
|
|
# main <- thread channels
|
|
var to_return: Channel[(LoadableResource, bool, string, SliceMem[byte])]
|
|
|
|
proc start*[T: LoadableResource](self: T) =
|
|
self.status = Started
|
|
if self.use_threads:
|
|
to_start.send self
|
|
else:
|
|
self.start_func(self)
|
|
|
|
proc `onload=`*[T: LoadableResource](self: T, onload_func: proc(ok: bool, err: string, data: SliceMem[byte])) =
|
|
self.onload_func = onload_func
|
|
|
|
proc onload*[T: LoadableResource](self: T, ok: bool, err: string, data = SliceMem[byte]()) =
|
|
if self.status == Started:
|
|
self.status = if ok: Finished else: Error
|
|
# if self.result != nil:
|
|
# self.result[] = (ok, err, data)
|
|
if self.onload_func != nil:
|
|
if self.use_threads:
|
|
to_return.send((self.LoadableResource, ok, err, data))
|
|
else:
|
|
self.onload_func(ok, err, data)
|
|
|
|
proc cancel*[T: LoadableResource](self: T) =
|
|
if self.status != Started:
|
|
return
|
|
if self.cancel_func != nil:
|
|
self.cancel_func()
|
|
self.status = NotStarted
|
|
|
|
# proc allDone*(list: seq[LoadableResource], done: proc()) =
|
|
|
|
|
|
# proc loadAll*(list: seq[LoadableResource], onload: proc(all_ok: bool, result: seq[Result])) =
|
|
# for i,res in list:
|
|
# if not defined(release):
|
|
# assert res.status == NotStarted, "All resources in the list must be not started"
|
|
# assert res.onload_func == nil, "All resources in the list must not have onload"
|
|
# new(res.result)
|
|
# res.start()
|
|
# list.allDone, proc() =
|
|
# var results = newSeqOfCap[Result](list.len)
|
|
# for res in list:
|
|
# results.add res.result[]
|
|
# res.result = nil
|
|
|
|
var worker: Thread[void]
|
|
proc workerThreadProc() {.thread.} =
|
|
while true:
|
|
let res = to_start.recv()
|
|
if res == nil:
|
|
break
|
|
cast[proc(self: LoadableResource) {.gcsafe.}](res.start_func)(res)
|
|
|
|
worker.createThread(workerThreadProc)
|
|
to_start.open()
|
|
to_return.open()
|
|
|
|
proc updateLoadableWorkerThreads*() =
|
|
while true:
|
|
let tried = to_return.tryRecv()
|
|
if not tried.dataAvailable:
|
|
break
|
|
let (res, ok, err, data) = tried.msg
|
|
res.onload_func(ok, err, data)
|
|
|
|
proc terminateLoadableWorkerThreads*() =
|
|
# TODO: test this
|
|
to_start.send(nil.LoadableResource)
|
|
worker.joinThread()
|
|
|
|
type Fetch* = ref object of LoadableResource
|
|
custom: pointer
|
|
|
|
when not defined(onlyLocalFiles):
|
|
when defined(emscripten):
|
|
type
|
|
FetchAttribute = enum
|
|
LoadToMemory = 1
|
|
StreamData = 2
|
|
PersistFile = 4
|
|
Append = 8
|
|
Replace = 16
|
|
NoDownload = 32
|
|
Synchronous = 64
|
|
Waitable = 128
|
|
emscripten_fetch_attr_t {.importc,nodecl,header:"emscripten/fetch.h".} = object
|
|
requestMethod: array[32, char]
|
|
userData: pointer
|
|
onsuccess: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
|
|
onerror: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
|
|
onprogress: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
|
|
onreadystatechange: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
|
|
attributes: FetchAttribute # uint32
|
|
timeoutMSecs: uint32
|
|
withCredentials: bool # actually int32
|
|
destinationPath: cstring
|
|
userName: cstring
|
|
password: cstring
|
|
requestHeaders: ptr UncheckedArray[cstring]
|
|
overriddenMimeType: cstring
|
|
requestData: cstring
|
|
requestDataSize: uint
|
|
emscripten_fetch_t {.importc,nodecl.} = object
|
|
id: uint32
|
|
userData: pointer
|
|
url: cstring
|
|
data: pointer
|
|
numBytes: uint64
|
|
dataOffset: uint64
|
|
totalBytes: uint64
|
|
readyState: uint16
|
|
status: uint16
|
|
statusText: array[64, char]
|
|
EmscriptenResult = enum
|
|
TimedOut = -8
|
|
NoData = -7
|
|
Failed = -6
|
|
InvalidParam = -5
|
|
UnknownTarget = -4
|
|
InvalidTarget = -3
|
|
FailedNotDeferred = -2
|
|
NotSupported = -1
|
|
Success = 0
|
|
Deferred = 1
|
|
const EMSCRIPTEN_RESULT_STRINGS = [
|
|
"TimedOut","NoData","Failed","InvalidParam",
|
|
"UnknownTarget","InvalidTarget","FailedNotDeferred",
|
|
"NotSupported","Success","Deferred"]
|
|
proc emscripten_fetch_attr_init(fetch_attr: var emscripten_fetch_attr_t) {.importc,nodecl.}
|
|
proc emscripten_fetch(fetch_attr: var emscripten_fetch_attr_t, url: cstring): ptr emscripten_fetch_t {.importc,nodecl.}
|
|
proc emscripten_fetch_wait(fetch: ptr emscripten_fetch_t, timeoutMSecs: float): EmscriptenResult {.importc,nodecl.}
|
|
proc emscripten_fetch_close(fetch: ptr emscripten_fetch_t): EmscriptenResult {.importc,nodecl.}
|
|
proc emscripten_fetch_get_response_headers_length(fetch: ptr emscripten_fetch_t): uint {.importc,nodecl.}
|
|
proc emscripten_fetch_get_response_headers(fetch: ptr emscripten_fetch_t, dst: cstring , dstSizeBytes: uint): uint {.importc,nodecl.}
|
|
proc emscripten_fetch_unpack_response_headers(headersString: cstring): ptr UncheckedArray[cstring] {.importc,nodecl.}
|
|
proc emscripten_fetch_free_unpacked_response_headers(unpackedHeaders: ptr UncheckedArray[cstring]) {.importc,nodecl.}
|
|
|
|
else:
|
|
import std/httpclient
|
|
|
|
import std/memfiles
|
|
import std/strutils
|
|
import std/strformat
|
|
|
|
func escapeUTF8*(s: string): string =
|
|
result = newStringOfCap(s.len + 5)
|
|
for i,c in s:
|
|
if c > ' ' and c <= '~':
|
|
result &= c
|
|
else:
|
|
result &= '%' & c.byte.toHex
|
|
|
|
proc loadUri*(
|
|
uri: string,
|
|
onload_func: proc(ok: bool, err: string, data: SliceMem[byte]) = nil,
|
|
range = (-1,-1),
|
|
auto_start = true,
|
|
): Fetch {.discardable.} =
|
|
|
|
echo "fetching ", uri
|
|
|
|
var start_func: proc(self: LoadableResource)
|
|
var self: Fetch
|
|
var uri = uri
|
|
var use_threads = false
|
|
when not defined(onlyLocalFiles):
|
|
when defined(emscripten):
|
|
const is_remote = true
|
|
else:
|
|
let is_remote = uri.startswith("http://") or uri.startswith("https://")
|
|
|
|
if is_remote:
|
|
uri = uri.escapeUTF8
|
|
when defined(emscripten):
|
|
start_func = proc(self: LoadableResource) =
|
|
var attr: emscripten_fetch_attr_t
|
|
emscripten_fetch_attr_init(attr)
|
|
attr.userData = cast[pointer](self)
|
|
attr.requestMethod.addr.copyMem("GET".cstring, 4)
|
|
attr.attributes = LoadToMemory
|
|
attr.onsuccess = proc(fetch: ptr emscripten_fetch_t) {.cdecl.} =
|
|
var self = cast[Fetch](fetch.userData)
|
|
self.custom = fetch.pointer
|
|
self.onload(true, "", newSliceMem(fetch.data, fetch.numBytes.int, proc() =
|
|
emscripten_fetch_close(fetch)
|
|
))
|
|
attr.onerror = proc(fetch: ptr emscripten_fetch_t) {.cdecl.} =
|
|
var self = cast[Fetch](fetch.userData)
|
|
let err_msg = $fetch.statusText.addr.cstring
|
|
let err = emscripten_fetch_close(fetch)
|
|
# self.onload(false, &"Error: {err_msg} ({EMSCRIPTEN_RESULT_STRINGS[err.int + 8]})", nil, 0)
|
|
self.onload(false, &"Error fetching {fetch.url}: {err_msg}")
|
|
discard emscripten_fetch(attr, uri.cstring)
|
|
else:
|
|
use_threads = true
|
|
var client = newHttpClient()
|
|
var response: string
|
|
start_func = proc(self: LoadableResource) =
|
|
let self = cast[Fetch](self)
|
|
var ok = false
|
|
try:
|
|
response = client.getContent(uri)
|
|
ok = true
|
|
except:
|
|
self.onload(false, &"Error fetching {uri}: {getCurrentExceptionMsg()}")
|
|
if ok:
|
|
self.onload(true, "", newSliceMem(response[0].addr.pointer, response.len, proc() =
|
|
discard response
|
|
))
|
|
|
|
if not is_remote:
|
|
start_func = proc(self: LoadableResource) =
|
|
try:
|
|
var memfile = memfiles.open(uri, mode=fmRead)
|
|
self.onload(true, "", newSliceMem(memfile.mem, memfile.size, proc() =
|
|
try:
|
|
memfile.close()
|
|
# this should never happen but destructors require it
|
|
except OSError: discard
|
|
))
|
|
except OSError:
|
|
self.onload(false, "Could not open file: " & uri)
|
|
proc str(): string = uri
|
|
self = newLoadableResource[Fetch](start_func, str, use_threads=use_threads)
|
|
self.onload_func = onload_func
|
|
if auto_start:
|
|
start(self)
|
|
return self
|