# The contents of this file are subject to the Common Public Attribution License
# Version 1.0 (the “License”); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# https://myou.dev/licenses/LICENSE-CPAL. The License is based on the Mozilla
# Public License Version 1.1 but Sections 14 and 15 have been added to cover use
# of software over a computer network and provide for limited attribution for
# the Original Developer. In addition, Exhibit A has been modified to be
# consistent with Exhibit B.
#
# Software distributed under the License is distributed on an “AS IS” basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
# the specific language governing rights and limitations under the License.
#
# The Original Code is Myou Engine.
#
# the Original Developer is the Initial Developer.
#
# The Initial Developer of the Original Code is the Myou Engine developers.
# All portions of the code written by the Myou Engine developers are Copyright
# (c) 2024. All Rights Reserved.
#
# Alternatively, the contents of this file may be used under the terms of the
# GNU Affero General Public License version 3 (the [AGPL-3] License), in which
# case the provisions of [AGPL-3] License are applicable instead of those above.
#
# If you wish to allow use of your version of this file only under the terms of
# the [AGPL-3] License and not to allow others to use your version of this file
# under the CPAL, indicate your decision by deleting the provisions above and
# replace them with the notice and other provisions required by the [AGPL-3]
# License. If you do not delete the provisions above, a recipient may use your
# version of this file under either the CPAL or the [AGPL-3] License.
## A LoadableResource is similar to a JS promise or a Future, except that you can
## store it to load it any number of times, and instead of providing an object,
## you get a pointer, a length, and a function to tell it you're done with it.
## You cannot have more than one callback, but you can obtain the results of many
## with `loadAll` or get a callback when many are done with `allDone`
##
## TODO IMPORTANT: probably not thread safe, call all functions in the same thread
## if you use loadAll and allDone
##
## TODO: investigate the use of signals instead of callbacks
{.hint[ConvFromXtoItselfNotNeeded]:off.}
import std/tables
import std/times
import std/os
import arr_ref # for SliceMem
import sugar
type LoadableResourceStatus* = enum
NotStarted
Started
Finished
Error
type FetchResult* = object
ok*: bool
err*: string
data*: SliceMem[byte]
size*: int
modified*: Time
type LoadableResource* = ref object of RootObj
status: LoadableResourceStatus
start_func: proc(self: LoadableResource) {.closure.}
onload_func: proc(res: FetchResult) {.closure.}
cancel_func: proc() {.closure.}
str*: proc(): string
use_threads: bool
proc newLoadableResource*[T: LoadableResource](
start: proc(self: LoadableResource),
str: proc(): string = nil,
use_threads = false,
): T =
new(result)
result.start_func = start
result.str = str
if str == nil:
result.str = proc(): string = ""
result.use_threads = use_threads
when compileOption("threads"):
# main -> thread channels
var to_start: Channel[LoadableResource]
# main <- thread channels
var to_return: Channel[(LoadableResource, FetchResult)]
proc start*[T: LoadableResource](self: T) =
self.status = Started
if self.use_threads:
when compileOption("threads"): to_start.send self
else:
self.start_func(self)
proc `onload=`*[T: LoadableResource](self: T, onload_func: proc(res: FetchResult)) =
self.onload_func = onload_func
proc onload*[T: LoadableResource](self: T, res: FetchResult) =
if self.status == Started:
self.status = if res.ok: Finished else: Error
if self.onload_func != nil:
if self.use_threads:
when compileOption("threads"):
to_return.send((self.LoadableResource, res))
else:
self.onload_func(res)
proc cancel*[T: LoadableResource](self: T) =
if self.status != Started:
return
if self.cancel_func != nil:
self.cancel_func()
self.status = NotStarted
# proc allDone*(list: seq[LoadableResource], done: proc()) =
# proc loadAll*(list: seq[LoadableResource], onload: proc(all_ok: bool, result: seq[Result])) =
# for i,res in list:
# if not defined(release):
# assert res.status == NotStarted, "All resources in the list must be not started"
# assert res.onload_func == nil, "All resources in the list must not have onload"
# new(res.result)
# res.start()
# list.allDone, proc() =
# var results = newSeqOfCap[Result](list.len)
# for res in list:
# results.add res.result[]
# res.result = nil
when compileOption("threads"):
var worker: Thread[void]
proc workerThreadProc() {.thread.} =
while true:
let res = to_start.recv()
if res == nil:
break
cast[proc(self: LoadableResource) {.gcsafe.}](res.start_func)(res)
worker.createThread(workerThreadProc)
to_start.open()
to_return.open()
proc updateLoadableWorkerThreads*() =
while true:
let tried = to_return.tryRecv()
if not tried.dataAvailable:
break
let (res, data) = tried.msg
res.onload_func(data)
proc terminateLoadableWorkerThreads*() =
# TODO: test this
to_start.send(nil.LoadableResource)
worker.joinThread()
type Fetch* = ref object of LoadableResource
custom: pointer
when not defined(onlyLocalFiles):
when defined(emscripten):
type
FetchAttribute = enum
LoadToMemory = 1
StreamData = 2
PersistFile = 4
Append = 8
Replace = 16
NoDownload = 32
Synchronous = 64
Waitable = 128
emscripten_fetch_attr_t {.importc,nodecl,header:"emscripten/fetch.h".} = object
requestMethod: array[32, char]
userData: pointer
onsuccess: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
onerror: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
onprogress: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
onreadystatechange: proc(fetch: ptr emscripten_fetch_t) {.cdecl.}
attributes: FetchAttribute # uint32
timeoutMSecs: uint32
withCredentials: bool # actually int32
destinationPath: cstring
userName: cstring
password: cstring
requestHeaders: ptr UncheckedArray[cstring]
overriddenMimeType: cstring
requestData: cstring
requestDataSize: uint
emscripten_fetch_t {.importc,nodecl.} = object
id: uint32
userData: pointer
url: cstring
data: pointer
numBytes: uint64
dataOffset: uint64
totalBytes: uint64
readyState: uint16
status: uint16
statusText: array[64, char]
EmscriptenResult = enum
TimedOut = -8
NoData = -7
Failed = -6
InvalidParam = -5
UnknownTarget = -4
InvalidTarget = -3
FailedNotDeferred = -2
NotSupported = -1
Success = 0
Deferred = 1
const EMSCRIPTEN_RESULT_STRINGS = [
"TimedOut","NoData","Failed","InvalidParam",
"UnknownTarget","InvalidTarget","FailedNotDeferred",
"NotSupported","Success","Deferred"]
proc emscripten_fetch_attr_init(fetch_attr: var emscripten_fetch_attr_t) {.importc,nodecl.}
proc emscripten_fetch(fetch_attr: var emscripten_fetch_attr_t, url: cstring): ptr emscripten_fetch_t {.importc,nodecl.}
proc emscripten_fetch_wait(fetch: ptr emscripten_fetch_t, timeoutMSecs: float): EmscriptenResult {.importc,nodecl.}
proc emscripten_fetch_close(fetch: ptr emscripten_fetch_t): EmscriptenResult {.importc,nodecl.}
proc emscripten_fetch_get_response_headers_length(fetch: ptr emscripten_fetch_t): uint {.importc,nodecl.}
proc emscripten_fetch_get_response_headers(fetch: ptr emscripten_fetch_t, dst: cstring , dstSizeBytes: uint): uint {.importc,nodecl.}
proc emscripten_fetch_unpack_response_headers(headersString: cstring): ptr UncheckedArray[cstring] {.importc,nodecl.}
proc emscripten_fetch_free_unpacked_response_headers(unpackedHeaders: ptr UncheckedArray[cstring]) {.importc,nodecl.}
else:
import std/httpclient
import std/httpcore
import std/memfiles
import std/strutils
import std/strformat
func escapeUTF8*(s: string): string =
result = newStringOfCap(s.len + 5)
for i,c in s:
if c > ' ' and c <= '~':
result &= c
else:
result &= '%' & c.byte.toHex
# TODO: automatically disable threads when not in main thread
type ProtocolHandler* = proc(uri: string): proc(self: LoadableResource)
var custom_protocol_handlers: Table[string, ProtocolHandler]
var log_uri_handler: proc(uri: string)
proc registerCustomProtocol*(prefix: string, handler: ProtocolHandler) =
## Registers a handler for a custom protocol. The function will be run for
## each uri that is requested, and will return a start_func, which in turn
## will call self.onload()
custom_protocol_handlers[prefix] = handler
proc registerLogUriHandler*(handler: proc(uri: string)) =
log_uri_handler = handler
proc loadUri*(
uri: string,
onload_func: proc(res: FetchResult) = nil,
range = (-1,-1),
auto_start = true,
use_threads = true,
headers_only = false,
): Fetch {.discardable.} =
if log_uri_handler != nil:
log_uri_handler(uri)
for k,v in custom_protocol_handlers:
if uri.startswith k:
proc str(): string = uri
var self = newLoadableResource[Fetch](v(uri), str, false)
self.onload_func = onload_func
if auto_start:
start(self)
return self
var start_func: proc(self: LoadableResource)
var self: Fetch
var uri = uri
when compileOption("threads"):
var use_threads = use_threads
else:
var use_threads = false
when not defined(onlyLocalFiles):
when defined(emscripten):
const is_remote = true
else:
let is_remote = uri.startswith("http://") or uri.startswith("https://")
if is_remote:
uri = uri.escapeUTF8
when defined(emscripten):
assert not headers_only, "TODO: emscripten HEAD and parsing header"
use_threads = false # API is already threaded
start_func = proc(self: LoadableResource) =
var attr: emscripten_fetch_attr_t
emscripten_fetch_attr_init(attr)
attr.userData = cast[pointer](self)
attr.requestMethod.addr.copyMem("GET".cstring, 4)
attr.attributes = LoadToMemory
attr.onsuccess = proc(fetch: ptr emscripten_fetch_t) {.cdecl.} =
var self = cast[Fetch](fetch.userData)
self.custom = fetch.pointer
self.onload(true, "", newSliceMem(fetch.data, fetch.numBytes.int, proc() =
emscripten_fetch_close(fetch)
))
attr.onerror = proc(fetch: ptr emscripten_fetch_t) {.cdecl.} =
var self = cast[Fetch](fetch.userData)
let err_msg = $fetch.statusText.addr.cstring
let err = emscripten_fetch_close(fetch)
# self.onload(false, &"Error: {err_msg} ({EMSCRIPTEN_RESULT_STRINGS[err.int + 8]})", nil, 0)
self.onload(false, &"Error fetching {fetch.url}: {err_msg}")
discard emscripten_fetch(attr, uri.cstring)
else:
var client = newHttpClient()
var content: string
start_func = proc(self: LoadableResource) =
let self = cast[Fetch](self)
var ok = false
var time: Time
var size: int
try:
let response = client.request(uri, if headers_only: HttpHead else: HttpGet)
content = response.body
try: time = parse(response.headers["last-modified"], "ddd, dd MMM yyyy HH:mm:ss 'GMT'").toTime
except: discard
try: size = parseInt(response.headers["content-length"])
except: discard
ok = true
except:
self.onload(FetchResult(err: &"Error fetching {uri}: {getCurrentExceptionMsg()}"))
if ok:
self.onload(FetchResult(
ok: true,
data: toSliceMem(content).to(byte),
size: if headers_only: size else: content.len,
modified: time,
))
if not is_remote:
use_threads = false # TODO: detect networked file system?
# TODO: also test if there's better perf in local
start_func = proc(self: LoadableResource) =
try:
if not uri.fileExists():
self.onload(FetchResult(
ok: false,
err: "File does not exist: " & uri,
))
return
var time: Time
var size: int
try: time = uri.getLastModificationTime()
except: discard
if headers_only:
try: size = uri.getFileSize().int
except: discard
self.onload(FetchResult(
ok: true,
size: size,
modified: time,
))
else:
var memfile = memfiles.open(uri, mode=fmRead)
self.onload(FetchResult(
ok: true,
data: newSliceMem(memfile.mem, memfile.size, proc() =
try:
memfile.close()
# this should never happen but destructors require it
except OSError: discard),
size: memfile.size,
modified: time,
))
except OSError:
self.onload(FetchResult(err: "Could not open file: " & uri))
proc str(): string = uri
self = newLoadableResource[Fetch](start_func, str, use_threads=use_threads)
self.onload_func = onload_func
if auto_start:
start(self)
return self