myou-engine/libs/arr_ref/slice_mem.nim

## SliceMem is a general purpose array that can be casted to any type, and does
## not create copies when assigned multiple times, like a reference type, unless
## you use `copy`. It can be created by itself, from another container such as
## a seq (without copying when possible), or from a pointer and a custom
## destructor.
## 
## It's called "SliceMem" because "MemSlice" is taken by std/memfiles.

import std/strformat
import std/hashes
import std/macros # for noMove()

type
    SliceMem*[T] = object
        data*: ptr UncheckedArray[T]
        byte_len*: int
        destroy_ref: ref CustomDestructor
    
    CustomDestructor = object
        destroy: proc() {.closure, raises: [].}

proc `=destroy`(s: var CustomDestructor) =
    if s.destroy != nil:
        s.destroy()
        s.destroy = nil

template toInt(p: pointer): int = cast[int](p)
# template toPointer(i: int): pointer = cast[pointer](p)

template toPointer*(s: SliceMem): pointer =
    s.data.pointer

proc newSliceMem*[T, U](container: sink U; p: pointer, byte_len: int): SliceMem[T] =
    ## Create a SliceMem from a container, a pointer, and a length in bytes.
    runnableExamples:
        let x = @[1,2,3,4,5]
        # You can omit the [int] here because the container is iterable
        let s = newSliceMem[int](x, x[0].addr, x.len * sizeof(x[0]))
    
    result = SliceMem[T](
        data: cast[ptr UncheckedArray[T]](p),
        byte_len: byte_len,
        destroy_ref: (ref CustomDestructor)(destroy: proc()=
            # we don't actually need to destroy the container here, destroying
            # the closure will do it for us. Calling it allows us to use custom
            # destructors though.
            discard container
        ),
    )

proc newSliceMem*[T](p: ptr T, len: int, destructor: proc() {.closure, raises: [].}): SliceMem[T] =
    # Create a SliceMem from a pointer to a type, a length, and a destructor
    # closure.
    runnableExamples:
        let x = createShared(int, 5)
        proc destroy() = deallocShared(x)
        let s = newSliceMem(x, 5, destroy)
    
    result = SliceMem[T](
        data: cast[ptr UncheckedArray[T]](p),
        byte_len: len * sizeof(T),
        destroy_ref: (ref CustomDestructor)(destroy: destructor),
    )

proc newSliceMem*(p: pointer, byte_len: int, destructor: proc() {.closure, raises: [].}): SliceMem[byte] {.inline.} =
    ## Create a SliceMem from a pointer without type, a length in bytes, and a
    ## destructor closure. Same as newSliceMem[T](...) but assumes type is byte.
    newSliceMem(cast[ptr byte](p), byte_len, destructor)

template newSliceMem*(container: not pointer, p, byte_len: untyped): untyped =
    ## Template to automatically determine the type for `newSliceMem[T,U]`
    newSliceMem[typeof(container.items)](container, p, byte_len)

proc newSliceMem*[T, U](container: sink U; first, last: pointer): SliceMem[T] =
    ## Create a SliceMem from a container and the two pointers of the first and
    ## last elements in the container. Usually you will want to use `toSliceMem`
    ## instead.
    result = SliceMem[T](
        data: cast[ptr UncheckedArray[T]](first),
        byte_len: (last.toInt -% first.toInt) + sizeof(T),
        destroy_ref: (ref CustomDestructor)(destroy: proc()=
            discard container
        ),
    )

proc newSliceMem*[T](size: int): SliceMem[T] =
    ## Create a SliceMem from new memory, similar to ArrRef[T].
    runnableExamples:
        let s = newSliceMem[int](10)
        for i,n in s.mpairs:
            n = (i+1)*11
        doAssert $s == "SliceMem([11, 22, 33, 44, 55, 66, 77, 88, 99, 110])"
    var r: ref byte
    unsafeNew(r, size * sizeof(T))
    result = SliceMem[T](
        data: cast[ptr UncheckedArray[T]](r),
        byte_len: size * sizeof(T),
        destroy_ref: (ref CustomDestructor)(destroy: proc()=
            discard r
        ),
    )

macro noMove(e: untyped): untyped =
    # remove the "move" from an expression
    if e.kind == nnkCommand and e[0].repr == "move": e[1]
    else: e

template toSliceMem*(container, slice: untyped): untyped =
    ## Create a SliceMem from a container and a slice that indicates the range.
    ## The container needs to have `[]` and `items()` (like `seq`). If it doesn't
    ## you need to pass pointers directly to `newSliceMem` instead.
    runnableExamples:
        var x = @[1,2,3,4,5]
        let a = x.toSliceMem(1..3)
        let b = toSliceMem(move x, 1..3) # This also works, and ensures that
                                         # the contents of x are not copied
    {.line.}:
        if slice.len != 0:
            # TODO: check that with boundChecks:off it doesn't try to read the value
            let first = noMove(container)[slice.a].addr
            let last = noMove(container)[slice.b].addr
            newSliceMem[typeof(noMove(container).items)](container, first, last)
        else:
            SliceMem[typeof(container.items)]()

template toSliceMem*(container): untyped =
    ## Create a SliceMem from a container, with all its contents. The container
    ## needs to have `[]` and `items()` (like `seq`). If it doesn't you need to
    ## pass pointers directly to `newSliceMem` instead.
    runnableExamples:
        var x = @[1,2,3,4,5]
        let a = x.toSliceMem
        let b = toSliceMem(move x) # This also works, and ensures that
                                   # the contents of x are not copied
    {.line.}:
        let s = noMove(container).low .. noMove(container).high
        when not compiles(container[s.a].addr):
            # expression container[x] has no address because it's a literal,
            # let's make it real
            let c = container
            toSliceMem(c, s)
        else:
            toSliceMem(container, s)

proc copy*[T](s: SliceMem[T]): SliceMem[T] =
    ## Creates a copy of a SliceMem that doesn't reference the original one.
    result = newSliceMem[byte](s.byte_len).to(T)
    copyMem(result.toPointer, s.toPointer, s.byte_len)

template len*[T](s: SliceMem[T]): Natural =
    s.byte_len div sizeof(T)

template low*[T](s: SliceMem[T]): Natural = 0

template high*[T](s: SliceMem[T]): int = s.len - 1

proc `[]`*[T](s: SliceMem[T], i: Natural): var T {.inline.} =
    when compileOption("rangechecks"):
        if i >= s.len:
            raise newException(RangeDefect, &"index out of range: {i} >= {s.len}")
    s.data[i]

proc `[]=`*[T](s: SliceMem[T], i: Natural, v: T) {.inline.} =
    when compileOption("rangechecks"):
        if i >= s.len:
            raise newException(RangeDefect, &"index out of range: {i} >= {s.len}")
    s.data[i] = v

template `[]`*[T](s: SliceMem[T], i: BackwardsIndex): T =
    s[s.len - i.int]

template `[]=`*[T](s: SliceMem[T], i: BackwardsIndex, v: T) =
    s[s.len - i.int] = v

template `[]`*[T](s: SliceMem[T], i: BackwardsIndex): T =
    s[s.len - i.int]

template `[]=`*[T](s: SliceMem[T], i: BackwardsIndex, v: T) =
    s[s.len - i.int] = v

template `[]`*[T](a: SliceMem[T], s: Slice[int]): var SliceMem[T] =
    toSliceMem(a, s)

iterator items*[T](s: SliceMem[T]): T =
    for i in 0 ..< s.len:
        yield s.data[i]

iterator pairs*[T](s: SliceMem[T]): tuple[key: int, val: T] =
    for i in 0 ..< s.len:
        yield (i, s.data[i])

iterator mitems*[T](s: SliceMem[T]): var T =
    for i in 0 ..< s.len:
        yield s.data[i]

iterator mpairs*[T](s: SliceMem[T]): tuple[key: int, val: var T] =
    for i in 0 ..< s.len:
        yield (i, s.data[i])

proc `$`*[T](s: SliceMem[T]): string =
    result = "SliceMem(["
    if s.byte_len != 0:
        let hi = s.high
        for i in 0 ..< hi:
            result &= $s.data[i] & ", "
        result &= $s.data[hi]
    result &= "])"

proc hash*[T](s: SliceMem[T]): Hash =
    # Make use of stdlib's murmur3
    hash(cast[ptr UncheckedArray[byte]](s.data).toOpenArray(0, s.byte_len - 1))

template toOpenArray*(s: SliceMem): untyped =
    s.data.toOpenArray(0, s.byte_len - 1)

template toOpenArrayByte*(s: SliceMem): untyped =
    cast[ptr UncheckedArray[byte]](s.data).toOpenArray(0, s.byte_len - 1)

template to*[T](s: SliceMem, typ: typedesc[T]): SliceMem[T] =
    ## Cast the SliceMem from one type to another
    cast[SliceMem[T]](s)

template to*[T](s: openArray[SliceMem], typ: typedesc[T]): seq[SliceMem[T]] =
    ## Cast a seq of SliceMems from one type to another
    var s2 = newSeqOfCap[SliceMem[T]](s.len)
    for slice in s:
        s2.add cast[SliceMem[T]](slice)
    s2

proc concat*[T](slices: varargs[SliceMem[T]]): SliceMem[T] =
    ## Concatenates a list of SliceMems into a single one. Contents are copied.
    runnableExamples:
        let a = @[1,2,3,4,5].toSliceMem
        let b = @[6,7,8,9,0].toSliceMem
        var x = concat(a,b)
        x = concat(a,b,a,a,b)  # You can pass any amount of slices as arguments
        x = concat(@[b,b,a,a]) # or you can pass a seq
    var total = 0
    for s in slices:
        total += s.len
    result = newSliceMem[T](total)
    var offset = 0
    for s in slices:
        copyMem(result[offset].addr, s.data, s.len * sizeof(T))
        offset += s.len

template `&`*[T,U](a: SliceMem[T], b: SliceMem[U]): SliceMem[T] =
    ## Concatenates two SliceMems (of any type) into a new one with the type of
    ## the first one.
    concat(a, b.to(T))

proc serializeToSeq*[T](slices: openArray[SliceMem[T]], align = sizeof(int)): seq[SliceMem[byte]] =
    ## Converts a list of SliceMems into a format that can be saved to file and
    ## deserialized later. See `serialize` and `deserialize`. This proc returns a
    ## seq that is ready to concatenate or to write to a file.
    let mask = align - 1
    assert((align and mask) == 0, "Align must be a power of two")
    var offset = 0
    let align_count = newSliceMem[int32](2)
    align_count[0] = align.int32
    align_count[1] = slices.len.int32
    result.add align_count.to(byte)
    offset += align_count.byte_len
    let lengths = newSliceMem[int64](slices.len)
    result.add lengths.to(byte)
    offset += lengths.byte_len
    let pad_bytes = newSliceMem[byte](align)
    for i,s in slices:
        let pad = ((align-(offset and mask)) and mask)
        if pad != 0:
            result.add pad_bytes[0 ..< pad]
            offset += pad
        lengths[i] = s.byte_len
        result.add s.to(byte)
        offset += s.byte_len

proc serialize*[T](slices: openArray[SliceMem[T]], align = sizeof(int)): SliceMem[byte] =
    ## Converts a list of SliceMems into a single segment of memory that can be
    ## later deserialized into the separate SliceMems again. Use `deserialize`
    ## for the inverse process. Use `serializeToSeq` to obtain the separate
    ## formatted slices before they're concatenated and avoid copies.
    runnableExamples:
        let a = @[1,2,3,4,5].toSliceMem
        let b = @[6,7,8,9,0].toSliceMem
        let serialized = serialize(@[a,b])
        let slices = serialized.to(int).deserialize
        doAssert $slices == "@[SliceMem([1, 2, 3, 4, 5]), SliceMem([6, 7, 8, 9, 0])]"
    concat(serializeToSeq(slices, align))

proc deserialize*[T](data: SliceMem[T]): seq[SliceMem[T]] =
    ## Reverts the process done by `serialize` except the conversion to byte.
    ## You can convert to the appropriate type before or after deserialization
    ## with `to` (see example in `serialize`)
    let bytes = data.to(byte)
    let i32 = data.to(int32)
    let align = i32[0]
    let count = i32[1]
    let mask = align - 1
    assert((align and mask) == 0, "Wrong format, align is invalid")
    let lengths = bytes[8 ..< 8 + count*sizeof(int64)].to(int64)
    var offset = 8 + lengths.byte_len
    for i in 0 ..< count:
        offset += ((align-(offset and mask)) and mask)
        let len = lengths[i].int
        result.add bytes[offset ..< offset+len].to(T)
        offset += len

template toString*(s: SliceMem): string =
    ## Copies the contents of the SliceMem to a new string of the same length.
    var str = newString(s.byte_len)
    if s.byte_len != 0:
        copyMem(str[0].addr, s.toPointer, s.byte_len)
    str