SliceMem: Add serialize, deserialize, toString, copy and documentation.
This commit is contained in:
parent
83011ea344
commit
a8a0b35297
|
|
@ -1,13 +1,15 @@
|
|||
|
||||
## SliceMem represents a slice of memory, but it includes a reference to the
|
||||
## original container (of any type) or even a custom destructor, so memory is
|
||||
## properly freed when it's no longer in use.
|
||||
## SliceMem is a general purpose array that can be casted to any type, and does
|
||||
## not create copies when assigned multiple times, like a reference type, unless
|
||||
## you use `copy`. It can be created by itself, from another container such as
|
||||
## a seq (without copying when possible), or from a pointer and a custom
|
||||
## destructor.
|
||||
##
|
||||
## It's called "SliceMem" because "MemSlice" is taken by std/memfiles.
|
||||
|
||||
import std/strformat
|
||||
import std/hashes
|
||||
import std/macros
|
||||
import std/macros # for noMove()
|
||||
|
||||
type
|
||||
SliceMem*[T] = object
|
||||
|
|
@ -31,6 +33,10 @@ template toPointer*(s: SliceMem): pointer =
|
|||
|
||||
proc newSliceMem*[T, U](container: sink U; p: pointer, byte_len: int): SliceMem[T] =
|
||||
## Create a SliceMem from a container, a pointer, and a length in bytes.
|
||||
runnableExamples:
|
||||
let x = @[1,2,3,4,5]
|
||||
let s = newSliceMem(x, x[0].addr, x.len * sizeof(x[0]))
|
||||
|
||||
result = SliceMem[T](
|
||||
data: cast[ptr UncheckedArray[T]](p),
|
||||
byte_len: byte_len,
|
||||
|
|
@ -45,6 +51,11 @@ proc newSliceMem*[T, U](container: sink U; p: pointer, byte_len: int): SliceMem[
|
|||
proc newSliceMem*[T](p: ptr T, byte_len: int, destructor: proc() {.closure, raises: [].}): SliceMem[T] =
|
||||
## Create a SliceMem from a pointer to a type, a length in bytes, and a
|
||||
## destructor closure.
|
||||
runnableExamples:
|
||||
let x = createShared(int, 5)
|
||||
proc destroy() = deallocShared(x)
|
||||
let s = newSliceMem(x, 5, destroy)
|
||||
|
||||
result = SliceMem[T](
|
||||
data: cast[ptr UncheckedArray[T]](p),
|
||||
byte_len: byte_len,
|
||||
|
|
@ -52,6 +63,8 @@ proc newSliceMem*[T](p: ptr T, byte_len: int, destructor: proc() {.closure, rais
|
|||
)
|
||||
|
||||
proc newSliceMem*(p: pointer, byte_len: int, destructor: proc() {.closure, raises: [].}): SliceMem[byte] {.inline.} =
|
||||
## Create a SliceMem from a pointer without type, a length in bytes, and a
|
||||
## destructor closure. Same as newSliceMem[T](...) but assumes type is byte.
|
||||
newSliceMem(cast[ptr byte](p), byte_len, destructor)
|
||||
|
||||
template newSliceMem*(container: not pointer, p, byte_len: untyped): untyped =
|
||||
|
|
@ -83,6 +96,7 @@ proc newSliceMem*[T](size: int): SliceMem[T] =
|
|||
)
|
||||
|
||||
macro noMove(e: untyped): untyped =
|
||||
# remove the "move" from an expression
|
||||
if e.kind == nnkCommand and e[0].repr == "move": e[1]
|
||||
else: e
|
||||
|
||||
|
|
@ -90,20 +104,43 @@ template toSliceMem*(container, slice: untyped): untyped =
|
|||
## Create a SliceMem from a container and a slice that indicates the range.
|
||||
## The container needs to have `[]` and `items()` (like `seq`). If it doesn't
|
||||
## you need to pass pointers directly to `newSliceMem` instead.
|
||||
if slice.len != 0:
|
||||
# TODO: check that with boundChecks:off it doesn't try to read the value
|
||||
let first = noMove(container)[slice.a].addr
|
||||
let last = noMove(container)[slice.b].addr
|
||||
newSliceMem[typeof(noMove(container).items)](container, first, last)
|
||||
else:
|
||||
SliceMem[typeof(container.items)]()
|
||||
runnableExamples:
|
||||
var x = @[1,2,3,4,5]
|
||||
let a = x.toSliceMem(1..3)
|
||||
let b = toSliceMem(move x, 1..3) # This also works, and ensures that
|
||||
# the contents of x are not copied
|
||||
{.line.}:
|
||||
if slice.len != 0:
|
||||
# TODO: check that with boundChecks:off it doesn't try to read the value
|
||||
let first = noMove(container)[slice.a].addr
|
||||
let last = noMove(container)[slice.b].addr
|
||||
newSliceMem[typeof(noMove(container).items)](container, first, last)
|
||||
else:
|
||||
SliceMem[typeof(container.items)]()
|
||||
|
||||
template toSliceMem*(container): untyped =
|
||||
## Create a SliceMem from a container, with all its contents. The container
|
||||
## needs to have `[]` and `items()` (like `seq`). If it doesn't you need to
|
||||
## pass pointers directly to `newSliceMem` instead.
|
||||
let s = noMove(container).low .. noMove(container).high
|
||||
toSliceMem(container, s)
|
||||
runnableExamples:
|
||||
var x = @[1,2,3,4,5]
|
||||
let a = x.toSliceMem
|
||||
let b = toSliceMem(move x) # This also works, and ensures that
|
||||
# the contents of x are not copied
|
||||
{.line.}:
|
||||
let s = noMove(container).low .. noMove(container).high
|
||||
when not compiles(container[s.a].addr):
|
||||
# expression container[x] has no address because it's a literal,
|
||||
# let's make it real
|
||||
let c = container
|
||||
toSliceMem(c, s)
|
||||
else:
|
||||
toSliceMem(container, s)
|
||||
|
||||
proc copy*[T](s: SliceMem[T]): SliceMem[T] =
|
||||
## Creates a copy of a SliceMem that doesn't reference the original one.
|
||||
result = newSliceMem[byte](s.byte_len).to(T)
|
||||
copyMem(result.toPointer, s.toPointer, s.byte_len)
|
||||
|
||||
template len*[T](s: SliceMem[T]): Natural =
|
||||
s.byte_len div sizeof(T)
|
||||
|
|
@ -175,15 +212,24 @@ template toOpenArrayByte*(s: SliceMem): untyped =
|
|||
cast[ptr UncheckedArray[byte]](s.data).toOpenArray(0, s.byte_len - 1)
|
||||
|
||||
template to*[T](s: SliceMem, typ: typedesc[T]): SliceMem[T] =
|
||||
## Cast the SliceMem from one type to another
|
||||
cast[SliceMem[T]](s)
|
||||
|
||||
template to*[T](s: seq[SliceMem], typ: typedesc[T]): seq[SliceMem[T]] =
|
||||
template to*[T](s: openArray[SliceMem], typ: typedesc[T]): seq[SliceMem[T]] =
|
||||
## Cast a seq of SliceMems from one type to another
|
||||
var s2 = newSeqOfCap[SliceMem[T]](s.len)
|
||||
for slice in s:
|
||||
s2.add cast[SliceMem[T]](slice)
|
||||
s2
|
||||
|
||||
template concatImpl(slices: untyped) =
|
||||
proc concat*[T](slices: varargs[SliceMem[T]]): SliceMem[T] =
|
||||
## Concatenates a list of SliceMems into a single one. Contents are copied.
|
||||
runnableExamples:
|
||||
let a = @[1,2,3,4,5].toSliceMem
|
||||
let b = @[6,7,8,9,0].toSliceMem
|
||||
var x = concat(a,b)
|
||||
x = concat(a,b,a,a,b) # You can pass any amount of slices as arguments
|
||||
x = concat(@[b,b,a,a]) # or you can pass a seq
|
||||
var total = 0
|
||||
for s in slices:
|
||||
total += s.len
|
||||
|
|
@ -193,11 +239,71 @@ template concatImpl(slices: untyped) =
|
|||
copyMem(result[offset].addr, s.data, s.len * sizeof(T))
|
||||
offset += s.len
|
||||
|
||||
proc concat*[T](slices: varargs[SliceMem[T]]): SliceMem[T] =
|
||||
concatImpl(slices)
|
||||
|
||||
proc concat*[T](slices: seq[SliceMem[T]]): SliceMem[T] =
|
||||
concatImpl(slices)
|
||||
|
||||
template `&`*[T,U](a: SliceMem[T], b: SliceMem[U]): SliceMem[T] =
|
||||
## Concatenates two SliceMems (of any type) into a new one with the type of
|
||||
## the first one.
|
||||
concat(a, b.to(T))
|
||||
|
||||
proc serializeToSeq*[T](slices: openArray[SliceMem[T]], align = sizeof(int)): seq[SliceMem[byte]] =
|
||||
## Converts a list of SliceMems into a format that can be saved to file and
|
||||
## deserialized later. See `serialize` and `deserialize`. This proc returns a
|
||||
## seq that is ready to concatenate or to write to a file.
|
||||
let mask = align - 1
|
||||
assert((align and mask) == 0, "Align must be a power of two")
|
||||
var offset = 0
|
||||
let align_count = newSliceMem[int32](2)
|
||||
align_count[0] = align.int32
|
||||
align_count[1] = slices.len.int32
|
||||
result.add align_count.to(byte)
|
||||
offset += align_count.byte_len
|
||||
let lengths = newSliceMem[int64](slices.len)
|
||||
result.add lengths.to(byte)
|
||||
offset += lengths.byte_len
|
||||
let pad_bytes = newSliceMem[byte](align)
|
||||
for i,s in slices:
|
||||
let pad = ((align-(offset and mask)) and mask)
|
||||
if pad != 0:
|
||||
result.add pad_bytes[0 ..< pad]
|
||||
offset += pad
|
||||
lengths[i] = s.byte_len
|
||||
result.add s.to(byte)
|
||||
offset += s.byte_len
|
||||
|
||||
proc serialize*[T](slices: openArray[SliceMem[T]], align = sizeof(int)): SliceMem[byte] =
|
||||
## Converts a list of SliceMems into a single segment of memory that can be
|
||||
## later deserialized into the separate SliceMems again. Use `deserialize`
|
||||
## for the inverse process. Use `serializeToSeq` to obtain the separate
|
||||
## formatted slices before they're concatenated and avoid copies.
|
||||
runnableExamples:
|
||||
let a = @[1,2,3,4,5].toSliceMem
|
||||
let b = @[6,7,8,9,0].toSliceMem
|
||||
let serialized = serialize(@[a,b])
|
||||
let slices = serialized.to(int).deserialize
|
||||
doAssert $slices == "@[SliceMem([1, 2, 3, 4, 5]), SliceMem([6, 7, 8, 9, 0])]"
|
||||
concat(serializeToSeq(slices, align))
|
||||
|
||||
proc deserialize*[T](data: SliceMem[T]): seq[SliceMem[T]] =
|
||||
## Reverts the process done by `serialize` except the conversion to byte.
|
||||
## You can convert to the appropriate type before or after deserialization
|
||||
## with `to` (see example in `serialize`)
|
||||
let bytes = data.to(byte)
|
||||
let i32 = data.to(int32)
|
||||
let align = i32[0]
|
||||
let count = i32[1]
|
||||
let mask = align - 1
|
||||
assert((align and mask) == 0, "Wrong format, align is invalid")
|
||||
let lengths = bytes[8 ..< 8 + count*sizeof(int64)].to(int64)
|
||||
var offset = 8 + lengths.byte_len
|
||||
for i in 0 ..< count:
|
||||
offset += ((align-(offset and mask)) and mask)
|
||||
let len = lengths[i].int
|
||||
result.add bytes[offset ..< offset+len].to(T)
|
||||
offset += len
|
||||
|
||||
template toString*(s: SliceMem): string =
|
||||
## Copies the contents of the SliceMem to a new string of the same length.
|
||||
var str = newString(s.byte_len)
|
||||
if s.byte_len != 0:
|
||||
copyMem(str[0].addr, s.toPointer, s.byte_len)
|
||||
str
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue