From a8a0b35297f5fbe40e968143358e920366517424 Mon Sep 17 00:00:00 2001 From: Alberto Torres Date: Fri, 6 Sep 2024 01:08:58 +0200 Subject: [PATCH] SliceMem: Add `serialize`, `deserialize`, `toString`, `copy` and documentation. --- libs/arr_ref/slice_mem.nim | 148 +++++++++++++++++++++++++++++++------ 1 file changed, 127 insertions(+), 21 deletions(-) diff --git a/libs/arr_ref/slice_mem.nim b/libs/arr_ref/slice_mem.nim index f1e29ad..4c41c22 100644 --- a/libs/arr_ref/slice_mem.nim +++ b/libs/arr_ref/slice_mem.nim @@ -1,13 +1,15 @@ -## SliceMem represents a slice of memory, but it includes a reference to the -## original container (of any type) or even a custom destructor, so memory is -## properly freed when it's no longer in use. +## SliceMem is a general purpose array that can be casted to any type, and does +## not create copies when assigned multiple times, like a reference type, unless +## you use `copy`. It can be created by itself, from another container such as +## a seq (without copying when possible), or from a pointer and a custom +## destructor. ## ## It's called "SliceMem" because "MemSlice" is taken by std/memfiles. import std/strformat import std/hashes -import std/macros +import std/macros # for noMove() type SliceMem*[T] = object @@ -31,6 +33,10 @@ template toPointer*(s: SliceMem): pointer = proc newSliceMem*[T, U](container: sink U; p: pointer, byte_len: int): SliceMem[T] = ## Create a SliceMem from a container, a pointer, and a length in bytes. + runnableExamples: + let x = @[1,2,3,4,5] + let s = newSliceMem(x, x[0].addr, x.len * sizeof(x[0])) + result = SliceMem[T]( data: cast[ptr UncheckedArray[T]](p), byte_len: byte_len, @@ -45,6 +51,11 @@ proc newSliceMem*[T, U](container: sink U; p: pointer, byte_len: int): SliceMem[ proc newSliceMem*[T](p: ptr T, byte_len: int, destructor: proc() {.closure, raises: [].}): SliceMem[T] = ## Create a SliceMem from a pointer to a type, a length in bytes, and a ## destructor closure. + runnableExamples: + let x = createShared(int, 5) + proc destroy() = deallocShared(x) + let s = newSliceMem(x, 5, destroy) + result = SliceMem[T]( data: cast[ptr UncheckedArray[T]](p), byte_len: byte_len, @@ -52,6 +63,8 @@ proc newSliceMem*[T](p: ptr T, byte_len: int, destructor: proc() {.closure, rais ) proc newSliceMem*(p: pointer, byte_len: int, destructor: proc() {.closure, raises: [].}): SliceMem[byte] {.inline.} = + ## Create a SliceMem from a pointer without type, a length in bytes, and a + ## destructor closure. Same as newSliceMem[T](...) but assumes type is byte. newSliceMem(cast[ptr byte](p), byte_len, destructor) template newSliceMem*(container: not pointer, p, byte_len: untyped): untyped = @@ -83,6 +96,7 @@ proc newSliceMem*[T](size: int): SliceMem[T] = ) macro noMove(e: untyped): untyped = + # remove the "move" from an expression if e.kind == nnkCommand and e[0].repr == "move": e[1] else: e @@ -90,20 +104,43 @@ template toSliceMem*(container, slice: untyped): untyped = ## Create a SliceMem from a container and a slice that indicates the range. ## The container needs to have `[]` and `items()` (like `seq`). If it doesn't ## you need to pass pointers directly to `newSliceMem` instead. - if slice.len != 0: - # TODO: check that with boundChecks:off it doesn't try to read the value - let first = noMove(container)[slice.a].addr - let last = noMove(container)[slice.b].addr - newSliceMem[typeof(noMove(container).items)](container, first, last) - else: - SliceMem[typeof(container.items)]() + runnableExamples: + var x = @[1,2,3,4,5] + let a = x.toSliceMem(1..3) + let b = toSliceMem(move x, 1..3) # This also works, and ensures that + # the contents of x are not copied + {.line.}: + if slice.len != 0: + # TODO: check that with boundChecks:off it doesn't try to read the value + let first = noMove(container)[slice.a].addr + let last = noMove(container)[slice.b].addr + newSliceMem[typeof(noMove(container).items)](container, first, last) + else: + SliceMem[typeof(container.items)]() template toSliceMem*(container): untyped = ## Create a SliceMem from a container, with all its contents. The container ## needs to have `[]` and `items()` (like `seq`). If it doesn't you need to ## pass pointers directly to `newSliceMem` instead. - let s = noMove(container).low .. noMove(container).high - toSliceMem(container, s) + runnableExamples: + var x = @[1,2,3,4,5] + let a = x.toSliceMem + let b = toSliceMem(move x) # This also works, and ensures that + # the contents of x are not copied + {.line.}: + let s = noMove(container).low .. noMove(container).high + when not compiles(container[s.a].addr): + # expression container[x] has no address because it's a literal, + # let's make it real + let c = container + toSliceMem(c, s) + else: + toSliceMem(container, s) + +proc copy*[T](s: SliceMem[T]): SliceMem[T] = + ## Creates a copy of a SliceMem that doesn't reference the original one. + result = newSliceMem[byte](s.byte_len).to(T) + copyMem(result.toPointer, s.toPointer, s.byte_len) template len*[T](s: SliceMem[T]): Natural = s.byte_len div sizeof(T) @@ -175,15 +212,24 @@ template toOpenArrayByte*(s: SliceMem): untyped = cast[ptr UncheckedArray[byte]](s.data).toOpenArray(0, s.byte_len - 1) template to*[T](s: SliceMem, typ: typedesc[T]): SliceMem[T] = + ## Cast the SliceMem from one type to another cast[SliceMem[T]](s) -template to*[T](s: seq[SliceMem], typ: typedesc[T]): seq[SliceMem[T]] = +template to*[T](s: openArray[SliceMem], typ: typedesc[T]): seq[SliceMem[T]] = + ## Cast a seq of SliceMems from one type to another var s2 = newSeqOfCap[SliceMem[T]](s.len) for slice in s: s2.add cast[SliceMem[T]](slice) s2 -template concatImpl(slices: untyped) = +proc concat*[T](slices: varargs[SliceMem[T]]): SliceMem[T] = + ## Concatenates a list of SliceMems into a single one. Contents are copied. + runnableExamples: + let a = @[1,2,3,4,5].toSliceMem + let b = @[6,7,8,9,0].toSliceMem + var x = concat(a,b) + x = concat(a,b,a,a,b) # You can pass any amount of slices as arguments + x = concat(@[b,b,a,a]) # or you can pass a seq var total = 0 for s in slices: total += s.len @@ -193,11 +239,71 @@ template concatImpl(slices: untyped) = copyMem(result[offset].addr, s.data, s.len * sizeof(T)) offset += s.len -proc concat*[T](slices: varargs[SliceMem[T]]): SliceMem[T] = - concatImpl(slices) - -proc concat*[T](slices: seq[SliceMem[T]]): SliceMem[T] = - concatImpl(slices) - template `&`*[T,U](a: SliceMem[T], b: SliceMem[U]): SliceMem[T] = + ## Concatenates two SliceMems (of any type) into a new one with the type of + ## the first one. concat(a, b.to(T)) + +proc serializeToSeq*[T](slices: openArray[SliceMem[T]], align = sizeof(int)): seq[SliceMem[byte]] = + ## Converts a list of SliceMems into a format that can be saved to file and + ## deserialized later. See `serialize` and `deserialize`. This proc returns a + ## seq that is ready to concatenate or to write to a file. + let mask = align - 1 + assert((align and mask) == 0, "Align must be a power of two") + var offset = 0 + let align_count = newSliceMem[int32](2) + align_count[0] = align.int32 + align_count[1] = slices.len.int32 + result.add align_count.to(byte) + offset += align_count.byte_len + let lengths = newSliceMem[int64](slices.len) + result.add lengths.to(byte) + offset += lengths.byte_len + let pad_bytes = newSliceMem[byte](align) + for i,s in slices: + let pad = ((align-(offset and mask)) and mask) + if pad != 0: + result.add pad_bytes[0 ..< pad] + offset += pad + lengths[i] = s.byte_len + result.add s.to(byte) + offset += s.byte_len + +proc serialize*[T](slices: openArray[SliceMem[T]], align = sizeof(int)): SliceMem[byte] = + ## Converts a list of SliceMems into a single segment of memory that can be + ## later deserialized into the separate SliceMems again. Use `deserialize` + ## for the inverse process. Use `serializeToSeq` to obtain the separate + ## formatted slices before they're concatenated and avoid copies. + runnableExamples: + let a = @[1,2,3,4,5].toSliceMem + let b = @[6,7,8,9,0].toSliceMem + let serialized = serialize(@[a,b]) + let slices = serialized.to(int).deserialize + doAssert $slices == "@[SliceMem([1, 2, 3, 4, 5]), SliceMem([6, 7, 8, 9, 0])]" + concat(serializeToSeq(slices, align)) + +proc deserialize*[T](data: SliceMem[T]): seq[SliceMem[T]] = + ## Reverts the process done by `serialize` except the conversion to byte. + ## You can convert to the appropriate type before or after deserialization + ## with `to` (see example in `serialize`) + let bytes = data.to(byte) + let i32 = data.to(int32) + let align = i32[0] + let count = i32[1] + let mask = align - 1 + assert((align and mask) == 0, "Wrong format, align is invalid") + let lengths = bytes[8 ..< 8 + count*sizeof(int64)].to(int64) + var offset = 8 + lengths.byte_len + for i in 0 ..< count: + offset += ((align-(offset and mask)) and mask) + let len = lengths[i].int + result.add bytes[offset ..< offset+len].to(T) + offset += len + +template toString*(s: SliceMem): string = + ## Copies the contents of the SliceMem to a new string of the same length. + var str = newString(s.byte_len) + if s.byte_len != 0: + copyMem(str[0].addr, s.toPointer, s.byte_len) + str +