diff --git a/src/main/java/com/amazon/ion/impl/bin/FixedInt.kt b/src/main/java/com/amazon/ion/impl/bin/FixedInt.kt new file mode 100644 index 000000000..1052cd5d8 --- /dev/null +++ b/src/main/java/com/amazon/ion/impl/bin/FixedInt.kt @@ -0,0 +1,68 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.impl.bin + +import java.lang.Long.numberOfLeadingZeros + +/** + * Functions for encoding FixedInts and FixedUInts. + * + * Expected usage is calling one of the `___length` functions, and then using the result as the input for + * [writeFixedIntOrUIntInto]. The length and write functions are separate so that callers can make decisions or + * compute other values based on the encoded size of the value. + */ +object FixedInt { + + /** + * Writes a FixedInt or FixedUInt encoding of [value] into [data] starting at [offset]. + * Use [fixedIntLength] or [fixedUIntLength] to get the value for the [numBytes] parameter. + */ + @JvmStatic + inline fun writeFixedIntOrUIntInto(data: ByteArray, offset: Int, value: Long, numBytes: Int) { + when (numBytes) { + 1 -> data[offset] = value.toByte() + 2 -> { + data[offset] = value.toByte() + data[offset + 1] = (value shr 8).toByte() + } + 3 -> { + data[offset] = value.toByte() + data[offset + 1] = (value shr 8).toByte() + data[offset + 2] = (value shr 16).toByte() + } + 4 -> { + data[offset] = value.toByte() + data[offset + 1] = (value shr 8).toByte() + data[offset + 2] = (value shr 16).toByte() + data[offset + 3] = (value shr 24).toByte() + } + else -> { + for (i in 0 until numBytes) { + data[offset + i] = (value shr 8 * i).toByte() + } + } + } + } + + /** Determine the length of FixedUInt for the provided value. */ + @JvmStatic + fun fixedUIntLength(value: Long): Int { + val numLeadingZeros = numberOfLeadingZeros(value) + val numMagnitudeBitsRequired = 64 - numLeadingZeros + return (numMagnitudeBitsRequired - 1) / 8 + 1 + } + + /** Determine the length of FixedInt for the provided value. */ + @JvmStatic + fun fixedIntLength(value: Long): Int { + val numMagnitudeBitsRequired: Int + if (value < 0) { + val numLeadingOnes = numberOfLeadingZeros(value.inv()) + numMagnitudeBitsRequired = 64 - numLeadingOnes + } else { + val numLeadingZeros = numberOfLeadingZeros(value) + numMagnitudeBitsRequired = 64 - numLeadingZeros + } + return numMagnitudeBitsRequired / 8 + 1 + } +} diff --git a/src/main/java/com/amazon/ion/impl/bin/IonEncoder_1_1.java b/src/main/java/com/amazon/ion/impl/bin/IonEncoder_1_1.java index c2629408f..0276c8278 100644 --- a/src/main/java/com/amazon/ion/impl/bin/IonEncoder_1_1.java +++ b/src/main/java/com/amazon/ion/impl/bin/IonEncoder_1_1.java @@ -97,7 +97,7 @@ public static int writeIntValue(WriteBuffer buffer, final long value) { buffer.writeByte(OpCodes.INTEGER_ZERO_LENGTH); return 1; } - int length = WriteBuffer.fixedIntLength(value); + int length = FixedInt.fixedIntLength(value); buffer.writeByte((byte) (OpCodes.INTEGER_ZERO_LENGTH + length)); buffer.writeFixedInt(value); return 1 + length; diff --git a/src/main/java/com/amazon/ion/impl/bin/PresenceBitmap.kt b/src/main/java/com/amazon/ion/impl/bin/PresenceBitmap.kt new file mode 100644 index 000000000..ac49259b7 --- /dev/null +++ b/src/main/java/com/amazon/ion/impl/bin/PresenceBitmap.kt @@ -0,0 +1,236 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.impl.bin + +import com.amazon.ion.* +import com.amazon.ion.impl.macro.* +import com.amazon.ion.impl.macro.Macro.* + +/** + * Utility class for setting, storing, reading, and writing presence bits. + * + * This class provides an API that maps 1:1 with parameters, with a maximum of 128 parameters. + * + * ### Usage – Binary Writer + * When stepping into an E-Expression, obtain a [PresenceBitmap] instance, [initialize] using the macro signature, and + * then reserve the correct number of bytes (see [byteSize]) to later encode the presence bits. + * While in the E-Expression, track the number of expressions or expression groups that have been written with that + * E-Expression as the immediate parent—this is the _parameter_ index. For each expression or expression group that is + * written directly in that container, call [PresenceBitmap.set] with the _parameter_ index and one of [VOID], + * [EXPRESSION], or [GROUP]. To omit an argument, callers to the binary writer will need to write an empty expression + * group (which should be elided and the corresponding presence bits set to `00`) or the binary writer must expose a + * `writeNoExpression()` method or similar. + * When stepping out of the E-Expression, use [PresenceBitmap.writeTo] to encode them into the appropriate location. + * + * ### Usage – Binary Reader + * When stepping into an E-Expression, obtain a [PresenceBitmap] instance, [initialize] using the macro signature, ensure + * that [byteSize] number of bytes is available in the reader's buffer, and call [readFrom] to populate the + * [PresenceBitmap] instance. Then, the presence bits for each parameter can be accessed by its _parameter_ index. + * + * ### Implementation Notes + * + * - We pretend that all parameters (including `!` (required) parameter) will get presence bits, and when reading we + * set the bits for the positions of the `!` parameters to `01` (single expression). + * - Since all the parameter cardinalities (other than `!`) use the same presence bit semantics, the writer doesn't + * need to inspect the signature to figure out what bits to put in our presence bits buffer. + * - Because we have dummy bits for `!` parameters, [PresenceBits] can present an API that corresponds 1:1 with + * parameters, so we don't need to separately keep track of a presence bit index and the parameter count. + * - Why longs instead of an array? + * - An array would add another level of indirection + * - An array would require a loop in order to reset all the bytes to zero. + * - Why only 128 parameters? + * - Until proven otherwise, we should not assume that an arbitrarily large number of parameters MUST be supported. + * - The number of parameters could be increased (within limits). It seems reasonable to try to keep this class small + * enough to fit in a single cache line for a modern system—typically 64 bytes. + * + * TODO: Consider whether we can "compile" a specific function that can read the presence bits when we compile a macro. + * That _might_ be more efficient than this approach. + */ +internal class PresenceBitmap { + + companion object { + const val VOID = 0b00L + const val EXPRESSION = 0b01L + const val GROUP = 0b10L + const val RESERVED = 0b11L + + private const val TWO_BIT_MASK = 0b11L + private const val PRESENCE_BITS_SIZE_THRESHOLD = 2 + private const val PB_SLOTS_PER_BYTE = 4 + private const val PB_SLOTS_PER_LONG = 32 + private const val PB_BITS_PER_SLOT = 2 + + const val MAX_SUPPORTED_PARAMETERS = PB_SLOTS_PER_LONG * 4 + } + + private var signature: List = emptyList() + + /** The number of parameters for which presence bits must be written. */ + private var size: Int = 0 + + /** The total number of parameters in the macro signature */ + val totalParameterCount: Int + get() = signature.size + + /** The first 32 presence bits slots */ + private var a: Long = 0 + /** The second 32 presence bits slots */ + private var b: Long = 0 + /** The third 32 presence bits slots */ + private var c: Long = 0 + /** The fourth 32 presence bits slots */ + private var d: Long = 0 + + /** The number of bytes required to encode this [PresenceBitmap] */ + val byteSize: Int + get() = size divideByRoundingUp PB_SLOTS_PER_BYTE + + /** Resets this [PresenceBitmap] for the given [macro]. */ + fun initialize(signature: List) { + if (signature.size > MAX_SUPPORTED_PARAMETERS) throw IonException("Macros with more than 128 parameters are not supported by this implementation.") + this.signature = signature + a = 0 + b = 0 + c = 0 + d = 0 + // TODO – performance: consider calculating this once for a macro when it is compiled + // Calculate the actual number of presence bits that will be encoded for the given signature. + val nonRequiredParametersCount = signature.count { it.cardinality != ParameterCardinality.One } + val usePresenceBits = nonRequiredParametersCount > PRESENCE_BITS_SIZE_THRESHOLD || signature.any { it.type.isTagless } + size = if (usePresenceBits) nonRequiredParametersCount else 0 + } + + /** + * Checks that all presence bits are valid for their corresponding parameters. + * Throws [IonException] if any are not. + */ + fun validate() { + val parameters = signature.iterator() + var i = 0 + while (parameters.hasNext()) { + val p = parameters.next() + val v = getUnchecked(i++) + val isValid = when (p.cardinality) { + ParameterCardinality.AtMostOne -> v == VOID || v == EXPRESSION + ParameterCardinality.One -> v == EXPRESSION + ParameterCardinality.AtLeastOne -> v == EXPRESSION || v == GROUP + ParameterCardinality.Any -> v != RESERVED + } + if (!isValid) throw IonException("Invalid argument for parameter: $p") + } + } + + /** + * Populates this [PresenceBitmap] from the given [ByteArray] that is positioned on the first + * byte that (potentially) contains presence bits. + * + * When complete, the buffer is positioned on the first byte that does not contain presence bits. + */ + fun readFrom(bytes: ByteArray, startInclusive: Int) { + // Doesn't always contain the full byte. We shift the bits over every time we read a value + // so that the next value is always the least significant bits. + var currentByte: Long = -1 + var currentPosition: Int = startInclusive + var bitmapIndex = 0 + var i = 0 + + val parameters = signature.iterator() + while (parameters.hasNext()) { + val p = parameters.next() + if (p.cardinality == ParameterCardinality.One) { + setUnchecked(i++, EXPRESSION) + } else { + if (bitmapIndex % PB_SLOTS_PER_BYTE == 0) { + currentByte = bytes[currentPosition++].toLong() + } + setUnchecked(i++, currentByte and TWO_BIT_MASK) + currentByte = currentByte shr PB_BITS_PER_SLOT + bitmapIndex++ + } + } + } + + /** + * Gets by _parameter_ index, which includes _required_ parameters that have no presence bits. + * The slots corresponding to a required parameter with always return [RESERVED]. + */ + operator fun get(index: Int): Long { + if (index >= totalParameterCount || index < 0) throw IndexOutOfBoundsException("$index") + return getUnchecked(index) + } + + /** Gets a presence bits "slot" without any bounds checking. See [get]. */ + private inline fun getUnchecked(index: Int): Long { + val shift = (index % PB_SLOTS_PER_LONG) * PB_BITS_PER_SLOT + when (index / PB_SLOTS_PER_LONG) { + 0 -> return (a shr shift) and TWO_BIT_MASK + 1 -> return (b shr shift) and TWO_BIT_MASK + 2 -> return (c shr shift) and TWO_BIT_MASK + 3 -> return (d shr shift) and TWO_BIT_MASK + else -> TODO("Unreachable") + } + } + + /** + * Sets a presence bits "slot" using bitwise OR with the existing contents. + * + * It is not possible to reset individual presence bits, nor + * is it possible to change the presence bits for a required parameter. + */ + operator fun set(index: Int, value: Long) { + if (index >= totalParameterCount || index < 0) throw IndexOutOfBoundsException("$index") + setUnchecked(index, value) + } + + /** Sets a presence bits "slot" without any bounds checking. See [set]. */ + private inline fun setUnchecked(index: Int, value: Long) { + val shiftedBits = (value shl ((index % PB_SLOTS_PER_LONG) * PB_BITS_PER_SLOT)) + when (index / PB_SLOTS_PER_LONG) { + 0 -> a = a or shiftedBits + 1 -> b = b or shiftedBits + 2 -> c = c or shiftedBits + 3 -> d = d or shiftedBits + } + } + + /** + * Writes this [PresenceBitmap] to [buffer] at the given [position]. + */ + fun writeTo(buffer: WriteBuffer, position: Long) { + if (size == 0) return + var resultBuffer: Long = 0 + var resultPosition = 0 + var writePosition = position + var i = 0 + val parameters = signature.iterator() + + while (parameters.hasNext()) { + val parameter = parameters.next() + val bits = getUnchecked(i++) + if (parameter.cardinality == ParameterCardinality.One) continue + val destShift = resultPosition * PB_BITS_PER_SLOT + resultBuffer = resultBuffer or (bits shl destShift) + resultPosition++ + if (resultPosition == PB_SLOTS_PER_LONG) { + buffer.writeFixedIntOrUIntAt(writePosition, resultBuffer, Long.SIZE_BYTES) + writePosition += Long.SIZE_BYTES + resultPosition = 0 + resultBuffer = 0 + } + } + + val numBytes = resultPosition divideByRoundingUp PB_SLOTS_PER_BYTE + if (numBytes > 0) buffer.writeFixedIntOrUIntAt(writePosition, resultBuffer, numBytes) + } + + /** + * Integer division that rounds up instead of down. + * E.g.: + * - 0/4 = 0 + * - 1/4 = 1 + * - ... + * - 4/4 = 1 + * - 5/4 = 2 + */ + private infix fun Int.divideByRoundingUp(other: Int): Int = (this + (other - 1)) / other +} diff --git a/src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java b/src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java index 973bd717f..29a824cb0 100644 --- a/src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java @@ -1408,33 +1408,13 @@ public int writeFlexUInt(final BigInteger value) { return numBytes; } - /** Get the length of FixedInt for the provided value. */ - public static int fixedIntLength(final long value) { - int numMagnitudeBitsRequired; - if (value < 0) { - int numLeadingOnes = Long.numberOfLeadingZeros(~value); - numMagnitudeBitsRequired = 64 - numLeadingOnes; - } else { - int numLeadingZeros = Long.numberOfLeadingZeros(value); - numMagnitudeBitsRequired = 64 - numLeadingZeros; - } - return numMagnitudeBitsRequired / 8 + 1; - } - /** * Writes a FixedInt to this WriteBuffer, using the minimum number of bytes needed to represent the number. * Returns the number of bytes that were needed to encode the value. */ public int writeFixedInt(final long value) { - int numBytes = fixedIntLength(value); - return _writeFixedIntOrUInt(value, numBytes); - } - - /** Get the length of FixedUInt for the provided value. */ - public static int fixedUIntLength(final long value) { - int numLeadingZeros = Long.numberOfLeadingZeros(value); - int numMagnitudeBitsRequired = 64 - numLeadingZeros; - return (numMagnitudeBitsRequired - 1) / 8 + 1; + int numBytes = FixedInt.fixedIntLength(value); + return writeFixedIntOrUInt(value, numBytes); } /** @@ -1445,7 +1425,7 @@ public int writeFixedUInt(final long value) { if (value < 0) { throw new IllegalArgumentException("Attempted to write a FixedUInt for " + value); } - int numBytes = fixedUIntLength(value); + int numBytes = FixedInt.fixedUIntLength(value); return _writeFixedIntOrUInt(value, numBytes); } @@ -1502,6 +1482,27 @@ private int _writeFixedIntOrUInt(final long value, final int numBytes) { return numBytes; } + /** + * Writes a FixedInt or FixedUInt to this WriteBuffer at the specified position. + * If the allocator's block size is ever less than 8 bytes, this may throw an IndexOutOfBoundsException. + */ + public void writeFixedIntOrUIntAt(final long position, final long value, final int numBytes) { + int index = index(position); + Block block = blocks.get(index); + int dataOffset = offset(position); + if (dataOffset + numBytes < block.capacity()) { + FixedInt.writeFixedIntOrUIntInto(block.data, dataOffset, value, numBytes); + } else { + FixedInt.writeFixedIntOrUIntInto(scratch, 0, value, numBytes); + if (index == blocks.size() - 1) { + allocateNewBlock(); + } + for (int i = 0; i < numBytes; i++) { + writeByteAt(position + i, scratch[i]); + } + } + } + /** * Writes a FixedInt or FixedUInt for an arbitrarily large integer that is represented * as a byte array in which the most significant byte is the first in the array, and the least diff --git a/src/main/java/com/amazon/ion/impl/macro/Macro.kt b/src/main/java/com/amazon/ion/impl/macro/Macro.kt index 19e9322a7..6f7618097 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Macro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Macro.kt @@ -8,14 +8,19 @@ package com.amazon.ion.impl.macro sealed interface Macro { val signature: List - data class Parameter(val variableName: String, val type: ParameterEncoding, val cardinality: ParameterCardinality) + data class Parameter(val variableName: String, val type: ParameterEncoding, val cardinality: ParameterCardinality) { + override fun toString() = "$type::$variableName${cardinality.sigil}" + } - enum class ParameterEncoding(val ionTextName: String) { + enum class ParameterEncoding(val ionTextName: String, val isTagless: Boolean = false) { Tagged("any"), + uint8("uint8", isTagless = true), // TODO: List all of the possible tagless encodings + // TODO: Update this to support macro shapes } enum class ParameterCardinality(val sigil: Char) { + // TODO: Rename to match spec. AtMostOne('?'), One('!'), AtLeastOne('+'), diff --git a/src/test/java/com/amazon/ion/TestUtils.java b/src/test/java/com/amazon/ion/TestUtils.java index 5ecd661c7..f3953cc5a 100644 --- a/src/test/java/com/amazon/ion/TestUtils.java +++ b/src/test/java/com/amazon/ion/TestUtils.java @@ -628,6 +628,7 @@ public static int byteLengthFromHexString(String hexString) { * @return a new byte array. */ private static byte[] octetStringToByteArray(String octetString, int radix) { + if (octetString.isEmpty()) return new byte[0]; String[] bytesAsStrings = octetString.split(" "); byte[] bytesAsBytes = new byte[bytesAsStrings.length]; for (int i = 0; i < bytesAsBytes.length; i++) { diff --git a/src/test/java/com/amazon/ion/impl/bin/PresenceBitmapTest.kt b/src/test/java/com/amazon/ion/impl/bin/PresenceBitmapTest.kt new file mode 100644 index 000000000..67d9c245e --- /dev/null +++ b/src/test/java/com/amazon/ion/impl/bin/PresenceBitmapTest.kt @@ -0,0 +1,333 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.impl.bin + +import com.amazon.ion.IonException +import com.amazon.ion.TestUtils.* +import com.amazon.ion.impl.macro.Macro.* +import java.io.ByteArrayOutputStream +import org.junit.Test +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.CsvSource + +class PresenceBitmapTest { + + companion object { + val taggedZeroToMany = Parameter("a", ParameterEncoding.Tagged, ParameterCardinality.Any) + val taggedExactlyOne = Parameter("b", ParameterEncoding.Tagged, ParameterCardinality.One) + val taggedZeroOrOne = Parameter("c", ParameterEncoding.Tagged, ParameterCardinality.AtMostOne) + val taggedOneToMany = Parameter("d", ParameterEncoding.Tagged, ParameterCardinality.AtLeastOne) + val taglessZeroToMany = Parameter("e", ParameterEncoding.uint8, ParameterCardinality.Any) + val taglessExactlyOne = Parameter("f", ParameterEncoding.uint8, ParameterCardinality.One) + val taglessZeroOrOne = Parameter("g", ParameterEncoding.uint8, ParameterCardinality.AtMostOne) + val taglessOneToMany = Parameter("h", ParameterEncoding.uint8, ParameterCardinality.AtLeastOne) + } + + @Test + fun `initialize should ensure that values are cleared`() { + val signature = listOf( + taggedExactlyOne, taggedZeroToMany, taggedZeroOrOne, taggedOneToMany, + taglessExactlyOne, taglessZeroToMany, taglessZeroOrOne, taglessOneToMany, + ) + val pb = PresenceBitmap() + pb.initialize(signature) + for (i in 0..7) pb[i] = PresenceBitmap.EXPRESSION + pb.initialize(signature) + for (i in 0..7) assertEquals(PresenceBitmap.VOID, pb[i]) + assertThrows { pb[8] } + } + + @Test + fun `when initializing with a too-large signature, should throw exception`() { + val pb = PresenceBitmap() + val signature = List(PresenceBitmap.MAX_SUPPORTED_PARAMETERS + 1) { taggedZeroToMany } + assertThrows { pb.initialize(signature) } + } + + @Test + fun `when calling set with an invalid index, should throw exception`() { + val pb = PresenceBitmap() + val signature = listOf(taggedZeroOrOne, taggedOneToMany, taggedExactlyOne) + pb.initialize(signature) + assertThrows { pb.set(-1, PresenceBitmap.EXPRESSION) } + assertThrows { pb.set(3, PresenceBitmap.EXPRESSION) } + } + + @Test + fun `when calling get with an invalid index, should throw exception`() { + val pb = PresenceBitmap() + val signature = listOf(taggedZeroOrOne, taggedOneToMany, taggedExactlyOne) + pb.initialize(signature) + assertThrows { pb.get(-1) } + assertThrows { pb.get(3) } + } + + @Test + fun `when calling set, the presence bits value for that parameter is _not_ validated`() { + val signature = listOf(taggedZeroOrOne, taggedOneToMany, taggedExactlyOne) + with(PresenceBitmap()) { + initialize(signature) + // PresenceBits is an internal only class, so we rely on callers to do the correct thing. + // There should not be an exception thrown for any of these. + set(0, value = PresenceBitmap.GROUP) + set(1, value = PresenceBitmap.VOID) + set(2, value = PresenceBitmap.GROUP) + set(2, value = PresenceBitmap.VOID) + } + } + + @ParameterizedTest + @CsvSource( + // For some reason `Long.decode()` doesn't support binary, so + // we're just using decimal for the presence values here. + "One, 0, false", + "One, 1, true", + "One, 2, false", + "One, 3, false", + "Any, 0, true", + "Any, 1, true", + "Any, 2, true", + "Any, 3, false", + "AtMostOne, 0, true", + "AtMostOne, 1, true", + "AtMostOne, 2, false", + "AtMostOne, 3, false", + "AtLeastOne, 0, false", + "AtLeastOne, 1, true", + "AtLeastOne, 2, true", + "AtLeastOne, 3, false", + ) + fun `validate() correctly throws exception when presence bits are invalid for signature`(cardinality: ParameterCardinality, presenceValue: Long, isValid: Boolean) { + val signature = listOf(Parameter("a", ParameterEncoding.uint8, cardinality)) + with(PresenceBitmap()) { + initialize(signature) + set(0, presenceValue) + if (isValid) { + validate() + } else { + assertThrows { validate() } + } + } + } + + @Test + fun `when all parameters are tagged and exactly-one, no presence bits are needed or written`() { + (0..128).forEach { n -> assertExpectedPresenceBitSizes(expectedByteSize = 0, signature = List(n) { taggedExactlyOne }) } + } + + @Test + fun `when all parameters are tagless and exactly-one, no presence bits are needed or written`() { + (0..128).forEach { n -> assertExpectedPresenceBitSizes(expectedByteSize = 0, signature = List(n) { taglessExactlyOne }) } + } + + @Test + fun `when all parameters are tagged and not exactly-one, should write expected number of presence bits`() { + // Index of an element in this list is the number of parameters in the signature + listOf(0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) + .forEachIndexed { numParameters, expectedByteSize -> + assertExpectedPresenceBitSizes(expectedByteSize, signature = List(numParameters) { taggedZeroToMany }) + } + } + + @Test + fun `when all parameters are tagless and not exactly-one, should write expected number of presence bits`() { + // Index of an element in this list is the number of parameters in the signature + listOf(0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3) + .forEachIndexed { numParameters, expectedByteSize -> + assertExpectedPresenceBitSizes(expectedByteSize, signature = List(numParameters) { taglessZeroToMany }) + } + } + + private fun assertExpectedPresenceBitSizes(expectedByteSize: Int, signature: List) { + val result = writePresenceBits { pb -> + pb.initialize(signature) + assertEquals(expectedByteSize, pb.byteSize) + assertEquals(signature.size, pb.totalParameterCount) + } + assertEquals(expectedByteSize, result.size) + } + + @Test + fun `read 4 parameters`() { + val signature = listOf(taggedZeroToMany, taggedZeroOrOne, taggedExactlyOne, taggedZeroToMany) + // Bits are read in pairs from right to left + // There are only three pairs of presence bits because "exactly-one" parameters do not get presence bits. + val bytes = bitStringToByteArray("100010") + + val pb = PresenceBitmap() + pb.initialize(signature) + pb.readFrom(bytes, 0) + + assertEquals(PresenceBitmap.GROUP, pb[0]) + assertEquals(PresenceBitmap.VOID, pb[1]) + // Should automatically populate the value for the exactly-one parameter + assertEquals(PresenceBitmap.EXPRESSION, pb[2]) + assertEquals(PresenceBitmap.GROUP, pb[3]) + } + + @Test + fun `write 4 parameters`() { + val signature = listOf(taggedZeroToMany, taggedZeroOrOne, taggedExactlyOne, taggedZeroToMany) + + val result = writePresenceBits { pb -> + pb.initialize(signature) + pb[0] = PresenceBitmap.EXPRESSION + pb[1] = PresenceBitmap.GROUP + pb[2] = PresenceBitmap.EXPRESSION + pb[3] = PresenceBitmap.GROUP + } + + assertEquals("00101001", result.toBitString()) + } + + @Test + fun `write presence bitmap`() { + // Ensures that the bits are written in the correct order for all possible sizes + (PresenceBitmap.MAX_SUPPORTED_PARAMETERS downTo 0).forEach { signatureSize -> + val signature = List(signatureSize) { taglessZeroToMany } + (0 until signatureSize).forEach { i -> + val parameterPresences = List(signatureSize) { j -> if (i == j) PresenceBitmap.GROUP else PresenceBitmap.EXPRESSION } + val expected = createBitStringFromParameterPresences(parameterPresences) + val actual = writePresenceBits { pb -> + pb.initialize(signature) + parameterPresences.forEachIndexed(pb::set) + } + assertEquals(expected, actual.toBitString()) + } + } + } + + @Test + fun `read presence bitmap`() { + // Ensures that the bits are read using the correct order + (PresenceBitmap.MAX_SUPPORTED_PARAMETERS downTo 0).forEach { signatureSize -> + val signature = List(signatureSize) { taglessZeroToMany } + (0 until signatureSize).forEach { i -> + val parameterPresences = List(signatureSize) { j -> if (i == j) PresenceBitmap.GROUP else PresenceBitmap.EXPRESSION } + val inputBits = bitStringToByteArray(createBitStringFromParameterPresences(parameterPresences)) + + val pb = PresenceBitmap() + pb.initialize(signature) + pb.readFrom(inputBits, 0) + + parameterPresences.forEachIndexed { l, expected -> assertEquals(expected, pb[l]) } + } + } + } + + @Test + fun `write presence bitmap with a required parameter`() { + // Ensures that the bits are read using the correct order + (PresenceBitmap.MAX_SUPPORTED_PARAMETERS downTo 0).forEach { signatureSize -> + (0 until signatureSize).forEach { i -> + val signature = List(signatureSize) { j -> if (j == i) taglessExactlyOne else taglessZeroToMany } + val parameterPresences = List(signatureSize) { j -> + when { + j < i -> PresenceBitmap.RESERVED + j == i -> PresenceBitmap.EXPRESSION + j > i -> PresenceBitmap.GROUP + else -> TODO("Unreachable") + } + } + val expected = createBitStringFromParameterPresences(parameterPresences.filter { it != PresenceBitmap.EXPRESSION }) + + val actual = writePresenceBits { pb -> + pb.initialize(signature) + parameterPresences.forEachIndexed(pb::set) + } + + assertEquals(expected, actual.toBitString()) + } + } + } + + @Test + fun `read presence bitmap with a required parameter`() { + // Ensures that the bits are read using the correct order + (PresenceBitmap.MAX_SUPPORTED_PARAMETERS downTo 0).forEach { signatureSize -> + (0 until signatureSize).forEach { i -> + val signature = List(signatureSize) { j -> if (j == i) taglessExactlyOne else taglessZeroToMany } + val parameterPresences = List(signatureSize) { j -> + when { + j < i -> PresenceBitmap.RESERVED + j == i -> PresenceBitmap.EXPRESSION + j > i -> PresenceBitmap.GROUP + else -> TODO("Unreachable") + } + } + val inputBitString = createBitStringFromParameterPresences(parameterPresences.filter { it != PresenceBitmap.EXPRESSION }) + val inputBits = bitStringToByteArray(inputBitString) + + val pb = PresenceBitmap() + pb.initialize(signature) + pb.readFrom(inputBits, 0) + + parameterPresences.forEachIndexed { l, expected -> assertEquals(expected, pb[l]) } + } + } + } + + private fun writePresenceBits(action: (PresenceBitmap) -> Unit): ByteArray { + val pb = PresenceBitmap() + action(pb) + val buffer = WriteBuffer(BlockAllocatorProviders.basicProvider().vendAllocator(32)) {} + buffer.reserve(pb.byteSize) + pb.writeTo(buffer, 0) + return buffer.toByteArray() + } + + private fun WriteBuffer.toByteArray() = ByteArrayOutputStream().also { writeTo(it) }.toByteArray() + private fun ByteArray.toBitString(): String = byteArrayToBitString(this) + + @ParameterizedTest + @CsvSource( + " '', '' ", + " 0, 00000000", + " 0 0, 00000000", + " 0 0 0, 00000000", + " 0 0 0 0, 00000000", + "0 0 0 0 0, 00000000 00000000", + "1 0 0 0 0, 00000001 00000000", + "0 1 0 0 0, 00000100 00000000", + "0 0 1 0 0, 00010000 00000000", + "0 0 0 1 0, 01000000 00000000", + "0 0 0 0 1, 00000000 00000001", + "2 0 0 0 0, 00000010 00000000", + "0 2 0 0 0, 00001000 00000000", + "0 0 2 0 0, 00100000 00000000", + "0 0 0 2 0, 10000000 00000000", + "0 0 0 0 2, 00000000 00000010", + ) + fun testCreateBitStringFromParameterPresences(presences: String, expectedBitString: String) { + val presenceList = presences.takeIf { it.isNotBlank() }?.split(" ")?.map { it.toLong() } ?: emptyList() + assertEquals(expectedBitString, createBitStringFromParameterPresences(presenceList)) + } + + /** + * The purpose of this utility function is to create a bit string containing a whole number + * of little endian bytes that represents a list of presence bit pairs. + */ + private fun createBitStringFromParameterPresences(parameterPresences: List): String { + val sb = StringBuilder() + // Calculate the number of bit-pairs needed to have a whole number of bytes. + val n = (((parameterPresences.size + 3) / 4) * 4) + for (i in 0 until n) { + // Calculate the little-endian position + val ii = i - 2 * (i % 4) + 3 + // If `getOrNull` returns null, we've gone past the end of the presence values, so pad with zeros + val parameterPresence = parameterPresences.getOrNull(ii) ?: 0 + val bits = when (parameterPresence) { + 0L -> "00" + 1L -> "01" + 2L -> "10" + 3L -> "11" + else -> TODO("Unreachable") + } + sb.append(bits) + if (i % 4 == 3) sb.append(' ') + } + return sb.toString().trim() + } +}