Skip to content

Commit

Permalink
Adds Ion 1.1 system symbols (#939)
Browse files Browse the repository at this point in the history
  • Loading branch information
popematt authored Sep 19, 2024
1 parent 047e37a commit d380bf0
Show file tree
Hide file tree
Showing 7 changed files with 130 additions and 17 deletions.
8 changes: 4 additions & 4 deletions src/main/java/com/amazon/ion/impl/IonCursorBinary.java
Original file line number Diff line number Diff line change
Expand Up @@ -2284,8 +2284,8 @@ private void setUserMacroInvocationMarker(IonTypeID valueTid, Marker markerToSet
*/
private void uncheckedReadMacroInvocationHeader(IonTypeID valueTid, Marker markerToSet) {
if (valueTid.macroId < 0) {
if (valueTid.lowerNibble == 0xE) {
// Opcode 0xEE: Read the macro ID as a FlexUInt.
if (valueTid.lowerNibble == 0x4) {
// Opcode 0xF4: Read the macro ID as a FlexUInt.
macroInvocationId = uncheckedReadFlexUInt_1_1();
} else if (valueTid.variableLength) {
// Opcode 0xF5: Read the macro ID as a FlexUInt, then read the length as a FlexUInt.
Expand Down Expand Up @@ -2438,8 +2438,8 @@ private boolean slowReadHeader(final int typeIdByte, final boolean isAnnotated,
*/
private boolean slowReadMacroInvocationHeader(IonTypeID valueTid, Marker markerToSet, long macroId) {
if (valueTid.macroId < 0) {
if (valueTid.lowerNibble == 0xE) {
// Opcode 0xEE: Read the macro ID as a FlexUInt.
if (valueTid.lowerNibble == 0x4) {
// Opcode 0xF4: Read the macro ID as a FlexUInt.
macroInvocationId = slowReadFlexUInt_1_1();
if (macroInvocationId < 0) {
return true;
Expand Down
19 changes: 13 additions & 6 deletions src/main/java/com/amazon/ion/impl/IonTypeID.java
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,9 @@ private static boolean isValid_1_1(byte id) {
id == 0x69
|| id == (byte) 0xD1
|| id == (byte) 0xE0
|| id == (byte) 0xF4
|| id == (byte) 0x8D
|| id == (byte) 0x8E
|| id == (byte) 0x8F
);
}

Expand Down Expand Up @@ -263,7 +265,7 @@ private IonTypeID(byte id, int minorVersion) {
// just to identify this byte.
lowerNibble = (id == DELIMITED_END_MARKER) ? DELIMITED_END_MARKER : (byte) (id & LOW_NIBBLE_BITMASK);
isNegativeInt = false; // Not applicable for Ion 1.1; sign is conveyed by the representation.
isMacroInvocation = (id >= 0x00 && id <= 0x5F) || id == E_EXPRESSION_FLEX_UINT
isMacroInvocation = (id >= 0x00 && id <= 0x5F) || id == E_EXPRESSION_WITH_FLEX_UINT_ADDRESS
|| id == SYSTEM_MACRO_INVOCATION || id == LENGTH_PREFIXED_MACRO_INVOCATION;
boolean isNopPad = false;
boolean isNull = false;
Expand All @@ -286,8 +288,10 @@ private IonTypeID(byte id, int minorVersion) {
macroId = id;
length = 0;
} else {
if (upperNibble == 0xF) {
// FlexUInt length-prefixed macro invocation.
if (id == E_EXPRESSION_WITH_FLEX_UINT_ADDRESS) {
variableLength = false;
length = 1;
} else if (id == LENGTH_PREFIXED_MACRO_INVOCATION) {
variableLength = true;
} else {
// System invocation; ID follows as a 1-byte FixedInt.
Expand Down Expand Up @@ -355,6 +359,9 @@ private IonTypeID(byte id, int minorVersion) {
isNopPad = true;
type = null;
length = variableLength ? -1 : 0;
} else if (id == SYSTEM_SYMBOL) {
type = IonType.SYMBOL;
length = 1;
} else { // 0xF
// System macro invocation.
type = null;
Expand Down Expand Up @@ -383,8 +390,8 @@ private IonTypeID(byte id, int minorVersion) {
type = IonType.LIST;
} else if (id == DELIMITED_SEXP || id == VARIABLE_LENGTH_SEXP) {
type = IonType.SEXP;
} else { // 0x4
// Variable length macro invocation
} else { // 0x4, 0x5
// E-Expression with FlexUInt Address or E-Expression with FlexUInt Length
type = null;
}
}
Expand Down
106 changes: 106 additions & 0 deletions src/main/java/com/amazon/ion/impl/SystemSymbols_1_1.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
package com.amazon.ion.impl

import java.util.*

enum class SystemSymbols_1_1(val id: Int, val text: String) {
// System SID 0 is reserved.
ION( /* */ 1, "\$ion"),
ION_1_0( /* */ 2, "\$ion_1_0"),
ION_SYMBOL_TABLE( /* */ 3, "\$ion_symbol_table"),
NAME( /* */ 4, "name"),
VERSION( /* */ 5, "version"),
IMPORTS( /* */ 6, "imports"),
SYMBOLS( /* */ 7, "symbols"),
MAX_ID( /* */ 8, "max_id"),
ION_SHARED_SYMBOL_TABLE( /* */ 9, "\$ion_shared_symbol_table"),
ION_ENCODING( /* */ 10, "\$ion_encoding"),
ION_LITERAL( /* */ 11, "\$ion_literal"),
ION_SHARED_MODULE( /* */ 12, "\$ion_shared_module"),
MACRO( /* */ 13, "macro"),
MACRO_TABLE( /* */ 14, "macro_table"),
SYMBOL_TABLE( /* */ 15, "symbol_table"),
MODULE( /* */ 16, "module"),
RETAIN( /* */ 17, "retain"),
EXPORT( /* */ 18, "export"),
CATALOG_KEY( /* */ 19, "catalog_key"),
IMPORT( /* */ 20, "import"),
THE_EMPTY_SYMBOL( /* */ 21, ""),
LITERAL( /* */ 22, "literal"),
IF_NONE( /* */ 23, "if_none"),
IF_SOME( /* */ 24, "if_some"),
IF_SINGLE( /* */ 25, "if_single"),
IF_MULTI( /* */ 26, "if_multi"),
FOR( /* */ 27, "for"),
FAIL( /* */ 28, "fail"),
VALUES( /* */ 29, "values"),
ANNOTATE( /* */ 30, "annotate"),
MAKE_STRING( /* */ 31, "make_string"),
MAKE_SYMBOL( /* */ 32, "make_symbol"),
MAKE_BLOB( /* */ 33, "make_blob"),
MAKE_DECIMAL( /* */ 34, "make_decimal"),
MAKE_TIMESTAMP( /* */ 35, "make_timestamp"),
MAKE_LIST( /* */ 36, "make_list"),
MAKE_SEXP( /* */ 37, "make_sexp"),
MAKE_STRUCT( /* */ 38, "make_struct"),
PARSE_ION( /* */ 39, "parse_ion"),
REPEAT( /* */ 40, "repeat"),
DELTA( /* */ 41, "delta"),
FLATTEN( /* */ 42, "flatten"),
SUM( /* */ 43, "sum"),
ADD_SYMBOLS( /* */ 44, "add_symbols"),
ADD_MACROS( /* */ 45, "add_macros"),
COMMENT( /* */ 46, "comment"),
FLEX_SYMBOL( /* */ 47, "flex_symbol"),
FLEX_INT( /* */ 48, "flex_int"),
FLEX_UINT( /* */ 49, "flex_uint"),
UINT8( /* */ 50, "uint8"),
UINT16( /* */ 51, "uint16"),
UINT32( /* */ 52, "uint32"),
UINT64( /* */ 53, "uint64"),
INT8( /* */ 54, "int8"),
INT16( /* */ 55, "int16"),
INT32( /* */ 56, "int32"),
INT64( /* */ 57, "int64"),
FLOAT16( /* */ 58, "float16"),
FLOAT32( /* */ 59, "float32"),
FLOAT64( /* */ 60, "float64"),
;

val utf8Bytes = text.encodeToByteArray()

companion object {
private val ALL_VALUES: Array<SystemSymbols_1_1> = entries.toTypedArray().apply {
// Put all system symbol enum values into an array, and ensure that they are sorted by ID in that array.
// This allows us to have O(1) lookup, but it doesn't rely on the enum's ordinal value, which could change.
Arrays.sort(this) { o1, o2 -> o1.id.compareTo(o2.id) }
}
init {
// Initialization checks to make sure that the system symbols are not misconfigured.
ALL_VALUES
.map { it.id }
.zipWithNext { a, b ->
check(b - a > -1) { "System symbols not sorted. Found $a before $b." }
check(b - a != 0) { "Duplicate ID $a in system symbols" }
check(b - a == 1) { "Gap in system symbols between $a and $b" }
}
}

/**
* Returns true if the [id] is a valid system symbol ID.
*/
@JvmStatic
operator fun contains(id: Int): Boolean {
return id > 0 && id <= SystemSymbols_1_1.ALL_VALUES.size
}

/**
* Returns the text of the given system symbol ID, or null if not a valid system symbol ID.
*/
@JvmStatic
operator fun get(id: Int): String {
return SystemSymbols_1_1.ALL_VALUES[id - 1].text
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -661,7 +661,7 @@ class IonRawBinaryWriter_1_1 internal constructor(
buffer.writeByte((OpCodes.BIASED_E_EXPRESSION_TWO_BYTE_FIXED_INT + lowNibble).toByte())
currentContainer.metadataOffset += buffer.writeFixedIntOrUInt(adjustedId, 2)
} else {
buffer.writeByte(OpCodes.E_EXPRESSION_FLEX_UINT)
buffer.writeByte(OpCodes.E_EXPRESSION_WITH_FLEX_UINT_ADDRESS)
currentContainer.metadataOffset += buffer.writeFlexUInt(id)
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/amazon/ion/impl/bin/OpCodes.java
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,14 @@ private OpCodes() {}
public static final byte NULL_TYPED = (byte) 0xEB;
public static final byte ONE_BYTE_NOP = (byte) 0xEC;
public static final byte VARIABLE_LENGTH_NOP = (byte) 0xED;
public static final byte E_EXPRESSION_FLEX_UINT = (byte) 0xEE;
public static final byte SYSTEM_SYMBOL = (byte) 0xEE;
public static final byte SYSTEM_MACRO_INVOCATION = (byte) 0xEF;

public static final byte DELIMITED_END_MARKER = (byte) 0xF0;
public static final byte DELIMITED_LIST = (byte) 0xF1;
public static final byte DELIMITED_SEXP = (byte) 0xF2;
public static final byte DELIMITED_STRUCT = (byte) 0xF3;
// 0xF4 Reserved
public static final byte E_EXPRESSION_WITH_FLEX_UINT_ADDRESS = (byte) 0xF4;
public static final byte LENGTH_PREFIXED_MACRO_INVOCATION = (byte) 0xF5;
public static final byte VARIABLE_LENGTH_INTEGER = (byte) 0xF6;
public static final byte VARIABLE_LENGTH_DECIMAL = (byte) 0xF7;
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/com/amazon/ion/impl/IonCursorBinaryTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -823,8 +823,8 @@ public void macroInvocationWithTwoByteFixedUIntId(InputType inputType) throws Ex
@ParameterizedTest(name = "inputType={0}")
@EnumSource(InputType.class)
public void macroInvocationWithFlexUIntId(InputType inputType) throws Exception {
// Opcode 0xEE; 3-byte FlexUInt 0xFC, 0xFF, 0xFF follows
testMacroInvocation(bytes(0xEE, 0xFC, 0xFF, 0xFF), inputType, 8, -1, 2097151, false);
// Opcode 0xF4; 3-byte FlexUInt 0xFC, 0xFF, 0xFF follows
testMacroInvocation(bytes(0xF4, 0xFC, 0xFF, 0xFF), inputType, 8, -1, 2097151, false);
}

@ParameterizedTest(name = "inputType={0}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1073,8 +1073,8 @@ class IonRawBinaryWriterTest_1_1 {
" 69695, 50 FF FF",
" 69696, 51 00 00",
" 1052735, 5F FF FF",
" 1052736, EE 04 82 80",
"${Int.MAX_VALUE}, EE F0 FF FF FF 0F"
" 1052736, F4 04 82 80",
"${Int.MAX_VALUE}, F4 F0 FF FF FF 0F"
)
fun `write a delimited e-expression with a multi-byte biased id`(id: Int, expectedBytes: String) {
assertWriterOutputEquals(expectedBytes) {
Expand Down

0 comments on commit d380bf0

Please sign in to comment.