Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Alternative implementation of cursors #313

Open
wants to merge 6 commits into
base: performance
Choose a base branch
from
Open

Alternative implementation of cursors #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
170c069
Rename chld columns to ref; remove link operation
ept Feb 2, 2021
08fd595
Creating and storing Cursor objects
ept Feb 2, 2021
64d10e6
Bounds checking when creating cursor
ept Feb 3, 2021
37d7311
Compute indexes for cursors
ept Feb 3, 2021
00293ab
Nicer error message
ept Feb 3, 2021
75bcd96
Merge branch 'performance' into cursors-v2
ept Feb 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 33 additions & 32 deletions backend/columnar.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,12 @@ const COLUMN_TYPE = {

const VALUE_TYPE = {
NULL: 0, FALSE: 1, TRUE: 2, LEB128_UINT: 3, LEB128_INT: 4, IEEE754: 5,
UTF8: 6, BYTES: 7, COUNTER: 8, TIMESTAMP: 9, MIN_UNKNOWN: 10, MAX_UNKNOWN: 15
UTF8: 6, BYTES: 7, COUNTER: 8, TIMESTAMP: 9, CURSOR: 10,
MIN_UNKNOWN: 11, MAX_UNKNOWN: 15
}

// make* actions must be at even-numbered indexes in this list
const ACTIONS = ['makeMap', 'set', 'makeList', 'del', 'makeText', 'inc', 'makeTable', 'link']
const ACTIONS = ['makeMap', 'set', 'makeList', 'del', 'makeText', 'inc', 'makeTable']

const OBJECT_TYPE = {makeMap: 'map', makeList: 'list', makeText: 'text', makeTable: 'table'}

Expand All @@ -49,8 +50,8 @@ const COMMON_COLUMNS = {
action: 4 << 3 | COLUMN_TYPE.INT_RLE,
valLen: 5 << 3 | COLUMN_TYPE.VALUE_LEN,
valRaw: 5 << 3 | COLUMN_TYPE.VALUE_RAW,
chldActor: 6 << 3 | COLUMN_TYPE.ACTOR_ID,
chldCtr: 6 << 3 | COLUMN_TYPE.INT_DELTA
refActor: 6 << 3 | COLUMN_TYPE.ACTOR_ID,
refCtr: 6 << 3 | COLUMN_TYPE.INT_RLE
}

const CHANGE_COLUMNS = Object.assign({
Expand Down Expand Up @@ -129,11 +130,11 @@ function parseAllOpIds(changes, single) {
op = copyObject(op)
if (op.obj !== '_root') op.obj = parseOpId(op.obj)
if (op.elemId && op.elemId !== '_head') op.elemId = parseOpId(op.elemId)
if (op.child) op.child = parseOpId(op.child)
if (op.ref) op.ref = parseOpId(op.ref)
if (op.pred) op.pred = op.pred.map(parseOpId)
if (op.obj.actorId) actors[op.obj.actorId] = true
if (op.elemId && op.elemId.actorId) actors[op.elemId.actorId] = true
if (op.child && op.child.actorId) actors[op.child.actorId] = true
if (op.ref && op.ref.actorId) actors[op.ref.actorId] = true
for (let pred of op.pred) actors[pred.actorId] = true
return op
})
Expand All @@ -151,7 +152,7 @@ function parseAllOpIds(changes, single) {
op.id = {counter: change.startOp + i, actorNum: change.actorNum, actorId: change.actor}
op.obj = actorIdToActorNum(op.obj, actorIds)
op.elemId = actorIdToActorNum(op.elemId, actorIds)
op.child = actorIdToActorNum(op.child, actorIds)
op.ref = actorIdToActorNum(op.ref, actorIds)
op.pred = op.pred.map(pred => actorIdToActorNum(pred, actorIds))
}
}
Expand Down Expand Up @@ -239,16 +240,18 @@ function encodeValue(op, columns) {
columns.valLen.appendValue(VALUE_TYPE.FALSE)
} else if (op.value === true) {
columns.valLen.appendValue(VALUE_TYPE.TRUE)
} else if (typeof op.value === 'string') {
const numBytes = columns.valRaw.appendRawString(op.value)
columns.valLen.appendValue(numBytes << 4 | VALUE_TYPE.UTF8)
} else if (ArrayBuffer.isView(op.value)) {
const numBytes = columns.valRaw.appendRawBytes(new Uint8Array(op.value.buffer))
columns.valLen.appendValue(numBytes << 4 | VALUE_TYPE.BYTES)
} else if (op.datatype === 'counter' && typeof op.value === 'number') {
encodeInteger(op.value, VALUE_TYPE.COUNTER, columns)
} else if (op.datatype === 'timestamp' && typeof op.value === 'number') {
encodeInteger(op.value, VALUE_TYPE.TIMESTAMP, columns)
} else if (op.datatype === 'cursor') {
columns.valLen.appendValue(VALUE_TYPE.CURSOR) // cursor's elemId is stored in the ref columns
} else if (typeof op.value === 'string') {
const numBytes = columns.valRaw.appendRawString(op.value)
columns.valLen.appendValue(numBytes << 4 | VALUE_TYPE.UTF8)
} else if (typeof op.datatype === 'number' && op.datatype >= VALUE_TYPE.MIN_UNKNOWN &&
op.datatype <= VALUE_TYPE.MAX_UNKNOWN && op.value instanceof Uint8Array) {
const numBytes = columns.valRaw.appendRawBytes(op.value)
Expand Down Expand Up @@ -310,6 +313,8 @@ function decodeValue(sizeTag, bytes) {
return {value: new Decoder(bytes).readInt53(), datatype: 'counter'}
} else if (sizeTag % 16 === VALUE_TYPE.TIMESTAMP) {
return {value: new Decoder(bytes).readInt53(), datatype: 'timestamp'}
} else if (sizeTag === VALUE_TYPE.CURSOR) {
return {value: '', datatype: 'cursor'}
} else {
return {value: bytes, datatype: sizeTag % 16}
}
Expand Down Expand Up @@ -366,8 +371,8 @@ function encodeOps(ops, forDocument) {
action : new RLEEncoder('uint'),
valLen : new RLEEncoder('uint'),
valRaw : new Encoder(),
chldActor : new RLEEncoder('uint'),
chldCtr : new DeltaEncoder()
refActor : new RLEEncoder('uint'),
refCtr : new RLEEncoder('uint')
}

if (forDocument) {
Expand All @@ -389,12 +394,12 @@ function encodeOps(ops, forDocument) {
encodeOperationAction(op, columns)
encodeValue(op, columns)

if (op.child && op.child.counter) {
columns.chldActor.appendValue(op.child.actorNum)
columns.chldCtr.appendValue(op.child.counter)
if (op.ref && op.ref.counter) {
columns.refActor.appendValue(op.ref.actorNum)
columns.refCtr.appendValue(op.ref.counter)
} else {
columns.chldActor.appendValue(null)
columns.chldCtr.appendValue(null)
columns.refActor.appendValue(null)
columns.refCtr.appendValue(null)
}

if (forDocument) {
Expand Down Expand Up @@ -439,10 +444,10 @@ function decodeOps(ops, forDocument) {
newOp.value = op.valLen
if (op.valLen_datatype) newOp.datatype = op.valLen_datatype
}
if (!!op.chldCtr !== !!op.chldActor) {
throw new RangeError(`Mismatched child columns: ${op.chldCtr} and ${op.chldActor}`)
if (!!op.refCtr !== !!op.refActor) {
throw new RangeError(`Mismatched ref columns: ${op.refCtr} and ${op.refActor}`)
}
if (op.chldCtr !== null) newOp.child = `${op.chldCtr}@${op.chldActor}`
if (op.refCtr !== null) newOp.ref = `${op.refCtr}@${op.refActor}`
if (forDocument) {
newOp.id = `${op.idCtr}@${op.idActor}`
newOp.succ = op.succNum.map(succ => `${succ.succCtr}@${succ.succActor}`)
Expand Down Expand Up @@ -1075,16 +1080,12 @@ function addPatchProperty(objects, property) {
for (let succId of op.succ) counter.succ[succId] = true

} else if (op.succ.length === 0) { // Ignore any ops that have been overwritten
if (op.actionName.startsWith('make')) {
if (op.actionName === 'set' && op.value.datatype === 'cursor') {
values[op.opId] = {elemId: op.ref, datatype: 'cursor'}
} else if (op.actionName.startsWith('make')) {
values[op.opId] = objects[op.opId]
} else if (op.actionName === 'set') {
values[op.opId] = op.value
} else if (op.actionName === 'link') {
// NB. This assumes that the ID of the child object is greater than the ID of the current
// object. This is true as long as link operations are only used to redo undone make*
// operations, but it will cease to be true once subtree moves are allowed.
if (!op.childId) throw new RangeError(`link operation ${op.opId} without a childId`)
values[op.opId] = objects[op.childId]
} else {
throw new RangeError(`Unexpected action type: ${op.actionName}`)
}
Expand Down Expand Up @@ -1137,8 +1138,8 @@ function constructPatch(documentBuffer) {

const keyActor = col.keyActor.readValue(), keyCtr = col.keyCtr.readValue()
const keyStr = col.keyStr.readValue(), insert = !!col.insert.readValue()
const chldActor = col.chldActor.readValue(), chldCtr = col.chldCtr.readValue()
const childId = chldActor === null ? null : `${chldCtr}@${actorIds[chldActor]}`
const refActor = col.refActor.readValue(), refCtr = col.refCtr.readValue()
const ref = refActor === null ? null : `${refCtr}@${actorIds[refActor]}`
const sizeTag = col.valLen.readValue()
const rawValue = col.valRaw.readRawBytes(sizeTag >> 4)
const value = decodeValue(sizeTag, rawValue)
Expand Down Expand Up @@ -1167,7 +1168,7 @@ function constructPatch(documentBuffer) {
if (property) addPatchProperty(objects, property)
property = {objId, key, ops: []}
}
property.ops.push({opId, actionName, value, childId, succ})
property.ops.push({opId, actionName, value, ref, succ})
}

if (property) addPatchProperty(objects, property)
Expand All @@ -1185,7 +1186,7 @@ function constructPatch(documentBuffer) {
function seekToOp(ops, docCols, actorIds) {
const { objActor, objCtr, keyActor, keyCtr, keyStr, idActor, idCtr, insert, action, consecutiveOps } = ops
const [objActorD, objCtrD, keyActorD, keyCtrD, keyStrD, idActorD, idCtrD, insertD, actionD,
valLenD, valRawD, chldActorD, chldCtrD, succNumD] = docCols.map(col => col.decoder)
valLenD, valRawD, refActorD, refCtrD, succNumD] = docCols.map(col => col.decoder)
let skipCount = 0, visibleCount = 0, elemVisible = false, nextObjActor = null, nextObjCtr = null
let nextIdActor = null, nextIdCtr = null, nextKeyStr = null, nextInsert = null, nextSuccNum = 0

Expand Down Expand Up @@ -2020,7 +2021,7 @@ class BackendDoc {
getAllColumns(changeCols) {
const expectedCols = [
'objActor', 'objCtr', 'keyActor', 'keyCtr', 'keyStr', 'idActor', 'idCtr', 'insert',
'action', 'valLen', 'valRaw', 'chldActor', 'chldCtr', 'predNum', 'predActor', 'predCtr'
'action', 'valLen', 'valRaw', 'refActor', 'refCtr', 'predNum', 'predActor', 'predCtr'
]
let allCols = {}
for (let i = 0; i < expectedCols.length; i++) {
Expand Down
Loading