reimplement varint encoding for values and block handles

src/blob_tree/value.rs

@@ -3,9 +3,10 @@
 // (found in the LICENSE-* files in the repository)
 
 use crate::coding::{Decode, DecodeError, Encode, EncodeError};
-use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
+use byteorder::{ReadBytesExt, WriteBytesExt};
 use std::io::{Read, Write};
 use value_log::{Slice, UserValue, ValueHandle};
+use varint_rs::{VarintReader, VarintWriter};
 
 /// A value which may or may not be inlined into an index tree
 ///
@@ -23,16 +24,16 @@ pub enum MaybeInlineValue {
 
 impl Encode for ValueHandle {
     fn encode_into<W: Write>(&self, writer: &mut W) -> Result<(), EncodeError> {
-        writer.write_u64::<BigEndian>(self.offset)?;
-        writer.write_u64::<BigEndian>(self.segment_id)?;
+        writer.write_u64_varint(self.offset)?;
+        writer.write_u64_varint(self.segment_id)?;
         Ok(())
     }
 }
 
 impl Decode for ValueHandle {
     fn decode_from<R: Read>(reader: &mut R) -> Result<Self, DecodeError> {
-        let offset = reader.read_u64::<BigEndian>()?;
-        let segment_id = reader.read_u64::<BigEndian>()?;
+        let offset = reader.read_u64_varint()?;
+        let segment_id = reader.read_u64_varint()?;
         Ok(Self { segment_id, offset })
     }
 }
@@ -48,14 +49,14 @@ impl Encode for MaybeInlineValue {
 
                 // NOTE: Values can be up to 2^32 bytes
                 #[allow(clippy::cast_possible_truncation)]
-                writer.write_u32::<BigEndian>(bytes.len() as u32)?;
+                writer.write_u32_varint(bytes.len() as u32)?;
 
                 writer.write_all(bytes)?;
             }
             Self::Indirect { vhandle, size } => {
                 writer.write_u8(TAG_INDIRECT)?;
                 vhandle.encode_into(writer)?;
-                writer.write_u32::<BigEndian>(*size)?;
+                writer.write_u32_varint(*size)?;
             }
         }
         Ok(())
@@ -68,14 +69,14 @@ impl Decode for MaybeInlineValue {
 
         match tag {
             TAG_INLINE => {
-                let len = reader.read_u32::<BigEndian>()? as usize;
+                let len = reader.read_u32_varint()? as usize;
                 let mut bytes = vec![0; len];
                 reader.read_exact(&mut bytes)?;
                 Ok(Self::Inline(Slice::from(bytes)))
             }
             TAG_INDIRECT => {
                 let vhandle = ValueHandle::decode_from(reader)?;
-                let size = reader.read_u32::<BigEndian>()?;
+                let size = reader.read_u32_varint()?;
                 Ok(Self::Indirect { vhandle, size })
             }
             x => Err(DecodeError::InvalidTag(("MaybeInlineValue", x))),

src/key.rs

@@ -6,11 +6,12 @@ use crate::{
     coding::{Decode, DecodeError, Encode, EncodeError},
     SeqNo, UserKey, ValueType,
 };
-use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
+use byteorder::{ReadBytesExt, WriteBytesExt};
 use std::{
     cmp::Reverse,
     io::{Read, Write},
 };
+use varint_rs::{VarintReader, VarintWriter};
 
 #[derive(Clone, PartialEq, Eq)]
 #[allow(clippy::module_name_repetitions)]
@@ -59,31 +60,32 @@ impl InternalKey {
 
 impl Encode for InternalKey {
     fn encode_into<W: Write>(&self, writer: &mut W) -> Result<(), EncodeError> {
+        writer.write_u64_varint(self.seqno)?;
+
+        writer.write_u8(u8::from(self.value_type))?;
+
         // NOTE: Truncation is okay and actually needed
         #[allow(clippy::cast_possible_truncation)]
-        writer.write_u16::<BigEndian>(self.user_key.len() as u16)?;
+        writer.write_u16_varint(self.user_key.len() as u16)?;
         writer.write_all(&self.user_key)?;
 
-        writer.write_u64::<BigEndian>(self.seqno)?;
-        writer.write_u8(u8::from(self.value_type))?;
-
         Ok(())
     }
 }
 
 impl Decode for InternalKey {
     fn decode_from<R: Read>(reader: &mut R) -> Result<Self, DecodeError> {
-        let key_len = reader.read_u16::<BigEndian>()?;
-        let mut key = vec![0; key_len.into()];
-        reader.read_exact(&mut key)?;
-
-        let seqno = reader.read_u64::<BigEndian>()?;
+        let seqno = reader.read_u64_varint()?;
 
         let value_type = reader.read_u8()?;
         let value_type = value_type
             .try_into()
             .map_err(|()| DecodeError::InvalidTag(("ValueType", value_type)))?;
 
+        let key_len = reader.read_u16_varint()?;
+        let mut key = vec![0; key_len.into()];
+        reader.read_exact(&mut key)?;
+
         Ok(Self::new(key, seqno, value_type))
     }
 }

src/segment/block_index/block_handle.rs

@@ -8,8 +8,8 @@ use crate::{
     value::UserKey,
     Slice,
 };
-use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use std::io::{Read, Write};
+use varint_rs::{VarintReader, VarintWriter};
 
 /// Points to a block on file
 #[derive(Clone, Debug)]
@@ -65,12 +65,11 @@ impl Ord for KeyedBlockHandle {
 
 impl Encode for KeyedBlockHandle {
     fn encode_into<W: Write>(&self, writer: &mut W) -> Result<(), EncodeError> {
-        writer.write_u64::<BigEndian>(self.offset)?;
+        writer.write_u64_varint(self.offset)?;
 
         // NOTE: Truncation is okay and actually needed
         #[allow(clippy::cast_possible_truncation)]
-        writer.write_u16::<BigEndian>(self.end_key.len() as u16)?;
-
+        writer.write_u16_varint(self.end_key.len() as u16)?;
         writer.write_all(&self.end_key)?;
 
         Ok(())
@@ -82,10 +81,9 @@ impl Decode for KeyedBlockHandle {
     where
         Self: Sized,
     {
-        let offset = reader.read_u64::<BigEndian>()?;
-
-        let key_len = reader.read_u16::<BigEndian>()?;
+        let offset = reader.read_u64_varint()?;
 
+        let key_len = reader.read_u16_varint()?;
         let mut key = vec![0; key_len.into()];
         reader.read_exact(&mut key)?;
 

src/value.rs

@@ -8,8 +8,8 @@ use crate::{
     segment::block::ItemSize,
     Slice,
 };
-use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use std::io::{Read, Write};
+use varint_rs::{VarintReader, VarintWriter};
 
 /// User defined key
 pub type UserKey = Slice;
@@ -172,7 +172,7 @@ impl Encode for InternalValue {
         if !self.is_tombstone() {
             // NOTE: We know values are limited to 32-bit length
             #[allow(clippy::cast_possible_truncation)]
-            writer.write_u32::<BigEndian>(self.value.len() as u32)?;
+            writer.write_u32_varint(self.value.len() as u32)?;
             writer.write_all(&self.value)?;
         }
 
@@ -192,7 +192,7 @@ impl Decode for InternalValue {
         } else {
             // NOTE: Only read value if we are actually a value
 
-            let value_len = reader.read_u32::<BigEndian>()?;
+            let value_len = reader.read_u32_varint()?;
             let mut value = vec![0; value_len as usize];
             reader.read_exact(&mut value)?;
 
@@ -231,18 +231,18 @@ mod tests {
             InternalValue::from_components(vec![1, 2, 3], vec![3, 2, 1], 1, ValueType::Value);
 
         #[rustfmt::skip]
-        let  bytes = &[
-            // Key
-            0, 3, 1, 2, 3,
-
+        let bytes = [
             // Seqno
-            0, 0, 0, 0, 0, 0, 0, 1,
+            1,
 
             // Type
             0,
+
+            // User key
+            3, 1, 2, 3,
 
-            // Value
-            0, 0, 0, 3, 3, 2, 1,
+            // User value
+            3, 3, 2, 1,
         ];
 
         // Deserialize the empty Value