[WIP] Stats pruning #979

robert3005 · 2024-10-04T15:01:51Z

There's a lot of cleanup to do here but this performs pruning, however, we don't
yet make use of that pruning result

robert3005 · 2024-10-04T15:02:23Z

encodings/bytebool/src/lib.rs

@@ -100,6 +100,15 @@ impl BoolArrayTrait for ByteBoolArray {
    fn maybe_null_slices_iter<'a>(&'a self) -> Box<dyn Iterator<Item = (usize, usize)> + 'a> {
        todo!()
    }
+
+    fn not(&self) -> Array {


rename to inverse

Invert? Negate?

robert3005 · 2024-10-04T15:02:27Z

encodings/roaring/src/boolean/mod.rs

@@ -104,6 +104,12 @@ impl BoolArrayTrait for RoaringBoolArray {
    fn maybe_null_slices_iter<'a>(&'a self) -> Box<dyn Iterator<Item = (usize, usize)> + 'a> {
        todo!()
    }
+
+    fn not(&self) -> Array {


rename to inverse

robert3005 · 2024-10-04T15:02:30Z

encodings/runend-bool/src/array.rs

@@ -113,6 +114,12 @@ impl BoolArrayTrait for RunEndBoolArray {
    fn maybe_null_slices_iter<'a>(&'a self) -> Box<dyn Iterator<Item = (usize, usize)> + 'a> {
        todo!()
    }
+
+    fn not(&self) -> Array {


rename to inverse

robert3005 · 2024-10-04T15:02:36Z

vortex-array/src/array/bool/mod.rs

@@ -103,6 +103,12 @@ impl BoolArrayTrait for BoolArray {
    fn maybe_null_slices_iter<'a>(&'a self) -> Box<dyn Iterator<Item = (usize, usize)> + 'a> {
        Box::new(BitSliceIterator::new(self.buffer(), 0, self.len()))
    }
+
+    fn not(&self) -> Array {


rename to inverse

robert3005 · 2024-10-04T15:02:40Z

vortex-array/src/array/chunked/variants.rs

@@ -54,6 +54,16 @@ impl BoolArrayTrait for ChunkedArray {
    fn maybe_null_slices_iter(&self) -> Box<dyn Iterator<Item = (usize, usize)>> {
        todo!()
    }
+
+    fn not(&self) -> Array {


rename to inverse

robert3005 · 2024-10-04T15:02:43Z

vortex-array/src/array/constant/variants.rs

@@ -78,6 +78,18 @@ impl BoolArrayTrait for ConstantArray {
            Box::new(iter::empty())
        }
    }
+
+    fn not(&self) -> Array {


rename to inverse

robert3005 · 2024-10-04T15:02:49Z

vortex-array/src/array/sparse/variants.rs

@@ -55,6 +55,18 @@ impl BoolArrayTrait for SparseArray {
    fn maybe_null_slices_iter(&self) -> Box<dyn Iterator<Item = (usize, usize)>> {
        todo!()
    }
+
+    fn not(&self) -> Array {


rename to inverse

robert3005 · 2024-10-04T15:02:56Z

vortex-array/src/array/constant/variants.rs

@@ -107,20 +119,6 @@ where
 }

 impl PrimitiveArrayTrait for ConstantArray {
-    fn f32_accessor(&self) -> Option<AccessorRef<f32>> {


undo this diff

robert3005 · 2024-10-04T15:03:09Z

vortex-array/src/compute/compare.rs

@@ -95,7 +95,7 @@ pub fn compare(
        vortex_bail!("Compare operations only support arrays of the same type");
    }

-    if left.is_encoding(Constant::ID) {
+    if left.is_encoding(Constant::ID) && !right.is_encoding(Constant::ID) {


make a separate pr and add tests

robert3005 · 2024-10-04T15:03:13Z

vortex-array/src/variants.rs

@@ -103,6 +103,8 @@ pub trait BoolArrayTrait: ArrayTrait {
    //                                         and the buffer is a slice of that array, omitted slices
    //                                         could be either true or false signified by the initial
    //                                         value returned.
+
+    fn not(&self) -> Array;


rename to inverse

robert3005 · 2024-10-04T15:03:48Z

vortex-expr/src/expr.rs

 pub trait VortexExpr: Debug + Send + Sync + PartialEq<dyn Any> {
    fn as_any(&self) -> &dyn Any;

    fn evaluate(&self, batch: &Array) -> VortexResult<Array>;

    fn references(&self) -> HashSet<Field>;

-    fn estimate_cost(&self, schema: &Schema) -> usize;


This has to come back at some point, however, it has to have different form

robert3005 · 2024-10-04T15:04:17Z

vortex-expr/src/expr.rs

@@ -43,9 +40,6 @@ fn unbox_any(any: &dyn Any) -> &dyn Any {
    }
 }

-#[derive(Debug, PartialEq, Hash, Clone, Eq)]
-pub struct NoOp;


There was no need to translate it, we will never see it as a datafusion table provider

robert3005 · 2024-10-04T15:04:31Z

vortex-expr/src/expr.rs

+                    // TODO(robert): This might be too broad of a check since it should be limited only to fields in the projection
+                    self.rhs
+                        .references()
+                        .intersection(&lhsp.references())
+                        .next()
+                        .is_none()
+                        .then_some(lhsp)


Narrow this check down

robert3005 · 2024-10-04T15:06:09Z

vortex-expr/src/expr.rs

-    fn estimate_cost(&self, schema: &Schema) -> usize;
+    fn project(&self, projection: &[Field]) -> Option<Arc<dyn VortexExpr>>;
+
+    fn is_constant(&self) -> bool;


This should actually be fn fold(&self) -> Option<Scalar>, i.e. can this be evaluated to a constant?

robert3005 · 2024-10-04T15:06:53Z

vortex-expr/src/lib.rs

@@ -10,20 +8,25 @@ pub use expr::*;
 pub use operators::*;

 pub fn split_conjunction(expr: &Arc<dyn VortexExpr>) -> Vec<Arc<dyn VortexExpr>> {


Split this diff to separate pr

robert3005 · 2024-10-04T15:07:03Z

vortex-flatbuffers/flatbuffers/vortex-serde/footer.fbs

    encoding: uint16;
+    metadata: [ubyte];


actually use this, it's never written for now

robert3005 · 2024-10-04T15:07:27Z

vortex-scalar/src/primitive.rs

@@ -171,14 +171,6 @@ macro_rules! primitive_scalar {
            }
        }

-        impl From<Option<$T>> for ScalarValue {


we can actually have a generic instead of generating all of the impls

robert3005 · 2024-10-04T15:07:32Z

vortex-scalar/src/value.rs

@@ -135,6 +135,12 @@ impl ScalarValue {
    }
 }

+impl<T: Into<ScalarValue>> From<Option<T>> for ScalarValue {


robert3005 · 2024-10-04T15:07:42Z

vortex-serde/Cargo.toml

 simplelog = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 vortex-alp = { path = "../encodings/alp" }
 vortex-fastlanes = { path = "../encodings/fastlanes" }
 vortex-sampling-compressor = { path = "../vortex-sampling-compressor" }

 [lints]
-workspace = true
+# workspace = true


bring this back

robert3005 · 2024-10-04T15:07:55Z

vortex-serde/src/layouts/pruning.rs

+    pub fn try_new(original_expr: &Arc<dyn VortexExpr>) -> Option<Self> {
+        let (expr, required_stats) = convert_to_pruning_expression(original_expr);
+        // TODO(robert): Could be constant false but right now we don't generate such expressions, need to have way to reduce constant expressions
+        if expr.is_constant() {


This should use fold

robert3005 · 2024-10-04T15:08:16Z

vortex-serde/src/layouts/pruning.rs

+    if let Some(nexp) = expr.as_any().downcast_ref::<Not>() {
+        if nexp.child().as_any().downcast_ref::<Column>().is_some() {
+            let mut references = HashMap::new();
+            let min_expr = replace_column_with_stat(expr, Stat::Min, &mut references);
+            let max_expr = replace_column_with_stat(expr, Stat::Max, &mut references);
+            return min_expr
+                .zip(max_expr)
+                .map(|(min_exp, max_exp)| {
+                    (
+                        Arc::new(BinaryExpr::new(min_exp, Operator::And, max_exp))
+                            as Arc<dyn VortexExpr>,
+                        references,
+                    )
+                })
+                .unwrap_or_else(|| {
+                    (
+                        Arc::new(Literal::new(Scalar::bool(false, Nullability::NonNullable))),
+                        HashMap::new(),
+                    )
+                });
+        }
+    }
+
    if expr.as_any().downcast_ref::<Column>().is_some() {
-        return (fallback, HashMap::new());
+        let mut references = HashMap::new();
+        let min_expr = replace_column_with_stat(expr, Stat::Min, &mut references);
+        let max_expr = replace_column_with_stat(expr, Stat::Max, &mut references);
+        return min_expr
+            .zip(max_expr)
+            .map(|(min_exp, max_exp)| {
+                (
+                    Arc::new(Not::new(Arc::new(BinaryExpr::new(
+                        min_exp,
+                        Operator::Or,
+                        max_exp,
+                    )))) as Arc<dyn VortexExpr>,
+                    references,
+                )
+            })
+            .unwrap_or_else(|| {
+                (
+                    Arc::new(Literal::new(Scalar::bool(false, Nullability::NonNullable))),
+                    HashMap::new(),
+                )
+            });


Extract this to separate function

robert3005 · 2024-10-04T15:09:15Z

vortex-serde/src/layouts/read/batch.rs

+pub struct BatchPruner {
+    children: Vec<Box<dyn LayoutReader>>,
+    selectors: Vec<Option<RowSelector>>,
+    projected_scans: Vec<(usize, PruningScan)>,


This is a bit too similar to BatchReader. Consider unifying

robert3005 · 2024-10-04T15:09:59Z

vortex-serde/src/layouts/read/builder.rs

-            "Mask arrays have to be integer arrays"
-        );
-        self.indices = Some(array);
+    pub fn with_row_selector(mut self, array: RowSelector) -> Self {


It's strictly more powerful to use row ranges than individual indices

robert3005 · 2024-10-04T15:10:28Z

vortex-serde/src/layouts/read/builder.rs

+        let pruner = if let Some(pscan) = pruning_scan {
+            if let Some(fr) = filter_reader {
+                Ok(LayoutPruner::new(
+                    self.reader,
+                    fr,
+                    message_cache.clone(),
+                    pscan,
+                ))
+            } else {
+                Err((self.reader, filter_reader))
+            }
+        } else {
+            Err((self.reader, filter_reader))
+        };
+


This is terrible, we have to push filtering in to readers to avoid this mess

robert3005 · 2024-10-04T15:12:35Z

vortex-serde/src/layouts/read/cache.rs

+    pub fn has_dtype(&self) -> bool {
+        self.dtype.is_some()
+    }


A bit of a hack. We know that if we don't have dtype it must be stored somehow. However, I think we can deduce stats schema and avoid storing dtype altogether if we store stats as column

robert3005 · 2024-10-04T15:13:19Z

vortex-serde/src/layouts/read/layouts/chunked.rs

+                        if let Some(pf) = &scan.filter {
+                            let field_prefix = scan
+                                .stats_projection
+                                .keys()
+                                .next()
+                                .cloned()
+                                .vortex_expect("Must have only single field ref");
+                            let field_stats = scan
+                                .stats_projection
+                                .get(&field_prefix)
+                                .vortex_expect("We got the key by iterating keys");
+                            let prefix_str = match field_prefix {
+                                Field::Name(n) => n.to_string(),
+                                Field::Index(i) => i.to_string(),
+                            };
+                            let expanded_batch = add_missing_columns(
+                                b,
+                                &prefix_str,
+                                field_stats,
+                                self.message_cache.dtype(),
+                            )?;
+                            let mask = pf.evaluate(&expanded_batch)?;
+                            let row_offsets = expanded_batch
+                                .with_dyn(|a| {
+                                    a.as_struct_array()
+                                        .ok_or_else(|| vortex_err!("Stats weren't a struct array"))
+                                        .map(|s| {
+                                            s.field_by_name(&format!("{prefix_str}_row_offset"))
+                                        })
+                                })?
+                                .ok_or_else(|| vortex_err!("Missing row offsets"))?;
+                            filter_offsets(row_offsets, mask, scan.row_count as usize)
+                                .map(|s| Some(PlanResult::Range(s)))


There is too much logic here

robert3005 · 2024-10-04T15:13:53Z

vortex-serde/src/layouts/read/layouts/chunked.rs

+                                        })
+                                })?
+                                .ok_or_else(|| vortex_err!("Missing row offsets"))?;
+                            filter_offsets(row_offsets, mask, scan.row_count as usize)


Since we return chunks that should be pruned and we want to produce rows to include we need to flip the mask in this function

robert3005 · 2024-10-04T15:14:10Z

vortex-serde/src/layouts/read/layouts/chunked.rs

+    }
+}
+
+fn filter_offsets(offsets: Array, mask: Array, row_count: usize) -> VortexResult<RowSelector> {


The mask has to be flipped before being passed to this function or this function flips the mask

robert3005 · 2024-10-04T15:32:31Z

vortex-serde/src/layouts/read/layouts/flat.rs

+                let dtype = if self.cache.has_dtype() {
+                    self.cache.dtype().clone()
+                } else {
+                    let dtype_fb_length = buf.get_u32_le();
+                    let dtype_buf = buf.split_to(dtype_fb_length as usize);
+                    let msg = unsafe { root_unchecked::<fb::Message>(&dtype_buf) }
+                        .header_as_schema()
+                        .ok_or_else(|| {
+                            vortex_err!(
+                                "Expected schema message; this was checked earlier in the function"
+                            )
+                        })?;
+                    IPCDType::read_flatbuffer(&msg)?.0
+                };


this likely goes away if we change the way stats are stored

robert3005 · 2024-10-04T15:34:35Z

vortex-serde/src/layouts/read/stream.rs

+        let mut input = None;
+        let mut filter_reader = None;
+        let mut layout_pruner = None;
+        let mut state = StreamingState::Init;
+        if let Ok(pruner) = pruner {
+            layout_pruner = Some(pruner);
+            state = StreamingState::Pruning;
        } else {
-            StreamingState::Init
-        };
+            (input, filter_reader) = pruner.map_err(|(i, f)| (Some(i), f)).unwrap_err();
+            if filter_reader.is_some() {
+                state = StreamingState::FilterInit;
+            }
+        }


I hate this code but ownership is hard

robert3005 · 2024-10-04T15:35:34Z

vortex-serde/src/layouts/read/selection.rs

+}
+
+impl RowSelector {
+    pub fn new(ranges: Vec<RowRange>) -> Self {


We should have a sorting constructor and unsafe non sorting one

robert3005 commented Oct 4, 2024

View reviewed changes

robert3005 force-pushed the rk/plannig branch from 30f2c0d to 804f6f3 Compare October 7, 2024 00:26

Pruning

4b9e898

robert3005 force-pushed the rk/plannig branch from 804f6f3 to 4b9e898 Compare October 7, 2024 00:57

robert3005 mentioned this pull request Oct 14, 2024

[WIP] Filter pushdown #1025

Closed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[WIP] Stats pruning #979

[WIP] Stats pruning #979

robert3005 commented Oct 4, 2024

robert3005 Oct 4, 2024

gatesn Oct 10, 2024 •

edited

Loading

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

robert3005 Oct 4, 2024

		@@ -10,20 +8,25 @@ pub use expr::*;
		pub use operators::*;

		pub fn split_conjunction(expr: &Arc<dyn VortexExpr>) -> Vec<Arc<dyn VortexExpr>> {

[WIP] Stats pruning #979

Are you sure you want to change the base?

[WIP] Stats pruning #979

Conversation

robert3005 commented Oct 4, 2024

Choose a reason for hiding this comment

gatesn Oct 10, 2024 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

gatesn Oct 10, 2024 •

edited

Loading