feat(mito): parquet memtable reader (#4967)

* wip: row group reader base

* wip: memtable row group reader

* Refactor MemtableRowGroupReader to streamline data fetching

 - Added early return when fetch_ranges is empty to optimize performance.
 - Replaced inline chunk data assignment with a call to `assign_dense_chunk` for cleaner code.

* wip: row group reader

* wip: reuse RowGroupReader

* wip: bulk part reader

* Enhance BulkPart Iteration with Filtering

 - Introduced `RangeBase` to `BulkIterContext` for improved filter handling.
 - Implemented filter application in `BulkPartIter` to prune batches based on predicates.
 - Updated `SimpleFilterContext::new_opt` to be public for broader access.

* chore: add prune test

* fix: clippy

* fix: introduce prune reader for memtable and add more prune test

* Enhance BulkPart read method to return Option<BoxedBatchIterator>

 - Modified `BulkPart::read` to return `Option<BoxedBatchIterator>` to handle cases where no row groups are selected.
 - Added logic to return `None` when all row groups are filtered out.
 - Updated tests to handle the new return type and added a test case to verify behavior when no row groups match the pr

* refactor/separate-paraquet-reader: Add helper function to parse parquet metadata and integrate it into BulkPartEncoder

* refactor/separate-paraquet-reader:
 Change BulkPartEncoder row_group_size from Option to usize and update tests

* refactor/separate-paraquet-reader: Add context module for bulk memtable iteration and refactor part reading

 • Introduce context module to encapsulate context for bulk memtable iteration.
 • Refactor BulkPart to use BulkIterContextRef for reading operations.
 • Remove redundant code in BulkPart by centralizing context creation and row group pruning logic in the new context module.
 • Create new file context.rs with structures and logic for handling iteration context.
 • Adjust part_reader.rs and row_group_reader.rs to reference the new BulkIterContextRef.

* refactor/separate-paraquet-reader: Refactor RowGroupReader traits and implementations in memtable and parquet reader modules

 • Rename RowGroupReaderVirtual to RowGroupReaderContext for clarity.
 • Replace BulkPartVirt with direct usage of BulkIterContextRef in MemtableRowGroupReader.
 • Simplify MemtableRowGroupReaderBuilder by directly passing context instead of creating a BulkPartVirt instance.
 • Update RowGroupReaderBase to use context field instead of virt, reflecting the trait renaming and usage.
 • Modify FileRangeVirt to FileRangeContextRef and adjust implementations accordingly.

* refactor/separate-paraquet-reader: Refactor column page reader creation and remove unused code

 • Centralize creation of SerializedPageReader in RowGroupBase::column_reader method.
 • Remove unused RowGroupCachedReader and related code from MemtableRowGroupPageFetcher.
 • Eliminate redundant error handling for invalid column index in multiple places.

* chore: rebase main and resolve conflicts

* fix: some comments

* chore: resolve conflicts

* chore: resolve conflicts

src/mito2/src/error.rs

@@ -723,10 +723,20 @@ pub enum Error {
 
     #[snafu(display("Failed to iter data part"))]
     ReadDataPart {
+        #[snafu(implicit)]
+        location: Location,
         #[snafu(source)]
         error: parquet::errors::ParquetError,
     },
 
+    #[snafu(display("Failed to read row group in memtable"))]
+    DecodeArrowRowGroup {
+        #[snafu(source)]
+        error: ArrowError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Invalid region options, {}", reason))]
     InvalidRegionOptions {
         reason: String,
@@ -1029,6 +1039,7 @@ impl ErrorExt for Error {
             RegionBusy { .. } => StatusCode::RegionBusy,
             GetSchemaMetadata { source, .. } => source.status_code(),
             Timeout { .. } => StatusCode::Cancelled,
+            DecodeArrowRowGroup { .. } => StatusCode::Internal,
         }
     }
 

src/mito2/src/memtable/bulk.rs

@@ -27,8 +27,12 @@ use crate::memtable::{
     BoxedBatchIterator, KeyValues, Memtable, MemtableId, MemtableRanges, MemtableRef, MemtableStats,
 };
 
+#[allow(unused)]
+mod context;
 #[allow(unused)]
 pub(crate) mod part;
+mod part_reader;
+mod row_group_reader;
 
 #[derive(Debug)]
 pub struct BulkMemtable {

src/mito2/src/memtable/bulk/context.rs

@@ -0,0 +1,117 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Context for iterating bulk memtable.
+
+use std::collections::VecDeque;
+use std::sync::Arc;
+
+use parquet::file::metadata::ParquetMetaData;
+use store_api::metadata::RegionMetadataRef;
+use store_api::storage::ColumnId;
+use table::predicate::Predicate;
+
+use crate::row_converter::McmpRowCodec;
+use crate::sst::parquet::file_range::RangeBase;
+use crate::sst::parquet::format::ReadFormat;
+use crate::sst::parquet::reader::SimpleFilterContext;
+use crate::sst::parquet::stats::RowGroupPruningStats;
+
+pub(crate) type BulkIterContextRef = Arc<BulkIterContext>;
+
+pub(crate) struct BulkIterContext {
+    pub(crate) base: RangeBase,
+    pub(crate) predicate: Option<Predicate>,
+}
+
+impl BulkIterContext {
+    pub(crate) fn new(
+        region_metadata: RegionMetadataRef,
+        projection: &Option<&[ColumnId]>,
+        predicate: Option<Predicate>,
+    ) -> Self {
+        let codec = McmpRowCodec::new_with_primary_keys(&region_metadata);
+
+        let simple_filters = predicate
+            .as_ref()
+            .iter()
+            .flat_map(|predicate| {
+                predicate
+                    .exprs()
+                    .iter()
+                    .filter_map(|expr| SimpleFilterContext::new_opt(&region_metadata, None, expr))
+            })
+            .collect();
+
+        let read_format = build_read_format(region_metadata, projection);
+
+        Self {
+            base: RangeBase {
+                filters: simple_filters,
+                read_format,
+                codec,
+                // we don't need to compat batch since all batch in memtable have the same schema.
+                compat_batch: None,
+            },
+            predicate,
+        }
+    }
+
+    /// Prunes row groups by stats.
+    pub(crate) fn row_groups_to_read(&self, file_meta: &Arc<ParquetMetaData>) -> VecDeque<usize> {
+        let region_meta = self.base.read_format.metadata();
+        let row_groups = file_meta.row_groups();
+        // expected_metadata is set to None since we always expect region metadata of memtable is up-to-date.
+        let stats = RowGroupPruningStats::new(row_groups, &self.base.read_format, None);
+        if let Some(predicate) = self.predicate.as_ref() {
+            predicate
+                .prune_with_stats(&stats, region_meta.schema.arrow_schema())
+                .iter()
+                .zip(0..file_meta.num_row_groups())
+                .filter_map(|(selected, row_group)| {
+                    if !*selected {
+                        return None;
+                    }
+                    Some(row_group)
+                })
+                .collect::<VecDeque<_>>()
+        } else {
+            (0..file_meta.num_row_groups()).collect()
+        }
+    }
+
+    pub(crate) fn read_format(&self) -> &ReadFormat {
+        &self.base.read_format
+    }
+}
+
+fn build_read_format(
+    region_metadata: RegionMetadataRef,
+    projection: &Option<&[ColumnId]>,
+) -> ReadFormat {
+    let read_format = if let Some(column_ids) = &projection {
+        ReadFormat::new(region_metadata, column_ids.iter().copied())
+    } else {
+        // No projection, lists all column ids to read.
+        ReadFormat::new(
+            region_metadata.clone(),
+            region_metadata
+                .column_metadatas
+                .iter()
+                .map(|col| col.column_id),
+        )
+    };
+
+    read_format
+}