Skip to content

allow filters referencing parquet struct columns to be pushed down #19556

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
adriangb wants to merge 12 commits into
base: main
Choose a base branch
from
Draft

allow filters referencing parquet struct columns to be pushed down #19556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
e24204a
allow filters referencing parquet struct columns to be pushed down
adriangb Dec 30, 2025
becce5c
remove test
adriangb Jan 7, 2026
e10b85f
handle upstream changes
adriangb Jan 7, 2026
7181e3a
fmt
adriangb Jan 7, 2026
3d94f4e
add benchmark for struct late materialization
adriangb Jan 8, 2026
8dfcd6e
update row counts
adriangb Jan 8, 2026
2a3bffd
optimize benchmark
adriangb Jan 8, 2026
c13a821
update row count
adriangb Jan 8, 2026
ce40cb3
update row count
adriangb Jan 8, 2026
39b37af
fix test
adriangb Jan 8, 2026
77bfbf4
update crc32
adriangb Jan 8, 2026
f56c2ad
update slt
adriangb Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 44 additions & 30 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 13 additions & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,3 +271,16 @@ incremental = false
inherits = "release"
debug = true
strip = false

[patch.crates-io]
parquet = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-array = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-buffer = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-cast = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-data = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-ipc = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-ord = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-schema = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-select = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
arrow-string = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
9 changes: 5 additions & 4 deletions datafusion/common/src/scalar/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8339,7 +8339,7 @@ mod tests {
)))),
];

let check_array = |array| {
let check_array = |array: Arc<dyn Array>| {
let is_null = is_null(&array).unwrap();
assert_eq!(is_null, BooleanArray::from(vec![true, false, false]));

Expand Down Expand Up @@ -8868,7 +8868,7 @@ mod tests {
.unwrap(),
ScalarValue::try_new_null(&DataType::Map(map_field_ref, false)).unwrap(),
ScalarValue::try_new_null(&DataType::Union(
UnionFields::new(vec![42], vec![field_ref]),
UnionFields::try_new(vec![42], vec![field_ref]).unwrap(),
Copy link
Contributor Author

@adriangb adriangb Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that this is just handling upstream changes in arrow, not actually related to this PR

UnionMode::Dense,
))
.unwrap(),
Expand Down Expand Up @@ -8971,13 +8971,14 @@ mod tests {
}

// Test union type
let union_fields = UnionFields::new(
let union_fields = UnionFields::try_new(
vec![0, 1],
vec![
Field::new("i32", DataType::Int32, false),
Field::new("f64", DataType::Float64, false),
],
);
)
.unwrap();
let union_result = ScalarValue::new_default(&DataType::Union(
union_fields.clone(),
UnionMode::Sparse,
Expand Down
2 changes: 1 addition & 1 deletion datafusion/datasource-avro/src/avro_to_arrow/schema.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ fn schema_to_field_with_props(
.map(|s| schema_to_field_with_props(s, None, has_nullable, None))
.collect::<Result<Vec<Field>>>()?;
let type_ids = 0_i8..fields.len() as i8;
DataType::Union(UnionFields::new(type_ids, fields), UnionMode::Dense)
DataType::Union(UnionFields::try_new(type_ids, fields)?, UnionMode::Dense)
}
}
AvroSchema::Record(RecordSchema { fields, .. }) => {
Expand Down
7 changes: 7 additions & 0 deletions datafusion/datasource-parquet/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,11 @@ tokio = { workspace = true }
[dev-dependencies]
chrono = { workspace = true }
criterion = { workspace = true }
datafusion-functions = { workspace = true }
datafusion-functions-nested = { workspace = true }
rand = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread"] }

# Note: add additional linter rules in lib.rs.
# Rust does not support workspace + new linter rules in subcrates yet
Expand All @@ -80,3 +83,7 @@ parquet_encryption = [
[[bench]]
name = "parquet_nested_filter_pushdown"
harness = false

[[bench]]
name = "parquet_struct_filter_pushdown"
harness = false
Loading
Loading