Incorrect filtering results when collect_statistics = true and column statistics are partially disabled

Describe the bug

When datafusion.execution.collect_statistics = true is enabled, and Parquet statistics are only partially collected (i.e., some columns have statistics disabled), filters using IS NULL or IS NOT NULL on the non-statistics columns produce incorrect results.
In this case, queries that should return rows instead return no results, even though the data clearly exists in the Parquet file.

To Reproduce

use DataFusion CLI v50.2.0

> set datafusion.execution.collect_statistics = true;
0 row(s) fetched.
Elapsed 0.001 seconds.

> COPY (
  SELECT
    column1 as id,
    column2 as value,
  FROM (VALUES (1, 10), (2, 20), (3, NULL))
)
TO 'test/data.parquet'
OPTIONS (
  'statistics_enabled::value' 'none',
);
+-------+
| count |
+-------+
| 3     |
+-------+
1 row(s) fetched.
Elapsed 0.008 seconds.

> CREATE EXTERNAL TABLE t (
  id INT,
  value INT,
)
STORED AS PARQUET
LOCATION 'test';
0 row(s) fetched.
Elapsed 0.001 seconds.

> select * from t where value is null;
+----+-------+
| id | value |
+----+-------+
+----+-------+
0 row(s) fetched.
Elapsed 0.007 seconds.

> select * from 'test/data.parquet' where value is not null;
+----+-------+
| id | value |
+----+-------+
+----+-------+
0 row(s) fetched.
Elapsed 0.002 seconds.

> select * from 'test/data.parquet';
+----+-------+
| id | value |
+----+-------+
| 1  | 10    |
| 2  | 20    |
| 3  | NULL  |
+----+-------+
3 row(s) fetched.
Elapsed 0.002 seconds.

> select * from t;
+----+-------+
| id | value |
+----+-------+
| 1  | 10    |
| 2  | 20    |
| 3  | NULL  |
+----+-------+
3 row(s) fetched.
Elapsed 0.002 seconds.

select path_in_schema,stats_min,stats_max,stats_null_count,stats_min_value,stats_max_value from parquet_metadata('test/data.parquet');
+----------------+-----------+-----------+------------------+-----------------+-----------------+
| path_in_schema | stats_min | stats_max | stats_null_count | stats_min_value | stats_max_value |
+----------------+-----------+-----------+------------------+-----------------+-----------------+
| "id"           | 1         | 3         | 0                | 1               | 3               |
| "value"        | NULL      | NULL      | NULL             | NULL            | NULL            |
+----------------+-----------+-----------+------------------+-----------------+-----------------+
2 row(s) fetched.
Elapsed 0.002 seconds.

Expected behavior

Filtering with IS NULL / IS NOT NULL should still correctly evaluate against the file data even if statistics for a column are missing.

Additional context

No response

The issue likely comes from

datafusion/datafusion/datasource-parquet/src/metadata.rs

Lines 247 to 322 in 408e1e4

    
           let mut has_statistics = false; 
        
           let mut num_rows = 0_usize; 
        
           let mut total_byte_size = 0_usize; 
        
           for row_group_meta in row_groups_metadata { 
        
               num_rows += row_group_meta.num_rows() as usize; 
        
               total_byte_size += row_group_meta.total_byte_size() as usize; 
        
               if !has_statistics { 
        
                   has_statistics = row_group_meta 
        
                       .columns() 
        
                       .iter() 
        
                       .any(|column| column.statistics().is_some()); 
        
               } 
        
           } 
        
           statistics.num_rows = Precision::Exact(num_rows); 
        
           statistics.total_byte_size = Precision::Exact(total_byte_size); 
        
           let file_metadata = metadata.file_metadata(); 
        
           let mut file_schema = parquet_to_arrow_schema( 
        
               file_metadata.schema_descr(), 
        
               file_metadata.key_value_metadata(), 
        
           )?; 
        
           if let Some(merged) = apply_file_schema_type_coercions(table_schema, &file_schema) 
        
           { 
        
               file_schema = merged; 
        
           } 
        
           statistics.column_statistics = if has_statistics { 
        
               let (mut max_accs, mut min_accs) = create_max_min_accs(table_schema); 
        
               let mut null_counts_array = 
        
                   vec![Precision::Exact(0); table_schema.fields().len()]; 
        
               let mut is_max_value_exact = vec![Some(true); table_schema.fields().len()]; 
        
               let mut is_min_value_exact = vec![Some(true); table_schema.fields().len()]; 
        
               table_schema 
        
                   .fields() 
        
                   .iter() 
        
                   .enumerate() 
        
                   .for_each(|(idx, field)| { 
        
                       match StatisticsConverter::try_new( 
        
                           field.name(), 
        
                           &file_schema, 
        
                           file_metadata.schema_descr(), 
        
                       ) { 
        
                           Ok(stats_converter) => { 
        
                               let mut accumulators = StatisticsAccumulators { 
        
                                   min_accs: &mut min_accs, 
        
                                   max_accs: &mut max_accs, 
        
                                   null_counts_array: &mut null_counts_array, 
        
                                   is_min_value_exact: &mut is_min_value_exact, 
        
                                   is_max_value_exact: &mut is_max_value_exact, 
        
                               }; 
        
                               summarize_min_max_null_counts( 
        
                                   &mut accumulators, 
        
                                   idx, 
        
                                   num_rows, 
        
                                   &stats_converter, 
        
                                   row_groups_metadata, 
        
                               ) 
        
                               .ok(); 
        
                           } 
        
                           Err(e) => { 
        
                               debug!("Failed to create statistics converter: {e}"); 
        
                               null_counts_array[idx] = Precision::Exact(num_rows); 
        
                           } 
        
                       } 
        
                   }); 
        
               get_col_stats( 
        
                   table_schema, 
        
                   null_counts_array, 
        
                   &mut max_accs, 
        
                   &mut min_accs, 
        
                   &mut is_max_value_exact, 
        
                   &mut is_min_value_exact, 
        
               )

When one column has statistics, the code processes all columns’ statistics. For columns with statistics disabled, their stats are recorded as NULL instead of being marked as Absent, which causes incorrect predicate pruning on IS NULL / IS NOT NULL filters.

take

I checked this on current main.

The original reproducer still fails on 50.2.0, but works on main. A git bisect points to 97b9029 (fix null count stats computation, #18276) as the first fixed commit.

So this looks like it was already fixed by #18276.

Navigation Menu

Repository navigation

Incorrect filtering results when `collect_statistics = true` and column statistics are partially disabled #18241

Describe the bug

To Reproduce

Expected behavior

Additional context

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

	let mut has_statistics = false;
	let mut num_rows = 0_usize;
	let mut total_byte_size = 0_usize;
	for row_group_meta in row_groups_metadata {
	num_rows += row_group_meta.num_rows() as usize;
	total_byte_size += row_group_meta.total_byte_size() as usize;

	if !has_statistics {
	has_statistics = row_group_meta
	.columns()
	.iter()
	.any(\|column\| column.statistics().is_some());
	}
	}
	statistics.num_rows = Precision::Exact(num_rows);
	statistics.total_byte_size = Precision::Exact(total_byte_size);

	let file_metadata = metadata.file_metadata();
	let mut file_schema = parquet_to_arrow_schema(
	file_metadata.schema_descr(),
	file_metadata.key_value_metadata(),
	)?;

	if let Some(merged) = apply_file_schema_type_coercions(table_schema, &file_schema)
	{
	file_schema = merged;
	}

	statistics.column_statistics = if has_statistics {
	let (mut max_accs, mut min_accs) = create_max_min_accs(table_schema);
	let mut null_counts_array =
	vec![Precision::Exact(0); table_schema.fields().len()];
	let mut is_max_value_exact = vec![Some(true); table_schema.fields().len()];
	let mut is_min_value_exact = vec![Some(true); table_schema.fields().len()];
	table_schema
	.fields()
	.iter()
	.enumerate()
	.for_each(\|(idx, field)\| {
	match StatisticsConverter::try_new(
	field.name(),
	&file_schema,
	file_metadata.schema_descr(),
	) {
	Ok(stats_converter) => {
	let mut accumulators = StatisticsAccumulators {
	min_accs: &mut min_accs,
	max_accs: &mut max_accs,
	null_counts_array: &mut null_counts_array,
	is_min_value_exact: &mut is_min_value_exact,
	is_max_value_exact: &mut is_max_value_exact,
	};
	summarize_min_max_null_counts(
	&mut accumulators,
	idx,
	num_rows,
	&stats_converter,
	row_groups_metadata,
	)
	.ok();
	}
	Err(e) => {
	debug!("Failed to create statistics converter: {e}");
	null_counts_array[idx] = Precision::Exact(num_rows);
	}
	}
	});

	get_col_stats(
	table_schema,
	null_counts_array,
	&mut max_accs,
	&mut min_accs,
	&mut is_max_value_exact,
	&mut is_min_value_exact,
	)

Incorrect filtering results when collect_statistics = true and column statistics are partially disabled #18241

Description

Describe the bug

To Reproduce

Expected behavior

Additional context

Activity

xhwhis commented on Oct 23, 2025

cdelmonte-zg commented on Aug 27, 2026

cdelmonte-zg commented on Aug 28, 2026

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions

Incorrect filtering results when `collect_statistics = true` and column statistics are partially disabled #18241