diff --git a/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_cow_null_identity_partition.q b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_cow_null_identity_partition.q new file mode 100644 index 000000000000..e88b8915fd6a --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_cow_null_identity_partition.q @@ -0,0 +1,35 @@ +-- SORT_QUERY_RESULTS + +set hive.explain.user=false; + +drop table if exists iceberg_cow_partitioned; + +create external table iceberg_cow_partitioned ( + index int, + string_col string, + boolean_col boolean, + str_col string, + tinyint_col int +) partitioned by spec(str_col, tinyint_col) +stored by iceberg +tblproperties ('write.update.mode'='copy-on-write'); + +insert into iceberg_cow_partitioned partition (str_col, tinyint_col) + values (1, 'a', true, null, 1); + +explain update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null; +update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null; + +select * from iceberg_cow_partitioned; + +-- Disable vectorization + +set hive.vectorized.execution.enabled=false; + +insert into iceberg_cow_partitioned partition (str_col, tinyint_col) + values (2, 'b', false, null, 2); + +explain update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null; +update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null; + +select * from iceberg_cow_partitioned; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_folded.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_folded.q.out index f046c4e58741..2dc19a43757d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_folded.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_folded.q.out @@ -141,12 +141,12 @@ STAGE PLANS: TableScan alias: ice_folded filterExpr: ((id > 4) and (p = 'a')) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (id > 4) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.4 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out index f8711b175f14..9461d2cfbf29 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_colstats_granularity.q.out @@ -71,7 +71,7 @@ STAGE PLANS: TableScan alias: ice_p filterExpr: (p = 'a') (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: id (type: int), 'a' (type: string) outputColumnNames: _col0, _col1 @@ -156,7 +156,7 @@ STAGE PLANS: TableScan alias: ice_p filterExpr: (p = 'a') (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: id (type: int), 'a' (type: string) outputColumnNames: _col0, _col1 @@ -198,7 +198,7 @@ STAGE PLANS: TableScan alias: ice_p filterExpr: (p = 'a') (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: id (type: int), 'a' (type: string) outputColumnNames: _col0, _col1 @@ -275,7 +275,7 @@ STAGE PLANS: TableScan alias: ice_p filterExpr: (p = 'a') (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: id (type: int), 'a' (type: string) outputColumnNames: _col0, _col1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index ec615722ee62..64b1f98bcc8a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -267,7 +267,7 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 3 (type: int) outputColumnNames: _col0, _col1, _col2 @@ -374,7 +374,7 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 4 (type: int) outputColumnNames: _col0, _col1, _col2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index cb6f6d671c93..e40017d1fe5e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -57,7 +57,7 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 @@ -164,7 +164,7 @@ STAGE PLANS: TableScan alias: ice_parquet_int filterExpr: (pcol = 2) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: strcol (type: string), intcol (type: int), 1 (type: int) outputColumnNames: _col0, _col1, _col2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out index fb78a8a72b80..25fb1c3fb4c2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_partition_pruner_cache_key.q.out @@ -272,9 +272,9 @@ STAGE PLANS: TableScan alias: tbl_ice_pp_key filterExpr: (a > 2) (type: boolean) - Statistics: Num rows: 8 Data size: 51110 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - Statistics: Num rows: 8 Data size: 51110 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() minReductionHashAggr: 0.99 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out index 262bae9056ef..05fb89172b94 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out @@ -58,7 +58,7 @@ STAGE PLANS: TableScan alias: ice_01 filterExpr: ds is null (type: boolean) - Statistics: Num rows: 1 Data size: 171 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 355 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_cow_null_identity_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_cow_null_identity_partition.q.out new file mode 100644 index 000000000000..0a4da3ba4039 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_cow_null_identity_partition.q.out @@ -0,0 +1,518 @@ +PREHOOK: query: drop table if exists iceberg_cow_partitioned +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists iceberg_cow_partitioned +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table iceberg_cow_partitioned ( + index int, + string_col string, + boolean_col boolean, + str_col string, + tinyint_col int +) partitioned by spec(str_col, tinyint_col) +stored by iceberg +tblproperties ('write.update.mode'='copy-on-write') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@iceberg_cow_partitioned +POSTHOOK: query: create external table iceberg_cow_partitioned ( + index int, + string_col string, + boolean_col boolean, + str_col string, + tinyint_col int +) partitioned by spec(str_col, tinyint_col) +stored by iceberg +tblproperties ('write.update.mode'='copy-on-write') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@iceberg_cow_partitioned +PREHOOK: query: insert into iceberg_cow_partitioned partition (str_col, tinyint_col) + values (1, 'a', true, null, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@iceberg_cow_partitioned +POSTHOOK: query: insert into iceberg_cow_partitioned partition (str_col, tinyint_col) + values (1, 'a', true, null, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@iceberg_cow_partitioned +PREHOOK: query: explain update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_cow_partitioned +PREHOOK: Output: default@iceberg_cow_partitioned +POSTHOOK: query: explain update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_cow_partitioned +POSTHOOK: Output: default@iceberg_cow_partitioned +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 7 <- Map 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: iceberg_cow_partitioned + filterExpr: str_col is null (type: boolean) + Statistics: Num rows: 1 Data size: 281 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), index (type: int), string_col (type: string), boolean_col (type: boolean), 'UPDATED NULLS' (type: string), tinyint_col (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 398 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 1259 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.iceberg_cow_partitioned + Execution mode: vectorized + Map 3 + Map Operator Tree: + TableScan + alias: iceberg_cow_partitioned + filterExpr: str_col is not null (type: boolean) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (FILE__PATH is not null and str_col is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: index (type: int), string_col (type: string), boolean_col (type: boolean), str_col (type: string), tinyint_col (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col7 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col7 (type: string) + Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: boolean), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: bigint), _col8 (type: bigint) + Execution mode: vectorized + Map 5 + Map Operator Tree: + TableScan + alias: iceberg_cow_partitioned + filterExpr: str_col is null (type: boolean) + Statistics: Num rows: 1 Data size: 281 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: str_col is null (type: boolean) + Reduce Output Operator + key expressions: FILE__PATH (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: FILE__PATH (type: string) + Statistics: Num rows: 1 Data size: 281 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: index (type: int), string_col (type: string), boolean_col (type: boolean), tinyint_col (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) + Filter Operator + predicate: (FILE__PATH is not null and str_col is null) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: FILE__PATH (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: FILE__PATH (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: vectorized + Reducer 4 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col7 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: boolean), _col3 (type: string), _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 1259 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.iceberg_cow_partitioned + Reducer 6 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: boolean), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 293 Basic stats: COMPLETE Column stats: PARTIAL + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST + partition by: _col7 + raw input shape: + window functions: + window function definition + alias: row_number_window_0 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 1 Data size: 293 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (row_number_window_0 = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 293 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: boolean), null (type: string), _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 385 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 1259 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.iceberg_cow_partitioned + Reducer 7 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col7 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col7: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST + partition by: _col7 + raw input shape: + window functions: + window function definition + alias: row_number_window_0 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (row_number_window_0 = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col7 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.iceberg_cow_partitioned + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_cow_partitioned +PREHOOK: Output: default@iceberg_cow_partitioned +POSTHOOK: query: update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_cow_partitioned +POSTHOOK: Output: default@iceberg_cow_partitioned +PREHOOK: query: select * from iceberg_cow_partitioned +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_cow_partitioned +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from iceberg_cow_partitioned +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_cow_partitioned +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 a true UPDATED NULLS 1 +PREHOOK: query: insert into iceberg_cow_partitioned partition (str_col, tinyint_col) + values (2, 'b', false, null, 2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@iceberg_cow_partitioned +POSTHOOK: query: insert into iceberg_cow_partitioned partition (str_col, tinyint_col) + values (2, 'b', false, null, 2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@iceberg_cow_partitioned +PREHOOK: query: explain update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_cow_partitioned +PREHOOK: Output: default@iceberg_cow_partitioned +POSTHOOK: query: explain update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_cow_partitioned +POSTHOOK: Output: default@iceberg_cow_partitioned +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 7 <- Map 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: iceberg_cow_partitioned + filterExpr: str_col is null (type: boolean) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint), index (type: int), string_col (type: string), boolean_col (type: boolean), 'UPDATED NULLS' (type: string), tinyint_col (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 497 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 1565 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.iceberg_cow_partitioned + Map 3 + Map Operator Tree: + TableScan + alias: iceberg_cow_partitioned + filterExpr: str_col is not null (type: boolean) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: FILE__PATH is not null (type: boolean) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: index (type: int), string_col (type: string), boolean_col (type: boolean), str_col (type: string), tinyint_col (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint), FILE__PATH (type: string), ROW__POSITION (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col7 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col7 (type: string) + Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: boolean), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: bigint), _col8 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: iceberg_cow_partitioned + filterExpr: str_col is null (type: boolean) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: str_col is null (type: boolean) + Reduce Output Operator + key expressions: FILE__PATH (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: FILE__PATH (type: string) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: index (type: int), string_col (type: string), boolean_col (type: boolean), tinyint_col (type: int), PARTITION__SPEC__ID (type: int), PARTITION__HASH (type: bigint) + Filter Operator + predicate: (FILE__PATH is not null and str_col is null) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: FILE__PATH (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: FILE__PATH (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 4 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col7 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), _col8 (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: boolean), _col3 (type: string), _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 584 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 1565 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.iceberg_cow_partitioned + Reducer 6 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: boolean), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: bigint), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE Column stats: PARTIAL + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST + partition by: _col7 + raw input shape: + window functions: + window function definition + alias: row_number_window_0 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (row_number_window_0 = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col5 (type: int), _col6 (type: bigint), _col7 (type: string), -1L (type: bigint), _col0 (type: int), _col1 (type: string), _col2 (type: boolean), null (type: string), _col4 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 484 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 1565 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.iceberg_cow_partitioned + Reducer 7 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col7 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col7: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST + partition by: _col7 + raw input shape: + window functions: + window function definition + alias: row_number_window_0 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (row_number_window_0 = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col7 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.iceberg_cow_partitioned + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_cow_partitioned +PREHOOK: Output: default@iceberg_cow_partitioned +POSTHOOK: query: update iceberg_cow_partitioned set str_col = 'UPDATED NULLS' where str_col is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_cow_partitioned +POSTHOOK: Output: default@iceberg_cow_partitioned +PREHOOK: query: select * from iceberg_cow_partitioned +PREHOOK: type: QUERY +PREHOOK: Input: default@iceberg_cow_partitioned +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from iceberg_cow_partitioned +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iceberg_cow_partitioned +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 a true UPDATED NULLS 1 +2 b false UPDATED NULLS 2 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 8306c650ebb9..6a79d2587f24 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; @@ -65,6 +66,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; @@ -496,6 +498,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, } cols = cols == null ? new ArrayList() : cols; + cols = includePartitionColumnsFromScanFilters(scanOp, cols); cppCtx.getPrunedColLists().put((Operator) nd, cols); RowSchema inputRS = scanOp.getSchema(); @@ -503,6 +506,75 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, return null; } + + /** + * For storage handlers with non-native partitions (Iceberg), keep identity partition source + * columns in the scan when they are referenced by pushed storage filters or filters above the + * scan in the local read pipeline. Without this, filters can reference partition columns that + * were pruned from the read projection. + */ + private static List includePartitionColumnsFromScanFilters(TableScanOperator scanOp, + List cols) { + TableScanDesc desc = scanOp.getConf(); + Table table = desc.getTableMetadata(); + if (table == null || !table.hasNonNativePartitionSupport()) { + return cols; + } + + ExprNodeGenericFuncDesc filterExpr = desc.getFilterExpr(); + if (filterExpr != null) { + cols = mergeFieldNodesWithDesc(cols, filterExpr); + } + + if (scanOp.getChildOperators() != null) { + for (Operator child : scanOp.getChildOperators()) { + cols = mergePartitionColumnsFromFiltersAboveScan(child, cols); + } + } + return cols; + } + + /** + * Walks operators above a TableScan in the local read pipeline and merges columns referenced by + * Filter predicates. Traversal follows passthrough operators (Select, PTF, lateral view join, + * Limit), mirroring predicate pushdown boundaries in {@code OpProcFactory.SimpleFilterPPD}. + * Stops at shuffle, join, grouping and sink operators. + */ + private static List mergePartitionColumnsFromFiltersAboveScan( + Operator op, List cols) { + if (isScanPipelineBoundary(op)) { + return cols; + } + + if (op instanceof FilterOperator filterOperator) { + cols = mergeFieldNodesWithDesc(cols, filterOperator.getConf().getPredicate()); + } else if (!isPassthroughOperatorAboveScan(op)) { + return cols; + } + + if (op.getChildOperators() == null) { + return cols; + } + for (Operator child : op.getChildOperators()) { + cols = mergePartitionColumnsFromFiltersAboveScan(child, cols); + } + return cols; + } + + private static boolean isPassthroughOperatorAboveScan(Operator op) { + return op instanceof SelectOperator + || op instanceof PTFOperator + || op instanceof LateralViewJoinOperator + || op instanceof LimitOperator; + } + + private static boolean isScanPipelineBoundary(Operator op) { + return op instanceof ReduceSinkOperator + || op instanceof GroupByOperator + || op instanceof CommonJoinOperator + || op instanceof UnionOperator + || op instanceof FileSinkOperator; + } } /** Sets up needed columns for TSOP. Mainly, transfers column names from input