Fix Pandas2 compatibility for Hive (#32752)

potiuk · Cloud Composer Team · commit 560efc0463b7 · 2024-05-15T21:56:59.000Z
Pandas2 removed deprecated iteritems() in a few classes with items() being replacement pandas-dev/pandas#37545 The Hive Hook used iteritems when converting to dataframes. Pandas2 upgrade has been enabled now by databricks-sql upgrade so it is now possible to use Pandas2 and update our constraints. GitOrigin-RevId: 00aa6ea72c4d72df6c62ac1dda9bb145bece2e1c
diff --git a/airflow/providers/apache/hive/hooks/hive.py b/airflow/providers/apache/hive/hooks/hive.py
@@ -376,7 +376,7 @@ def _infer_field_types_from_df(df: pandas.DataFrame) -> dict[Any, Any]:
             }
 
             order_type = OrderedDict()
-            for col, dtype in df.dtypes.iteritems():
+            for col, dtype in df.dtypes.items():
                 order_type[col] = dtype_kind_hive_type[dtype.kind]
             return order_type
 

Original file line number	Diff line number	Diff line change
`@@ -376,7 +376,7 @@ def _infer_field_types_from_df(df: pandas.DataFrame) -> dict[Any, Any]:`
`376`	`376`	`}`
`377`	`377`
`378`	`378`	`order_type = OrderedDict()`
`379`		`- for col, dtype in df.dtypes.iteritems():`
	`379`	`+ for col, dtype in df.dtypes.items():`
`380`	`380`	`order_type[col] = dtype_kind_hive_type[dtype.kind]`
`381`	`381`	`return order_type`
`382`	`382`