@@ -824,20 +824,63 @@ def test_write_pandas_with_dynamic_parts(odps, use_arrow):
824
824
@pyarrow_case
825
825
@pandas_case
826
826
@odps2_typed_case
827
- def test_write_pandas_with_complex_type_and_mapping (odps ):
827
+ def test_write_pandas_with_arrow_complex_type (odps ):
828
828
if Version (pa .__version__ ) < Version ("1.0.0" ):
829
829
pytest .skip ("casting nested type is not supported in arrow < 1.0.0" )
830
- test_table_name = tn ("pyodps_t_tmp_write_pd_complex_type " )
830
+ test_table_name = tn ("pyodps_t_tmp_write_arrow_complex_type " )
831
831
odps .delete_table (test_table_name , if_exists = True )
832
832
833
- table = odps .create_table (
834
- test_table_name ,
835
- "idx string, list_data array<bigint>, "
836
- "list_struct_data array<struct<name:string, val: bigint>>, "
837
- "map_data map<string, bigint>" ,
838
- table_properties = {"columnar.nested.type" : "true" },
839
- lifecycle = 1 ,
833
+ data = pd .DataFrame (
834
+ [
835
+ [
836
+ "05ac09c4" ,
837
+ [134 , 256 ],
838
+ [None , {"name" : "col1" , "val" : 134 }],
839
+ ],
840
+ ["cfae9054" , [5431 ], [{"name" : "col2" , "val" : 2345 }]],
841
+ [
842
+ "6029501d" ,
843
+ [145 , None , 561 ],
844
+ [{"name" : "ddd" , "val" : 2341 }, {"name" : None , "val" : None }],
845
+ ],
846
+ [
847
+ "c653e520" ,
848
+ [7412 , 234 ],
849
+ [None , {"name" : "uvw" , "val" : None }],
850
+ ],
851
+ ["59caed0d" , [295 , 1674 ], None ],
852
+ ],
853
+ columns = ["idx" , "list_data" , "list_struct_data" ],
840
854
)
855
+ arrow_data = pa .Table .from_pandas (data )
856
+ try :
857
+ table_kwargs = {
858
+ "table_properties" : {"columnar.nested.type" : "true" },
859
+ }
860
+ odps .write_table (
861
+ test_table_name ,
862
+ arrow_data ,
863
+ create_table = True ,
864
+ lifecycle = 1 ,
865
+ table_kwargs = table_kwargs ,
866
+ )
867
+ table = odps .get_table (test_table_name )
868
+ pd .testing .assert_frame_equal (
869
+ data .sort_values ("idx" ).reset_index (drop = True ),
870
+ table .to_pandas ().sort_values ("idx" ).reset_index (drop = True ),
871
+ )
872
+ finally :
873
+ odps .delete_table (test_table_name , if_exists = True )
874
+
875
+
876
+ @pyarrow_case
877
+ @pandas_case
878
+ @odps2_typed_case
879
+ def test_write_pandas_with_complex_type_and_mapping (odps ):
880
+ if Version (pa .__version__ ) < Version ("1.0.0" ):
881
+ pytest .skip ("casting nested type is not supported in arrow < 1.0.0" )
882
+ test_table_name = tn ("pyodps_t_tmp_write_pd_complex_type" )
883
+ odps .delete_table (test_table_name , if_exists = True )
841
884
842
885
data = pd .DataFrame (
843
886
[
@@ -870,19 +913,24 @@ def test_write_pandas_with_complex_type_and_mapping(odps):
870
913
"list_struct_data" : "array<struct<name:string, val: bigint>>" ,
871
914
"map_data" : "map<string, bigint>" ,
872
915
}
916
+ table_kwargs = {
917
+ "table_properties" : {"columnar.nested.type" : "true" },
918
+ }
873
919
odps .write_table (
874
920
test_table_name ,
875
921
data ,
876
922
type_mapping = type_mapping ,
877
923
create_table = True ,
878
924
lifecycle = 1 ,
925
+ table_kwargs = table_kwargs ,
879
926
)
927
+ table = odps .get_table (test_table_name )
880
928
pd .testing .assert_frame_equal (
881
929
data .sort_values ("idx" ).reset_index (drop = True ),
882
930
table .to_pandas ().sort_values ("idx" ).reset_index (drop = True ),
883
931
)
884
932
finally :
885
- table . drop ( )
933
+ odps . delete_table ( test_table_name , if_exists = True )
886
934
887
935
888
936
@pyarrow_case
0 commit comments