paxus_etl - Details for Job 193

Details for Job 193

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 47

Enable zooming

digraph G { subgraph clusterstage_966 { label="Stage 966 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_967 { label="Stage 967 (skipped)"; subgraph cluster2906 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 905 [label="FileScanRDD [905]\nsave at NativeMethodAccessorImpl.java:0"]; 906 [label="MapPartitionsRDD [906]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2902 { label="WholeStageCodegen (8)"; 907 [label="MapPartitionsRDD [907]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2901 { label="Exchange"; 908 [label="MapPartitionsRDD [908]\nsave at NativeMethodAccessorImpl.java:0"]; } } 905->906; 906->907; 907->908; }

digraph G { subgraph clusterstage_968 { label="Stage 968 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2992 { label="InMemoryTableScan"; 953 [label="MapPartitionsRDD [953]\nsave at NativeMethodAccessorImpl.java:0"]; 954 [label="MapPartitionsRDD [954]\nsave at NativeMethodAccessorImpl.java:0"]; 955 [label="MapPartitionsRDD [955]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2987 { label="WholeStageCodegen (25)"; 956 [label="MapPartitionsRDD [956]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2986 { label="Exchange"; 957 [label="MapPartitionsRDD [957]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->953; 953->954; 954->955; 955->956; 956->957; }

56,57

digraph G { subgraph clusterstage_969 { label="Stage 969 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2974 { label="InMemoryTableScan"; 943 [label="MapPartitionsRDD [943]\nsave at NativeMethodAccessorImpl.java:0"]; 944 [label="MapPartitionsRDD [944]\nsave at NativeMethodAccessorImpl.java:0"]; 945 [label="MapPartitionsRDD [945]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2969 { label="WholeStageCodegen (21)"; 946 [label="MapPartitionsRDD [946]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2968 { label="Exchange"; 947 [label="MapPartitionsRDD [947]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->943; 943->944; 944->945; 945->946; 946->947; }

56,57

digraph G { subgraph clusterstage_970 { label="Stage 970 (skipped)"; subgraph cluster2957 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryconfigtable"; 934 [label="FileScanRDD [934]\nsave at NativeMethodAccessorImpl.java:0"]; 935 [label="MapPartitionsRDD [935]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2953 { label="WholeStageCodegen (18)"; 936 [label="MapPartitionsRDD [936]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2952 { label="Exchange"; 937 [label="MapPartitionsRDD [937]\nsave at NativeMethodAccessorImpl.java:0"]; } } 934->935; 935->936; 936->937; }

digraph G { subgraph clusterstage_971 { label="Stage 971 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2965 { label="InMemoryTableScan"; 938 [label="MapPartitionsRDD [938]\nsave at NativeMethodAccessorImpl.java:0"]; 939 [label="MapPartitionsRDD [939]\nsave at NativeMethodAccessorImpl.java:0"]; 940 [label="MapPartitionsRDD [940]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2960 { label="WholeStageCodegen (19)"; 941 [label="MapPartitionsRDD [941]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2959 { label="Exchange"; 942 [label="MapPartitionsRDD [942]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->938; 938->939; 939->940; 940->941; 941->942; }

56,57

digraph G { subgraph clusterstage_972 { label="Stage 972 (skipped)"; subgraph cluster2891 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryconfigtable"; 896 [label="FileScanRDD [896]\nsave at NativeMethodAccessorImpl.java:0"]; 897 [label="MapPartitionsRDD [897]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2887 { label="WholeStageCodegen (6)"; 898 [label="MapPartitionsRDD [898]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2886 { label="Exchange"; 899 [label="MapPartitionsRDD [899]\nsave at NativeMethodAccessorImpl.java:0"]; } } 896->897; 897->898; 898->899; }

digraph G { subgraph clusterstage_973 { label="Stage 973 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2941 { label="InMemoryTableScan"; 924 [label="MapPartitionsRDD [924]\nsave at NativeMethodAccessorImpl.java:0"]; 925 [label="MapPartitionsRDD [925]\nsave at NativeMethodAccessorImpl.java:0"]; 926 [label="MapPartitionsRDD [926]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2936 { label="WholeStageCodegen (15)"; 927 [label="MapPartitionsRDD [927]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2935 { label="Exchange"; 928 [label="MapPartitionsRDD [928]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->924; 924->925; 925->926; 926->927; 927->928; }

56,57

digraph G { subgraph clusterstage_974 { label="Stage 974 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2932 { label="InMemoryTableScan"; 919 [label="MapPartitionsRDD [919]\nsave at NativeMethodAccessorImpl.java:0"]; 920 [label="MapPartitionsRDD [920]\nsave at NativeMethodAccessorImpl.java:0"]; 921 [label="MapPartitionsRDD [921]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2927 { label="WholeStageCodegen (13)"; 922 [label="MapPartitionsRDD [922]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2926 { label="Exchange"; 923 [label="MapPartitionsRDD [923]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->919; 919->920; 920->921; 921->922; 922->923; }

56,57

digraph G { subgraph clusterstage_975 { label="Stage 975 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2923 { label="InMemoryTableScan"; 914 [label="MapPartitionsRDD [914]\nsave at NativeMethodAccessorImpl.java:0"]; 915 [label="MapPartitionsRDD [915]\nsave at NativeMethodAccessorImpl.java:0"]; 916 [label="MapPartitionsRDD [916]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2918 { label="WholeStageCodegen (11)"; 917 [label="MapPartitionsRDD [917]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2917 { label="Exchange"; 918 [label="MapPartitionsRDD [918]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->914; 914->915; 915->916; 916->917; 917->918; }

56,57

digraph G { subgraph clusterstage_976 { label="Stage 976 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2899 { label="InMemoryTableScan"; 900 [label="MapPartitionsRDD [900]\nsave at NativeMethodAccessorImpl.java:0"]; 901 [label="MapPartitionsRDD [901]\nsave at NativeMethodAccessorImpl.java:0"]; 902 [label="MapPartitionsRDD [902]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2894 { label="WholeStageCodegen (7)"; 903 [label="MapPartitionsRDD [903]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2893 { label="Exchange"; 904 [label="MapPartitionsRDD [904]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->900; 900->901; 901->902; 902->903; 903->904; }

56,57

digraph G { subgraph clusterstage_977 { label="Stage 977 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2884 { label="InMemoryTableScan"; 891 [label="MapPartitionsRDD [891]\nsave at NativeMethodAccessorImpl.java:0"]; 892 [label="MapPartitionsRDD [892]\nsave at NativeMethodAccessorImpl.java:0"]; 893 [label="MapPartitionsRDD [893]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2879 { label="WholeStageCodegen (5)"; 894 [label="MapPartitionsRDD [894]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2878 { label="Exchange"; 895 [label="MapPartitionsRDD [895]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->891; 891->892; 892->893; 893->894; 894->895; }

56,57

digraph G { subgraph clusterstage_978 { label="Stage 978 (skipped)"; subgraph cluster2876 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 887 [label="FileScanRDD [887]\nsave at NativeMethodAccessorImpl.java:0"]; 888 [label="MapPartitionsRDD [888]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2872 { label="WholeStageCodegen (4)"; 889 [label="MapPartitionsRDD [889]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2871 { label="Exchange"; 890 [label="MapPartitionsRDD [890]\nsave at NativeMethodAccessorImpl.java:0"]; } } 887->888; 888->889; 889->890; }

digraph G { subgraph clusterstage_979 { label="Stage 979 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2863 { label="InMemoryTableScan"; 877 [label="MapPartitionsRDD [877]\nsave at NativeMethodAccessorImpl.java:0"]; 878 [label="MapPartitionsRDD [878]\nsave at NativeMethodAccessorImpl.java:0"]; 879 [label="MapPartitionsRDD [879]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2858 { label="WholeStageCodegen (2)"; 880 [label="MapPartitionsRDD [880]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2857 { label="Exchange"; 881 [label="MapPartitionsRDD [881]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->877; 877->878; 878->879; 879->880; 880->881; }

56,57

digraph G { subgraph clusterstage_980 { label="Stage 980 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2869 { label="InMemoryTableScan"; 882 [label="MapPartitionsRDD [882]\nsave at NativeMethodAccessorImpl.java:0"]; 883 [label="MapPartitionsRDD [883]\nsave at NativeMethodAccessorImpl.java:0"]; 884 [label="MapPartitionsRDD [884]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2866 { label="WholeStageCodegen (3)"; 885 [label="MapPartitionsRDD [885]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2865 { label="Exchange"; 886 [label="MapPartitionsRDD [886]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->882; 882->883; 883->884; 884->885; 885->886; }

56,57

digraph G { subgraph clusterstage_981 { label="Stage 981 (skipped)"; subgraph cluster2855 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.usagetable"; 873 [label="FileScanRDD [873]\nsave at NativeMethodAccessorImpl.java:0"]; 874 [label="MapPartitionsRDD [874]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2851 { label="WholeStageCodegen (1)"; 875 [label="MapPartitionsRDD [875]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2850 { label="Exchange"; 876 [label="MapPartitionsRDD [876]\nsave at NativeMethodAccessorImpl.java:0"]; } } 873->874; 874->875; 875->876; }

digraph G { subgraph clusterstage_982 { label="Stage 982 (skipped)"; subgraph cluster3035 { label="Exchange"; 968 [label="ShuffledRowRDD [968]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3038 { label="Exchange"; 969 [label="ShuffledRowRDD [969]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3026 { label="WholeStageCodegen (30)"; 970 [label="ZippedPartitionsRDD2 [970]\nsave at NativeMethodAccessorImpl.java:0"]; 971 [label="MapPartitionsRDD [971]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3044 { label="Exchange"; 972 [label="ShuffledRowRDD [972]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3040 { label="WholeStageCodegen (31)"; 973 [label="MapPartitionsRDD [973]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3023 { label="WholeStageCodegen (32)"; 974 [label="ZippedPartitionsRDD2 [974]\nsave at NativeMethodAccessorImpl.java:0"]; 975 [label="MapPartitionsRDD [975]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3022 { label="Exchange"; 976 [label="MapPartitionsRDD [976]\nsave at NativeMethodAccessorImpl.java:0"]; } } 968->970; 969->970; 970->971; 972->973; 971->974; 973->974; 974->975; 975->976; }

876,968

881,969

886,972

digraph G { subgraph clusterstage_983 { label="Stage 983 (skipped)"; subgraph cluster3235 { label="Exchange"; 1027 [label="ShuffledRowRDD [1027]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3238 { label="Exchange"; 1028 [label="ShuffledRowRDD [1028]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3226 { label="WholeStageCodegen (57)"; 1029 [label="ZippedPartitionsRDD2 [1029]\nsave at NativeMethodAccessorImpl.java:0"]; 1030 [label="MapPartitionsRDD [1030]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3225 { label="Window"; 1031 [label="MapPartitionsRDD [1031]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3221 { label="WholeStageCodegen (58)"; 1032 [label="MapPartitionsRDD [1032]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3220 { label="Exchange"; 1033 [label="MapPartitionsRDD [1033]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1027->1029; 1028->1029; 1029->1030; 1030->1031; 1031->1032; 1032->1033; }

890,1027

895,1028

digraph G { subgraph clusterstage_984 { label="Stage 984 (skipped)"; subgraph cluster3380 { label="AQEShuffleRead"; 1071 [label="ShuffledRowRDD [1071]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3377 { label="WholeStageCodegen (71)"; 1072 [label="MapPartitionsRDD [1072]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3385 { label="AQEShuffleRead"; 1073 [label="ShuffledRowRDD [1073]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3382 { label="WholeStageCodegen (72)"; 1074 [label="MapPartitionsRDD [1074]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3373 { label="WholeStageCodegen (73)"; 1075 [label="ZippedPartitionsRDD2 [1075]\nsave at NativeMethodAccessorImpl.java:0"]; 1076 [label="MapPartitionsRDD [1076]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3372 { label="Exchange"; 1077 [label="MapPartitionsRDD [1077]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1071->1072; 1073->1074; 1072->1075; 1074->1075; 1075->1076; 1076->1077; }

976,1071

1033,1073

digraph G { subgraph clusterstage_985 { label="Stage 985 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3068 { label="WholeStageCodegen (33)"; 978 [label="MapPartitionsRDD [978]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3078 { label="Exchange"; 979 [label="ShuffledRowRDD [979]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3074 { label="WholeStageCodegen (34)"; 980 [label="MapPartitionsRDD [980]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3061 { label="WholeStageCodegen (35)"; 981 [label="ZippedPartitionsRDD2 [981]\nsave at NativeMethodAccessorImpl.java:0"]; 982 [label="MapPartitionsRDD [982]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3060 { label="Window"; 983 [label="MapPartitionsRDD [983]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3056 { label="WholeStageCodegen (36)"; 984 [label="MapPartitionsRDD [984]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3055 { label="Exchange"; 985 [label="MapPartitionsRDD [985]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->978; 979->980; 978->981; 980->981; 981->982; 982->983; 983->984; 984->985; }

899,977

904,979

digraph G { subgraph clusterstage_986 { label="Stage 986 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2914 { label="InMemoryTableScan"; 909 [label="MapPartitionsRDD [909]\nsave at NativeMethodAccessorImpl.java:0"]; 910 [label="MapPartitionsRDD [910]\nsave at NativeMethodAccessorImpl.java:0"]; 911 [label="MapPartitionsRDD [911]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2909 { label="WholeStageCodegen (9)"; 912 [label="MapPartitionsRDD [912]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2908 { label="Exchange"; 913 [label="MapPartitionsRDD [913]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->909; 909->910; 910->911; 911->912; 912->913; }

56,57

digraph G { subgraph clusterstage_987 { label="Stage 987 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3264 { label="Exchange"; 1035 [label="ShuffledRowRDD [1035]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3252 { label="WholeStageCodegen (59)"; 1036 [label="ZippedPartitionsRDD2 [1036]\nsave at NativeMethodAccessorImpl.java:0"]; 1037 [label="MapPartitionsRDD [1037]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3251 { label="Window"; 1038 [label="MapPartitionsRDD [1038]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3247 { label="WholeStageCodegen (60)"; 1039 [label="MapPartitionsRDD [1039]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3246 { label="Exchange"; 1040 [label="MapPartitionsRDD [1040]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1036; 1035->1036; 1036->1037; 1037->1038; 1038->1039; 1039->1040; }

908,1034

913,1035

digraph G { subgraph clusterstage_988 { label="Stage 988 (skipped)"; subgraph cluster3402 { label="AQEShuffleRead"; 1078 [label="ShuffledRowRDD [1078]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3399 { label="WholeStageCodegen (74)"; 1079 [label="MapPartitionsRDD [1079]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3407 { label="AQEShuffleRead"; 1080 [label="ShuffledRowRDD [1080]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3404 { label="WholeStageCodegen (75)"; 1081 [label="MapPartitionsRDD [1081]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3395 { label="WholeStageCodegen (76)"; 1082 [label="ZippedPartitionsRDD2 [1082]\nsave at NativeMethodAccessorImpl.java:0"]; 1083 [label="MapPartitionsRDD [1083]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3394 { label="Exchange"; 1084 [label="MapPartitionsRDD [1084]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1078->1079; 1080->1081; 1079->1082; 1081->1082; 1082->1083; 1083->1084; }

1077,1078

985,1080

digraph G { subgraph clusterstage_989 { label="Stage 989 (skipped)"; subgraph cluster3418 { label="AQEShuffleRead"; 1085 [label="ShuffledRowRDD [1085]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3415 { label="WholeStageCodegen (77)"; 1086 [label="MapPartitionsRDD [1086]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3423 { label="AQEShuffleRead"; 1087 [label="ShuffledRowRDD [1087]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3420 { label="WholeStageCodegen (78)"; 1088 [label="MapPartitionsRDD [1088]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3411 { label="WholeStageCodegen (79)"; 1089 [label="ZippedPartitionsRDD2 [1089]\nsave at NativeMethodAccessorImpl.java:0"]; 1090 [label="MapPartitionsRDD [1090]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3410 { label="Exchange"; 1091 [label="MapPartitionsRDD [1091]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1085->1086; 1087->1088; 1086->1089; 1088->1089; 1089->1090; 1090->1091; }

1084,1085

1040,1087

digraph G { subgraph clusterstage_990 { label="Stage 990 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3093 { label="WholeStageCodegen (37)"; 986 [label="MapPartitionsRDD [986]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3104 { label="Exchange"; 987 [label="ShuffledRowRDD [987]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3100 { label="WholeStageCodegen (38)"; 988 [label="MapPartitionsRDD [988]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3086 { label="WholeStageCodegen (39)"; 989 [label="ZippedPartitionsRDD2 [989]\nsave at NativeMethodAccessorImpl.java:0"]; 990 [label="MapPartitionsRDD [990]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3085 { label="Window"; 991 [label="MapPartitionsRDD [991]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3081 { label="WholeStageCodegen (40)"; 992 [label="MapPartitionsRDD [992]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3080 { label="Exchange"; 993 [label="MapPartitionsRDD [993]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->986; 987->988; 986->989; 988->989; 989->990; 990->991; 991->992; 992->993; }

899,977

918,987

digraph G { subgraph clusterstage_991 { label="Stage 991 (skipped)"; subgraph cluster3434 { label="AQEShuffleRead"; 1092 [label="ShuffledRowRDD [1092]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3431 { label="WholeStageCodegen (80)"; 1093 [label="MapPartitionsRDD [1093]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3439 { label="AQEShuffleRead"; 1094 [label="ShuffledRowRDD [1094]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3436 { label="WholeStageCodegen (81)"; 1095 [label="MapPartitionsRDD [1095]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3427 { label="WholeStageCodegen (82)"; 1096 [label="ZippedPartitionsRDD2 [1096]\nsave at NativeMethodAccessorImpl.java:0"]; 1097 [label="MapPartitionsRDD [1097]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3426 { label="Exchange"; 1098 [label="MapPartitionsRDD [1098]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1092->1093; 1094->1095; 1093->1096; 1095->1096; 1096->1097; 1097->1098; }

1091,1092

993,1094

digraph G { subgraph clusterstage_992 { label="Stage 992 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3285 { label="Exchange"; 1041 [label="ShuffledRowRDD [1041]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3272 { label="WholeStageCodegen (61)"; 1042 [label="ZippedPartitionsRDD2 [1042]\nsave at NativeMethodAccessorImpl.java:0"]; 1043 [label="MapPartitionsRDD [1043]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3271 { label="Window"; 1044 [label="MapPartitionsRDD [1044]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3267 { label="WholeStageCodegen (62)"; 1045 [label="MapPartitionsRDD [1045]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3266 { label="Exchange"; 1046 [label="MapPartitionsRDD [1046]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1042; 1041->1042; 1042->1043; 1043->1044; 1044->1045; 1045->1046; }

908,1034

923,1041

digraph G { subgraph clusterstage_993 { label="Stage 993 (skipped)"; subgraph cluster3450 { label="AQEShuffleRead"; 1099 [label="ShuffledRowRDD [1099]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3447 { label="WholeStageCodegen (83)"; 1100 [label="MapPartitionsRDD [1100]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3455 { label="AQEShuffleRead"; 1101 [label="ShuffledRowRDD [1101]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3452 { label="WholeStageCodegen (84)"; 1102 [label="MapPartitionsRDD [1102]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3443 { label="WholeStageCodegen (85)"; 1103 [label="ZippedPartitionsRDD2 [1103]\nsave at NativeMethodAccessorImpl.java:0"]; 1104 [label="MapPartitionsRDD [1104]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3442 { label="Exchange"; 1105 [label="MapPartitionsRDD [1105]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1099->1100; 1101->1102; 1100->1103; 1102->1103; 1103->1104; 1104->1105; }

1098,1099

1046,1101

digraph G { subgraph clusterstage_994 { label="Stage 994 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3119 { label="WholeStageCodegen (41)"; 994 [label="MapPartitionsRDD [994]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3130 { label="Exchange"; 995 [label="ShuffledRowRDD [995]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3126 { label="WholeStageCodegen (42)"; 996 [label="MapPartitionsRDD [996]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3112 { label="WholeStageCodegen (43)"; 997 [label="ZippedPartitionsRDD2 [997]\nsave at NativeMethodAccessorImpl.java:0"]; 998 [label="MapPartitionsRDD [998]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3111 { label="Window"; 999 [label="MapPartitionsRDD [999]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3107 { label="WholeStageCodegen (44)"; 1000 [label="MapPartitionsRDD [1000]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3106 { label="Exchange"; 1001 [label="MapPartitionsRDD [1001]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->994; 995->996; 994->997; 996->997; 997->998; 998->999; 999->1000; 1000->1001; }

899,977

928,995

digraph G { subgraph clusterstage_995 { label="Stage 995 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2950 { label="InMemoryTableScan"; 929 [label="MapPartitionsRDD [929]\nsave at NativeMethodAccessorImpl.java:0"]; 930 [label="MapPartitionsRDD [930]\nsave at NativeMethodAccessorImpl.java:0"]; 931 [label="MapPartitionsRDD [931]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2945 { label="WholeStageCodegen (17)"; 932 [label="MapPartitionsRDD [932]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2944 { label="Exchange"; 933 [label="MapPartitionsRDD [933]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->929; 929->930; 930->931; 931->932; 932->933; }

56,57

digraph G { subgraph clusterstage_996 { label="Stage 996 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3306 { label="Exchange"; 1047 [label="ShuffledRowRDD [1047]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3293 { label="WholeStageCodegen (63)"; 1048 [label="ZippedPartitionsRDD2 [1048]\nsave at NativeMethodAccessorImpl.java:0"]; 1049 [label="MapPartitionsRDD [1049]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3292 { label="Window"; 1050 [label="MapPartitionsRDD [1050]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3288 { label="WholeStageCodegen (64)"; 1051 [label="MapPartitionsRDD [1051]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3287 { label="Exchange"; 1052 [label="MapPartitionsRDD [1052]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1048; 1047->1048; 1048->1049; 1049->1050; 1050->1051; 1051->1052; }

908,1034

933,1047

digraph G { subgraph clusterstage_997 { label="Stage 997 (skipped)"; subgraph cluster3466 { label="AQEShuffleRead"; 1106 [label="ShuffledRowRDD [1106]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3463 { label="WholeStageCodegen (86)"; 1107 [label="MapPartitionsRDD [1107]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3471 { label="AQEShuffleRead"; 1108 [label="ShuffledRowRDD [1108]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3468 { label="WholeStageCodegen (87)"; 1109 [label="MapPartitionsRDD [1109]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3459 { label="WholeStageCodegen (88)"; 1110 [label="ZippedPartitionsRDD2 [1110]\nsave at NativeMethodAccessorImpl.java:0"]; 1111 [label="MapPartitionsRDD [1111]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3458 { label="Exchange"; 1112 [label="MapPartitionsRDD [1112]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1106->1107; 1108->1109; 1107->1110; 1109->1110; 1110->1111; 1111->1112; }

1105,1106

1001,1108

digraph G { subgraph clusterstage_998 { label="Stage 998 (skipped)"; subgraph cluster3482 { label="AQEShuffleRead"; 1113 [label="ShuffledRowRDD [1113]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3479 { label="WholeStageCodegen (89)"; 1114 [label="MapPartitionsRDD [1114]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3487 { label="AQEShuffleRead"; 1115 [label="ShuffledRowRDD [1115]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3484 { label="WholeStageCodegen (90)"; 1116 [label="MapPartitionsRDD [1116]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3475 { label="WholeStageCodegen (91)"; 1117 [label="ZippedPartitionsRDD2 [1117]\nsave at NativeMethodAccessorImpl.java:0"]; 1118 [label="MapPartitionsRDD [1118]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3474 { label="Exchange"; 1119 [label="MapPartitionsRDD [1119]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1113->1114; 1115->1116; 1114->1117; 1116->1117; 1117->1118; 1118->1119; }

1112,1113

1052,1115

digraph G { subgraph clusterstage_999 { label="Stage 999 (skipped)"; subgraph cluster3153 { label="Exchange"; 1002 [label="ShuffledRowRDD [1002]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3149 { label="WholeStageCodegen (45)"; 1003 [label="MapPartitionsRDD [1003]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3159 { label="Exchange"; 1004 [label="ShuffledRowRDD [1004]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3155 { label="WholeStageCodegen (46)"; 1005 [label="MapPartitionsRDD [1005]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3142 { label="WholeStageCodegen (47)"; 1006 [label="ZippedPartitionsRDD2 [1006]\nsave at NativeMethodAccessorImpl.java:0"]; 1007 [label="MapPartitionsRDD [1007]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3141 { label="Window"; 1008 [label="MapPartitionsRDD [1008]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3137 { label="WholeStageCodegen (48)"; 1009 [label="MapPartitionsRDD [1009]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3136 { label="Exchange"; 1010 [label="MapPartitionsRDD [1010]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1002->1003; 1004->1005; 1003->1006; 1005->1006; 1006->1007; 1007->1008; 1008->1009; 1009->1010; }

937,1002

942,1004

digraph G { subgraph clusterstage_1000 { label="Stage 1000 (skipped)"; subgraph cluster3498 { label="AQEShuffleRead"; 1120 [label="ShuffledRowRDD [1120]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3495 { label="WholeStageCodegen (92)"; 1121 [label="MapPartitionsRDD [1121]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3503 { label="AQEShuffleRead"; 1122 [label="ShuffledRowRDD [1122]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3500 { label="WholeStageCodegen (93)"; 1123 [label="MapPartitionsRDD [1123]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3491 { label="WholeStageCodegen (94)"; 1124 [label="ZippedPartitionsRDD2 [1124]\nsave at NativeMethodAccessorImpl.java:0"]; 1125 [label="MapPartitionsRDD [1125]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3490 { label="Exchange"; 1126 [label="MapPartitionsRDD [1126]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1120->1121; 1122->1123; 1121->1124; 1123->1124; 1124->1125; 1125->1126; }

1119,1120

1010,1122

digraph G { subgraph clusterstage_1001 { label="Stage 1001 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3327 { label="Exchange"; 1053 [label="ShuffledRowRDD [1053]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3314 { label="WholeStageCodegen (65)"; 1054 [label="ZippedPartitionsRDD2 [1054]\nsave at NativeMethodAccessorImpl.java:0"]; 1055 [label="MapPartitionsRDD [1055]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3313 { label="Window"; 1056 [label="MapPartitionsRDD [1056]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3309 { label="WholeStageCodegen (66)"; 1057 [label="MapPartitionsRDD [1057]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3308 { label="Exchange"; 1058 [label="MapPartitionsRDD [1058]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1054; 1053->1054; 1054->1055; 1055->1056; 1056->1057; 1057->1058; }

908,1034

947,1053

digraph G { subgraph clusterstage_1002 { label="Stage 1002 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2983 { label="InMemoryTableScan"; 948 [label="MapPartitionsRDD [948]\nsave at NativeMethodAccessorImpl.java:0"]; 949 [label="MapPartitionsRDD [949]\nsave at NativeMethodAccessorImpl.java:0"]; 950 [label="MapPartitionsRDD [950]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2978 { label="WholeStageCodegen (23)"; 951 [label="MapPartitionsRDD [951]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2977 { label="Exchange"; 952 [label="MapPartitionsRDD [952]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->948; 948->949; 949->950; 950->951; 951->952; }

56,57

digraph G { subgraph clusterstage_1003 { label="Stage 1003 (skipped)"; subgraph cluster3153 { label="Exchange"; 1002 [label="ShuffledRowRDD [1002]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3174 { label="WholeStageCodegen (49)"; 1011 [label="MapPartitionsRDD [1011]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3185 { label="Exchange"; 1012 [label="ShuffledRowRDD [1012]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3181 { label="WholeStageCodegen (50)"; 1013 [label="MapPartitionsRDD [1013]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3167 { label="WholeStageCodegen (51)"; 1014 [label="ZippedPartitionsRDD2 [1014]\nsave at NativeMethodAccessorImpl.java:0"]; 1015 [label="MapPartitionsRDD [1015]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3166 { label="Window"; 1016 [label="MapPartitionsRDD [1016]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3162 { label="WholeStageCodegen (52)"; 1017 [label="MapPartitionsRDD [1017]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3161 { label="Exchange"; 1018 [label="MapPartitionsRDD [1018]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1002->1011; 1012->1013; 1011->1014; 1013->1014; 1014->1015; 1015->1016; 1016->1017; 1017->1018; }

937,1002

952,1012

digraph G { subgraph clusterstage_1004 { label="Stage 1004 (skipped)"; subgraph cluster3514 { label="AQEShuffleRead"; 1127 [label="ShuffledRowRDD [1127]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3511 { label="WholeStageCodegen (95)"; 1128 [label="MapPartitionsRDD [1128]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3519 { label="AQEShuffleRead"; 1129 [label="ShuffledRowRDD [1129]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3516 { label="WholeStageCodegen (96)"; 1130 [label="MapPartitionsRDD [1130]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3507 { label="WholeStageCodegen (97)"; 1131 [label="ZippedPartitionsRDD2 [1131]\nsave at NativeMethodAccessorImpl.java:0"]; 1132 [label="MapPartitionsRDD [1132]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3506 { label="Exchange"; 1133 [label="MapPartitionsRDD [1133]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1127->1128; 1129->1130; 1128->1131; 1130->1131; 1131->1132; 1132->1133; }

1126,1127

1058,1129

digraph G { subgraph clusterstage_1005 { label="Stage 1005 (skipped)"; subgraph cluster3530 { label="AQEShuffleRead"; 1134 [label="ShuffledRowRDD [1134]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3527 { label="WholeStageCodegen (98)"; 1135 [label="MapPartitionsRDD [1135]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3535 { label="AQEShuffleRead"; 1136 [label="ShuffledRowRDD [1136]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3532 { label="WholeStageCodegen (99)"; 1137 [label="MapPartitionsRDD [1137]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3523 { label="WholeStageCodegen (100)"; 1138 [label="ZippedPartitionsRDD2 [1138]\nsave at NativeMethodAccessorImpl.java:0"]; 1139 [label="MapPartitionsRDD [1139]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3522 { label="Exchange"; 1140 [label="MapPartitionsRDD [1140]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1134->1135; 1136->1137; 1135->1138; 1137->1138; 1138->1139; 1139->1140; }

1133,1134

1018,1136

digraph G { subgraph clusterstage_1006 { label="Stage 1006 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3369 { label="Exchange"; 1065 [label="ShuffledRowRDD [1065]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3356 { label="WholeStageCodegen (69)"; 1066 [label="ZippedPartitionsRDD2 [1066]\nsave at NativeMethodAccessorImpl.java:0"]; 1067 [label="MapPartitionsRDD [1067]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3355 { label="Window"; 1068 [label="MapPartitionsRDD [1068]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3351 { label="WholeStageCodegen (70)"; 1069 [label="MapPartitionsRDD [1069]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3350 { label="Exchange"; 1070 [label="MapPartitionsRDD [1070]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1066; 1065->1066; 1066->1067; 1067->1068; 1068->1069; 1069->1070; }

908,1034

957,1065

digraph G { subgraph clusterstage_1007 { label="Stage 1007 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3001 { label="InMemoryTableScan"; 958 [label="MapPartitionsRDD [958]\nsave at NativeMethodAccessorImpl.java:0"]; 959 [label="MapPartitionsRDD [959]\nsave at NativeMethodAccessorImpl.java:0"]; 960 [label="MapPartitionsRDD [960]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2996 { label="WholeStageCodegen (27)"; 961 [label="MapPartitionsRDD [961]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2995 { label="Exchange"; 962 [label="MapPartitionsRDD [962]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->958; 958->959; 959->960; 960->961; 961->962; }

56,57

digraph G { subgraph clusterstage_1008 { label="Stage 1008 (skipped)"; subgraph cluster3153 { label="Exchange"; 1002 [label="ShuffledRowRDD [1002]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3200 { label="WholeStageCodegen (53)"; 1019 [label="MapPartitionsRDD [1019]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3211 { label="Exchange"; 1020 [label="ShuffledRowRDD [1020]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3207 { label="WholeStageCodegen (54)"; 1021 [label="MapPartitionsRDD [1021]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3193 { label="WholeStageCodegen (55)"; 1022 [label="ZippedPartitionsRDD2 [1022]\nsave at NativeMethodAccessorImpl.java:0"]; 1023 [label="MapPartitionsRDD [1023]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3192 { label="Window"; 1024 [label="MapPartitionsRDD [1024]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3188 { label="WholeStageCodegen (56)"; 1025 [label="MapPartitionsRDD [1025]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3187 { label="Exchange"; 1026 [label="MapPartitionsRDD [1026]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1002->1019; 1020->1021; 1019->1022; 1021->1022; 1022->1023; 1023->1024; 1024->1025; 1025->1026; }

937,1002

962,1020

digraph G { subgraph clusterstage_1009 { label="Stage 1009 (skipped)"; subgraph cluster3546 { label="AQEShuffleRead"; 1141 [label="ShuffledRowRDD [1141]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3543 { label="WholeStageCodegen (101)"; 1142 [label="MapPartitionsRDD [1142]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3551 { label="AQEShuffleRead"; 1143 [label="ShuffledRowRDD [1143]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3548 { label="WholeStageCodegen (102)"; 1144 [label="MapPartitionsRDD [1144]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3539 { label="WholeStageCodegen (103)"; 1145 [label="ZippedPartitionsRDD2 [1145]\nsave at NativeMethodAccessorImpl.java:0"]; 1146 [label="MapPartitionsRDD [1146]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3538 { label="Exchange"; 1147 [label="MapPartitionsRDD [1147]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1141->1142; 1143->1144; 1142->1145; 1144->1145; 1145->1146; 1146->1147; }

1140,1141

1070,1143

digraph G { subgraph clusterstage_1010 { label="Stage 1010 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3010 { label="InMemoryTableScan"; 963 [label="MapPartitionsRDD [963]\nsave at NativeMethodAccessorImpl.java:0"]; 964 [label="MapPartitionsRDD [964]\nsave at NativeMethodAccessorImpl.java:0"]; 965 [label="MapPartitionsRDD [965]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3005 { label="WholeStageCodegen (29)"; 966 [label="MapPartitionsRDD [966]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3004 { label="Exchange"; 967 [label="MapPartitionsRDD [967]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->963; 963->964; 964->965; 965->966; 966->967; }

56,57

digraph G { subgraph clusterstage_1011 { label="Stage 1011 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3348 { label="Exchange"; 1059 [label="ShuffledRowRDD [1059]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3335 { label="WholeStageCodegen (67)"; 1060 [label="ZippedPartitionsRDD2 [1060]\nsave at NativeMethodAccessorImpl.java:0"]; 1061 [label="MapPartitionsRDD [1061]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3334 { label="Window"; 1062 [label="MapPartitionsRDD [1062]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3330 { label="WholeStageCodegen (68)"; 1063 [label="MapPartitionsRDD [1063]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3329 { label="Exchange"; 1064 [label="MapPartitionsRDD [1064]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1060; 1059->1060; 1060->1061; 1061->1062; 1062->1063; 1063->1064; }

908,1034

967,1059

digraph G { subgraph clusterstage_1012 { label="Stage 1012 (skipped)"; subgraph cluster3562 { label="AQEShuffleRead"; 1148 [label="ShuffledRowRDD [1148]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3559 { label="WholeStageCodegen (104)"; 1149 [label="MapPartitionsRDD [1149]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3567 { label="AQEShuffleRead"; 1150 [label="ShuffledRowRDD [1150]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3564 { label="WholeStageCodegen (105)"; 1151 [label="MapPartitionsRDD [1151]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3555 { label="WholeStageCodegen (106)"; 1152 [label="ZippedPartitionsRDD2 [1152]\nsave at NativeMethodAccessorImpl.java:0"]; 1153 [label="MapPartitionsRDD [1153]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3554 { label="Exchange"; 1154 [label="MapPartitionsRDD [1154]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1148->1149; 1150->1151; 1149->1152; 1151->1152; 1152->1153; 1153->1154; }

1147,1148

1026,1150

digraph G { subgraph clusterstage_1013 { label="Stage 1013"; subgraph cluster3579 { label="AQEShuffleRead"; 1155 [label="ShuffledRowRDD [1155]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3576 { label="WholeStageCodegen (107)"; 1156 [label="MapPartitionsRDD [1156]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3584 { label="AQEShuffleRead"; 1157 [label="ShuffledRowRDD [1157]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3581 { label="WholeStageCodegen (108)"; 1158 [label="MapPartitionsRDD [1158]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3571 { label="WholeStageCodegen (109)"; 1159 [label="ZippedPartitionsRDD2 [1159]\nsave at NativeMethodAccessorImpl.java:0"]; 1160 [label="MapPartitionsRDD [1160]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3570 { label="Exchange"; 1161 [label="MapPartitionsRDD [1161]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1155->1156; 1157->1158; 1156->1159; 1158->1159; 1159->1160; 1160->1161; }

1154,1155

1064,1157

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
1013	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:29:17	2 s	632/632			186.2 MB	63.3 MB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

1013

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:29:17

2 s

632/632

186.2 MB

63.3 MB

Skipped Stages (47)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
1012	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/632
1011	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
1010	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
1009	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/661
1008	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
1007	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
1006	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
1005	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/735
1004	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/717
1003	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
1002	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
1001	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
1000	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/872
999	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
998	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/80
997	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/90
996	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
995	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
994	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
993	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/85
992	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
991	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/97
990	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
989	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/87
988	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/845
987	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
986	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
985	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
984	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/780
983	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
982	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
981	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/757
980	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
979	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
978	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
977	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
976	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
975	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
974	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
973	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
972	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
971	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
970	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
969	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
968	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
967	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
966	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433