paxus_etl - Details for Job 191

Details for Job 191

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 41

Enable zooming

digraph G { subgraph clusterstage_879 { label="Stage 879 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_880 { label="Stage 880 (skipped)"; subgraph cluster2906 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 905 [label="FileScanRDD [905]\nsave at NativeMethodAccessorImpl.java:0"]; 906 [label="MapPartitionsRDD [906]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2902 { label="WholeStageCodegen (8)"; 907 [label="MapPartitionsRDD [907]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2901 { label="Exchange"; 908 [label="MapPartitionsRDD [908]\nsave at NativeMethodAccessorImpl.java:0"]; } } 905->906; 906->907; 907->908; }

digraph G { subgraph clusterstage_881 { label="Stage 881 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2992 { label="InMemoryTableScan"; 953 [label="MapPartitionsRDD [953]\nsave at NativeMethodAccessorImpl.java:0"]; 954 [label="MapPartitionsRDD [954]\nsave at NativeMethodAccessorImpl.java:0"]; 955 [label="MapPartitionsRDD [955]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2987 { label="WholeStageCodegen (25)"; 956 [label="MapPartitionsRDD [956]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2986 { label="Exchange"; 957 [label="MapPartitionsRDD [957]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->953; 953->954; 954->955; 955->956; 956->957; }

56,57

digraph G { subgraph clusterstage_882 { label="Stage 882 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2974 { label="InMemoryTableScan"; 943 [label="MapPartitionsRDD [943]\nsave at NativeMethodAccessorImpl.java:0"]; 944 [label="MapPartitionsRDD [944]\nsave at NativeMethodAccessorImpl.java:0"]; 945 [label="MapPartitionsRDD [945]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2969 { label="WholeStageCodegen (21)"; 946 [label="MapPartitionsRDD [946]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2968 { label="Exchange"; 947 [label="MapPartitionsRDD [947]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->943; 943->944; 944->945; 945->946; 946->947; }

56,57

digraph G { subgraph clusterstage_883 { label="Stage 883 (skipped)"; subgraph cluster2957 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryconfigtable"; 934 [label="FileScanRDD [934]\nsave at NativeMethodAccessorImpl.java:0"]; 935 [label="MapPartitionsRDD [935]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2953 { label="WholeStageCodegen (18)"; 936 [label="MapPartitionsRDD [936]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2952 { label="Exchange"; 937 [label="MapPartitionsRDD [937]\nsave at NativeMethodAccessorImpl.java:0"]; } } 934->935; 935->936; 936->937; }

digraph G { subgraph clusterstage_884 { label="Stage 884 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2965 { label="InMemoryTableScan"; 938 [label="MapPartitionsRDD [938]\nsave at NativeMethodAccessorImpl.java:0"]; 939 [label="MapPartitionsRDD [939]\nsave at NativeMethodAccessorImpl.java:0"]; 940 [label="MapPartitionsRDD [940]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2960 { label="WholeStageCodegen (19)"; 941 [label="MapPartitionsRDD [941]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2959 { label="Exchange"; 942 [label="MapPartitionsRDD [942]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->938; 938->939; 939->940; 940->941; 941->942; }

56,57

digraph G { subgraph clusterstage_885 { label="Stage 885 (skipped)"; subgraph cluster2891 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryconfigtable"; 896 [label="FileScanRDD [896]\nsave at NativeMethodAccessorImpl.java:0"]; 897 [label="MapPartitionsRDD [897]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2887 { label="WholeStageCodegen (6)"; 898 [label="MapPartitionsRDD [898]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2886 { label="Exchange"; 899 [label="MapPartitionsRDD [899]\nsave at NativeMethodAccessorImpl.java:0"]; } } 896->897; 897->898; 898->899; }

digraph G { subgraph clusterstage_886 { label="Stage 886 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2941 { label="InMemoryTableScan"; 924 [label="MapPartitionsRDD [924]\nsave at NativeMethodAccessorImpl.java:0"]; 925 [label="MapPartitionsRDD [925]\nsave at NativeMethodAccessorImpl.java:0"]; 926 [label="MapPartitionsRDD [926]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2936 { label="WholeStageCodegen (15)"; 927 [label="MapPartitionsRDD [927]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2935 { label="Exchange"; 928 [label="MapPartitionsRDD [928]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->924; 924->925; 925->926; 926->927; 927->928; }

56,57

digraph G { subgraph clusterstage_887 { label="Stage 887 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2932 { label="InMemoryTableScan"; 919 [label="MapPartitionsRDD [919]\nsave at NativeMethodAccessorImpl.java:0"]; 920 [label="MapPartitionsRDD [920]\nsave at NativeMethodAccessorImpl.java:0"]; 921 [label="MapPartitionsRDD [921]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2927 { label="WholeStageCodegen (13)"; 922 [label="MapPartitionsRDD [922]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2926 { label="Exchange"; 923 [label="MapPartitionsRDD [923]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->919; 919->920; 920->921; 921->922; 922->923; }

56,57

digraph G { subgraph clusterstage_888 { label="Stage 888 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2923 { label="InMemoryTableScan"; 914 [label="MapPartitionsRDD [914]\nsave at NativeMethodAccessorImpl.java:0"]; 915 [label="MapPartitionsRDD [915]\nsave at NativeMethodAccessorImpl.java:0"]; 916 [label="MapPartitionsRDD [916]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2918 { label="WholeStageCodegen (11)"; 917 [label="MapPartitionsRDD [917]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2917 { label="Exchange"; 918 [label="MapPartitionsRDD [918]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->914; 914->915; 915->916; 916->917; 917->918; }

56,57

digraph G { subgraph clusterstage_889 { label="Stage 889 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2899 { label="InMemoryTableScan"; 900 [label="MapPartitionsRDD [900]\nsave at NativeMethodAccessorImpl.java:0"]; 901 [label="MapPartitionsRDD [901]\nsave at NativeMethodAccessorImpl.java:0"]; 902 [label="MapPartitionsRDD [902]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2894 { label="WholeStageCodegen (7)"; 903 [label="MapPartitionsRDD [903]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2893 { label="Exchange"; 904 [label="MapPartitionsRDD [904]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->900; 900->901; 901->902; 902->903; 903->904; }

56,57

digraph G { subgraph clusterstage_890 { label="Stage 890 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2884 { label="InMemoryTableScan"; 891 [label="MapPartitionsRDD [891]\nsave at NativeMethodAccessorImpl.java:0"]; 892 [label="MapPartitionsRDD [892]\nsave at NativeMethodAccessorImpl.java:0"]; 893 [label="MapPartitionsRDD [893]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2879 { label="WholeStageCodegen (5)"; 894 [label="MapPartitionsRDD [894]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2878 { label="Exchange"; 895 [label="MapPartitionsRDD [895]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->891; 891->892; 892->893; 893->894; 894->895; }

56,57

digraph G { subgraph clusterstage_891 { label="Stage 891 (skipped)"; subgraph cluster2876 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 887 [label="FileScanRDD [887]\nsave at NativeMethodAccessorImpl.java:0"]; 888 [label="MapPartitionsRDD [888]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2872 { label="WholeStageCodegen (4)"; 889 [label="MapPartitionsRDD [889]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2871 { label="Exchange"; 890 [label="MapPartitionsRDD [890]\nsave at NativeMethodAccessorImpl.java:0"]; } } 887->888; 888->889; 889->890; }

digraph G { subgraph clusterstage_892 { label="Stage 892 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2863 { label="InMemoryTableScan"; 877 [label="MapPartitionsRDD [877]\nsave at NativeMethodAccessorImpl.java:0"]; 878 [label="MapPartitionsRDD [878]\nsave at NativeMethodAccessorImpl.java:0"]; 879 [label="MapPartitionsRDD [879]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2858 { label="WholeStageCodegen (2)"; 880 [label="MapPartitionsRDD [880]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2857 { label="Exchange"; 881 [label="MapPartitionsRDD [881]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->877; 877->878; 878->879; 879->880; 880->881; }

56,57

digraph G { subgraph clusterstage_893 { label="Stage 893 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2869 { label="InMemoryTableScan"; 882 [label="MapPartitionsRDD [882]\nsave at NativeMethodAccessorImpl.java:0"]; 883 [label="MapPartitionsRDD [883]\nsave at NativeMethodAccessorImpl.java:0"]; 884 [label="MapPartitionsRDD [884]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2866 { label="WholeStageCodegen (3)"; 885 [label="MapPartitionsRDD [885]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2865 { label="Exchange"; 886 [label="MapPartitionsRDD [886]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->882; 882->883; 883->884; 884->885; 885->886; }

56,57

digraph G { subgraph clusterstage_894 { label="Stage 894 (skipped)"; subgraph cluster2855 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.usagetable"; 873 [label="FileScanRDD [873]\nsave at NativeMethodAccessorImpl.java:0"]; 874 [label="MapPartitionsRDD [874]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2851 { label="WholeStageCodegen (1)"; 875 [label="MapPartitionsRDD [875]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2850 { label="Exchange"; 876 [label="MapPartitionsRDD [876]\nsave at NativeMethodAccessorImpl.java:0"]; } } 873->874; 874->875; 875->876; }

digraph G { subgraph clusterstage_895 { label="Stage 895 (skipped)"; subgraph cluster3035 { label="Exchange"; 968 [label="ShuffledRowRDD [968]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3038 { label="Exchange"; 969 [label="ShuffledRowRDD [969]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3026 { label="WholeStageCodegen (30)"; 970 [label="ZippedPartitionsRDD2 [970]\nsave at NativeMethodAccessorImpl.java:0"]; 971 [label="MapPartitionsRDD [971]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3044 { label="Exchange"; 972 [label="ShuffledRowRDD [972]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3040 { label="WholeStageCodegen (31)"; 973 [label="MapPartitionsRDD [973]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3023 { label="WholeStageCodegen (32)"; 974 [label="ZippedPartitionsRDD2 [974]\nsave at NativeMethodAccessorImpl.java:0"]; 975 [label="MapPartitionsRDD [975]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3022 { label="Exchange"; 976 [label="MapPartitionsRDD [976]\nsave at NativeMethodAccessorImpl.java:0"]; } } 968->970; 969->970; 970->971; 972->973; 971->974; 973->974; 974->975; 975->976; }

876,968

881,969

886,972

digraph G { subgraph clusterstage_896 { label="Stage 896 (skipped)"; subgraph cluster3235 { label="Exchange"; 1027 [label="ShuffledRowRDD [1027]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3238 { label="Exchange"; 1028 [label="ShuffledRowRDD [1028]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3226 { label="WholeStageCodegen (57)"; 1029 [label="ZippedPartitionsRDD2 [1029]\nsave at NativeMethodAccessorImpl.java:0"]; 1030 [label="MapPartitionsRDD [1030]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3225 { label="Window"; 1031 [label="MapPartitionsRDD [1031]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3221 { label="WholeStageCodegen (58)"; 1032 [label="MapPartitionsRDD [1032]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3220 { label="Exchange"; 1033 [label="MapPartitionsRDD [1033]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1027->1029; 1028->1029; 1029->1030; 1030->1031; 1031->1032; 1032->1033; }

890,1027

895,1028

digraph G { subgraph clusterstage_897 { label="Stage 897 (skipped)"; subgraph cluster3380 { label="AQEShuffleRead"; 1071 [label="ShuffledRowRDD [1071]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3377 { label="WholeStageCodegen (71)"; 1072 [label="MapPartitionsRDD [1072]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3385 { label="AQEShuffleRead"; 1073 [label="ShuffledRowRDD [1073]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3382 { label="WholeStageCodegen (72)"; 1074 [label="MapPartitionsRDD [1074]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3373 { label="WholeStageCodegen (73)"; 1075 [label="ZippedPartitionsRDD2 [1075]\nsave at NativeMethodAccessorImpl.java:0"]; 1076 [label="MapPartitionsRDD [1076]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3372 { label="Exchange"; 1077 [label="MapPartitionsRDD [1077]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1071->1072; 1073->1074; 1072->1075; 1074->1075; 1075->1076; 1076->1077; }

976,1071

1033,1073

digraph G { subgraph clusterstage_898 { label="Stage 898 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3068 { label="WholeStageCodegen (33)"; 978 [label="MapPartitionsRDD [978]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3078 { label="Exchange"; 979 [label="ShuffledRowRDD [979]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3074 { label="WholeStageCodegen (34)"; 980 [label="MapPartitionsRDD [980]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3061 { label="WholeStageCodegen (35)"; 981 [label="ZippedPartitionsRDD2 [981]\nsave at NativeMethodAccessorImpl.java:0"]; 982 [label="MapPartitionsRDD [982]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3060 { label="Window"; 983 [label="MapPartitionsRDD [983]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3056 { label="WholeStageCodegen (36)"; 984 [label="MapPartitionsRDD [984]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3055 { label="Exchange"; 985 [label="MapPartitionsRDD [985]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->978; 979->980; 978->981; 980->981; 981->982; 982->983; 983->984; 984->985; }

899,977

904,979

digraph G { subgraph clusterstage_899 { label="Stage 899 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2914 { label="InMemoryTableScan"; 909 [label="MapPartitionsRDD [909]\nsave at NativeMethodAccessorImpl.java:0"]; 910 [label="MapPartitionsRDD [910]\nsave at NativeMethodAccessorImpl.java:0"]; 911 [label="MapPartitionsRDD [911]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2909 { label="WholeStageCodegen (9)"; 912 [label="MapPartitionsRDD [912]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2908 { label="Exchange"; 913 [label="MapPartitionsRDD [913]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->909; 909->910; 910->911; 911->912; 912->913; }

56,57

digraph G { subgraph clusterstage_900 { label="Stage 900 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3264 { label="Exchange"; 1035 [label="ShuffledRowRDD [1035]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3252 { label="WholeStageCodegen (59)"; 1036 [label="ZippedPartitionsRDD2 [1036]\nsave at NativeMethodAccessorImpl.java:0"]; 1037 [label="MapPartitionsRDD [1037]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3251 { label="Window"; 1038 [label="MapPartitionsRDD [1038]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3247 { label="WholeStageCodegen (60)"; 1039 [label="MapPartitionsRDD [1039]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3246 { label="Exchange"; 1040 [label="MapPartitionsRDD [1040]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1036; 1035->1036; 1036->1037; 1037->1038; 1038->1039; 1039->1040; }

908,1034

913,1035

digraph G { subgraph clusterstage_901 { label="Stage 901 (skipped)"; subgraph cluster3402 { label="AQEShuffleRead"; 1078 [label="ShuffledRowRDD [1078]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3399 { label="WholeStageCodegen (74)"; 1079 [label="MapPartitionsRDD [1079]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3407 { label="AQEShuffleRead"; 1080 [label="ShuffledRowRDD [1080]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3404 { label="WholeStageCodegen (75)"; 1081 [label="MapPartitionsRDD [1081]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3395 { label="WholeStageCodegen (76)"; 1082 [label="ZippedPartitionsRDD2 [1082]\nsave at NativeMethodAccessorImpl.java:0"]; 1083 [label="MapPartitionsRDD [1083]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3394 { label="Exchange"; 1084 [label="MapPartitionsRDD [1084]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1078->1079; 1080->1081; 1079->1082; 1081->1082; 1082->1083; 1083->1084; }

1077,1078

985,1080

digraph G { subgraph clusterstage_902 { label="Stage 902 (skipped)"; subgraph cluster3418 { label="AQEShuffleRead"; 1085 [label="ShuffledRowRDD [1085]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3415 { label="WholeStageCodegen (77)"; 1086 [label="MapPartitionsRDD [1086]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3423 { label="AQEShuffleRead"; 1087 [label="ShuffledRowRDD [1087]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3420 { label="WholeStageCodegen (78)"; 1088 [label="MapPartitionsRDD [1088]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3411 { label="WholeStageCodegen (79)"; 1089 [label="ZippedPartitionsRDD2 [1089]\nsave at NativeMethodAccessorImpl.java:0"]; 1090 [label="MapPartitionsRDD [1090]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3410 { label="Exchange"; 1091 [label="MapPartitionsRDD [1091]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1085->1086; 1087->1088; 1086->1089; 1088->1089; 1089->1090; 1090->1091; }

1084,1085

1040,1087

digraph G { subgraph clusterstage_903 { label="Stage 903 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3093 { label="WholeStageCodegen (37)"; 986 [label="MapPartitionsRDD [986]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3104 { label="Exchange"; 987 [label="ShuffledRowRDD [987]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3100 { label="WholeStageCodegen (38)"; 988 [label="MapPartitionsRDD [988]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3086 { label="WholeStageCodegen (39)"; 989 [label="ZippedPartitionsRDD2 [989]\nsave at NativeMethodAccessorImpl.java:0"]; 990 [label="MapPartitionsRDD [990]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3085 { label="Window"; 991 [label="MapPartitionsRDD [991]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3081 { label="WholeStageCodegen (40)"; 992 [label="MapPartitionsRDD [992]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3080 { label="Exchange"; 993 [label="MapPartitionsRDD [993]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->986; 987->988; 986->989; 988->989; 989->990; 990->991; 991->992; 992->993; }

899,977

918,987

digraph G { subgraph clusterstage_904 { label="Stage 904 (skipped)"; subgraph cluster3434 { label="AQEShuffleRead"; 1092 [label="ShuffledRowRDD [1092]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3431 { label="WholeStageCodegen (80)"; 1093 [label="MapPartitionsRDD [1093]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3439 { label="AQEShuffleRead"; 1094 [label="ShuffledRowRDD [1094]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3436 { label="WholeStageCodegen (81)"; 1095 [label="MapPartitionsRDD [1095]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3427 { label="WholeStageCodegen (82)"; 1096 [label="ZippedPartitionsRDD2 [1096]\nsave at NativeMethodAccessorImpl.java:0"]; 1097 [label="MapPartitionsRDD [1097]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3426 { label="Exchange"; 1098 [label="MapPartitionsRDD [1098]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1092->1093; 1094->1095; 1093->1096; 1095->1096; 1096->1097; 1097->1098; }

1091,1092

993,1094

digraph G { subgraph clusterstage_905 { label="Stage 905 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3285 { label="Exchange"; 1041 [label="ShuffledRowRDD [1041]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3272 { label="WholeStageCodegen (61)"; 1042 [label="ZippedPartitionsRDD2 [1042]\nsave at NativeMethodAccessorImpl.java:0"]; 1043 [label="MapPartitionsRDD [1043]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3271 { label="Window"; 1044 [label="MapPartitionsRDD [1044]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3267 { label="WholeStageCodegen (62)"; 1045 [label="MapPartitionsRDD [1045]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3266 { label="Exchange"; 1046 [label="MapPartitionsRDD [1046]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1042; 1041->1042; 1042->1043; 1043->1044; 1044->1045; 1045->1046; }

908,1034

923,1041

digraph G { subgraph clusterstage_906 { label="Stage 906 (skipped)"; subgraph cluster3450 { label="AQEShuffleRead"; 1099 [label="ShuffledRowRDD [1099]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3447 { label="WholeStageCodegen (83)"; 1100 [label="MapPartitionsRDD [1100]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3455 { label="AQEShuffleRead"; 1101 [label="ShuffledRowRDD [1101]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3452 { label="WholeStageCodegen (84)"; 1102 [label="MapPartitionsRDD [1102]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3443 { label="WholeStageCodegen (85)"; 1103 [label="ZippedPartitionsRDD2 [1103]\nsave at NativeMethodAccessorImpl.java:0"]; 1104 [label="MapPartitionsRDD [1104]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3442 { label="Exchange"; 1105 [label="MapPartitionsRDD [1105]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1099->1100; 1101->1102; 1100->1103; 1102->1103; 1103->1104; 1104->1105; }

1098,1099

1046,1101

digraph G { subgraph clusterstage_907 { label="Stage 907 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3119 { label="WholeStageCodegen (41)"; 994 [label="MapPartitionsRDD [994]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3130 { label="Exchange"; 995 [label="ShuffledRowRDD [995]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3126 { label="WholeStageCodegen (42)"; 996 [label="MapPartitionsRDD [996]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3112 { label="WholeStageCodegen (43)"; 997 [label="ZippedPartitionsRDD2 [997]\nsave at NativeMethodAccessorImpl.java:0"]; 998 [label="MapPartitionsRDD [998]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3111 { label="Window"; 999 [label="MapPartitionsRDD [999]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3107 { label="WholeStageCodegen (44)"; 1000 [label="MapPartitionsRDD [1000]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3106 { label="Exchange"; 1001 [label="MapPartitionsRDD [1001]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->994; 995->996; 994->997; 996->997; 997->998; 998->999; 999->1000; 1000->1001; }

899,977

928,995

digraph G { subgraph clusterstage_908 { label="Stage 908 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2950 { label="InMemoryTableScan"; 929 [label="MapPartitionsRDD [929]\nsave at NativeMethodAccessorImpl.java:0"]; 930 [label="MapPartitionsRDD [930]\nsave at NativeMethodAccessorImpl.java:0"]; 931 [label="MapPartitionsRDD [931]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2945 { label="WholeStageCodegen (17)"; 932 [label="MapPartitionsRDD [932]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2944 { label="Exchange"; 933 [label="MapPartitionsRDD [933]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->929; 929->930; 930->931; 931->932; 932->933; }

56,57

digraph G { subgraph clusterstage_909 { label="Stage 909 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3306 { label="Exchange"; 1047 [label="ShuffledRowRDD [1047]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3293 { label="WholeStageCodegen (63)"; 1048 [label="ZippedPartitionsRDD2 [1048]\nsave at NativeMethodAccessorImpl.java:0"]; 1049 [label="MapPartitionsRDD [1049]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3292 { label="Window"; 1050 [label="MapPartitionsRDD [1050]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3288 { label="WholeStageCodegen (64)"; 1051 [label="MapPartitionsRDD [1051]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3287 { label="Exchange"; 1052 [label="MapPartitionsRDD [1052]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1048; 1047->1048; 1048->1049; 1049->1050; 1050->1051; 1051->1052; }

908,1034

933,1047

digraph G { subgraph clusterstage_910 { label="Stage 910 (skipped)"; subgraph cluster3466 { label="AQEShuffleRead"; 1106 [label="ShuffledRowRDD [1106]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3463 { label="WholeStageCodegen (86)"; 1107 [label="MapPartitionsRDD [1107]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3471 { label="AQEShuffleRead"; 1108 [label="ShuffledRowRDD [1108]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3468 { label="WholeStageCodegen (87)"; 1109 [label="MapPartitionsRDD [1109]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3459 { label="WholeStageCodegen (88)"; 1110 [label="ZippedPartitionsRDD2 [1110]\nsave at NativeMethodAccessorImpl.java:0"]; 1111 [label="MapPartitionsRDD [1111]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3458 { label="Exchange"; 1112 [label="MapPartitionsRDD [1112]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1106->1107; 1108->1109; 1107->1110; 1109->1110; 1110->1111; 1111->1112; }

1105,1106

1001,1108

digraph G { subgraph clusterstage_911 { label="Stage 911 (skipped)"; subgraph cluster3482 { label="AQEShuffleRead"; 1113 [label="ShuffledRowRDD [1113]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3479 { label="WholeStageCodegen (89)"; 1114 [label="MapPartitionsRDD [1114]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3487 { label="AQEShuffleRead"; 1115 [label="ShuffledRowRDD [1115]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3484 { label="WholeStageCodegen (90)"; 1116 [label="MapPartitionsRDD [1116]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3475 { label="WholeStageCodegen (91)"; 1117 [label="ZippedPartitionsRDD2 [1117]\nsave at NativeMethodAccessorImpl.java:0"]; 1118 [label="MapPartitionsRDD [1118]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3474 { label="Exchange"; 1119 [label="MapPartitionsRDD [1119]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1113->1114; 1115->1116; 1114->1117; 1116->1117; 1117->1118; 1118->1119; }

1112,1113

1052,1115

digraph G { subgraph clusterstage_912 { label="Stage 912 (skipped)"; subgraph cluster3153 { label="Exchange"; 1002 [label="ShuffledRowRDD [1002]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3149 { label="WholeStageCodegen (45)"; 1003 [label="MapPartitionsRDD [1003]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3159 { label="Exchange"; 1004 [label="ShuffledRowRDD [1004]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3155 { label="WholeStageCodegen (46)"; 1005 [label="MapPartitionsRDD [1005]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3142 { label="WholeStageCodegen (47)"; 1006 [label="ZippedPartitionsRDD2 [1006]\nsave at NativeMethodAccessorImpl.java:0"]; 1007 [label="MapPartitionsRDD [1007]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3141 { label="Window"; 1008 [label="MapPartitionsRDD [1008]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3137 { label="WholeStageCodegen (48)"; 1009 [label="MapPartitionsRDD [1009]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3136 { label="Exchange"; 1010 [label="MapPartitionsRDD [1010]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1002->1003; 1004->1005; 1003->1006; 1005->1006; 1006->1007; 1007->1008; 1008->1009; 1009->1010; }

937,1002

942,1004

digraph G { subgraph clusterstage_913 { label="Stage 913 (skipped)"; subgraph cluster3498 { label="AQEShuffleRead"; 1120 [label="ShuffledRowRDD [1120]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3495 { label="WholeStageCodegen (92)"; 1121 [label="MapPartitionsRDD [1121]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3503 { label="AQEShuffleRead"; 1122 [label="ShuffledRowRDD [1122]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3500 { label="WholeStageCodegen (93)"; 1123 [label="MapPartitionsRDD [1123]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3491 { label="WholeStageCodegen (94)"; 1124 [label="ZippedPartitionsRDD2 [1124]\nsave at NativeMethodAccessorImpl.java:0"]; 1125 [label="MapPartitionsRDD [1125]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3490 { label="Exchange"; 1126 [label="MapPartitionsRDD [1126]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1120->1121; 1122->1123; 1121->1124; 1123->1124; 1124->1125; 1125->1126; }

1119,1120

1010,1122

digraph G { subgraph clusterstage_914 { label="Stage 914 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3327 { label="Exchange"; 1053 [label="ShuffledRowRDD [1053]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3314 { label="WholeStageCodegen (65)"; 1054 [label="ZippedPartitionsRDD2 [1054]\nsave at NativeMethodAccessorImpl.java:0"]; 1055 [label="MapPartitionsRDD [1055]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3313 { label="Window"; 1056 [label="MapPartitionsRDD [1056]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3309 { label="WholeStageCodegen (66)"; 1057 [label="MapPartitionsRDD [1057]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3308 { label="Exchange"; 1058 [label="MapPartitionsRDD [1058]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1054; 1053->1054; 1054->1055; 1055->1056; 1056->1057; 1057->1058; }

908,1034

947,1053

digraph G { subgraph clusterstage_915 { label="Stage 915 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2983 { label="InMemoryTableScan"; 948 [label="MapPartitionsRDD [948]\nsave at NativeMethodAccessorImpl.java:0"]; 949 [label="MapPartitionsRDD [949]\nsave at NativeMethodAccessorImpl.java:0"]; 950 [label="MapPartitionsRDD [950]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2978 { label="WholeStageCodegen (23)"; 951 [label="MapPartitionsRDD [951]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2977 { label="Exchange"; 952 [label="MapPartitionsRDD [952]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->948; 948->949; 949->950; 950->951; 951->952; }

56,57

digraph G { subgraph clusterstage_916 { label="Stage 916 (skipped)"; subgraph cluster3153 { label="Exchange"; 1002 [label="ShuffledRowRDD [1002]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3174 { label="WholeStageCodegen (49)"; 1011 [label="MapPartitionsRDD [1011]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3185 { label="Exchange"; 1012 [label="ShuffledRowRDD [1012]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3181 { label="WholeStageCodegen (50)"; 1013 [label="MapPartitionsRDD [1013]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3167 { label="WholeStageCodegen (51)"; 1014 [label="ZippedPartitionsRDD2 [1014]\nsave at NativeMethodAccessorImpl.java:0"]; 1015 [label="MapPartitionsRDD [1015]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3166 { label="Window"; 1016 [label="MapPartitionsRDD [1016]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3162 { label="WholeStageCodegen (52)"; 1017 [label="MapPartitionsRDD [1017]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3161 { label="Exchange"; 1018 [label="MapPartitionsRDD [1018]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1002->1011; 1012->1013; 1011->1014; 1013->1014; 1014->1015; 1015->1016; 1016->1017; 1017->1018; }

937,1002

952,1012

digraph G { subgraph clusterstage_917 { label="Stage 917 (skipped)"; subgraph cluster3514 { label="AQEShuffleRead"; 1127 [label="ShuffledRowRDD [1127]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3511 { label="WholeStageCodegen (95)"; 1128 [label="MapPartitionsRDD [1128]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3519 { label="AQEShuffleRead"; 1129 [label="ShuffledRowRDD [1129]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3516 { label="WholeStageCodegen (96)"; 1130 [label="MapPartitionsRDD [1130]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3507 { label="WholeStageCodegen (97)"; 1131 [label="ZippedPartitionsRDD2 [1131]\nsave at NativeMethodAccessorImpl.java:0"]; 1132 [label="MapPartitionsRDD [1132]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3506 { label="Exchange"; 1133 [label="MapPartitionsRDD [1133]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1127->1128; 1129->1130; 1128->1131; 1130->1131; 1131->1132; 1132->1133; }

1126,1127

1058,1129

digraph G { subgraph clusterstage_918 { label="Stage 918 (skipped)"; subgraph cluster3530 { label="AQEShuffleRead"; 1134 [label="ShuffledRowRDD [1134]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3527 { label="WholeStageCodegen (98)"; 1135 [label="MapPartitionsRDD [1135]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3535 { label="AQEShuffleRead"; 1136 [label="ShuffledRowRDD [1136]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3532 { label="WholeStageCodegen (99)"; 1137 [label="MapPartitionsRDD [1137]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3523 { label="WholeStageCodegen (100)"; 1138 [label="ZippedPartitionsRDD2 [1138]\nsave at NativeMethodAccessorImpl.java:0"]; 1139 [label="MapPartitionsRDD [1139]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3522 { label="Exchange"; 1140 [label="MapPartitionsRDD [1140]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1134->1135; 1136->1137; 1135->1138; 1137->1138; 1138->1139; 1139->1140; }

1133,1134

1018,1136

digraph G { subgraph clusterstage_919 { label="Stage 919 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3369 { label="Exchange"; 1065 [label="ShuffledRowRDD [1065]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3356 { label="WholeStageCodegen (69)"; 1066 [label="ZippedPartitionsRDD2 [1066]\nsave at NativeMethodAccessorImpl.java:0"]; 1067 [label="MapPartitionsRDD [1067]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3355 { label="Window"; 1068 [label="MapPartitionsRDD [1068]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3351 { label="WholeStageCodegen (70)"; 1069 [label="MapPartitionsRDD [1069]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3350 { label="Exchange"; 1070 [label="MapPartitionsRDD [1070]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1066; 1065->1066; 1066->1067; 1067->1068; 1068->1069; 1069->1070; }

908,1034

957,1065

digraph G { subgraph clusterstage_920 { label="Stage 920"; subgraph cluster3546 { label="AQEShuffleRead"; 1141 [label="ShuffledRowRDD [1141]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3543 { label="WholeStageCodegen (101)"; 1142 [label="MapPartitionsRDD [1142]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3551 { label="AQEShuffleRead"; 1143 [label="ShuffledRowRDD [1143]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3548 { label="WholeStageCodegen (102)"; 1144 [label="MapPartitionsRDD [1144]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3539 { label="WholeStageCodegen (103)"; 1145 [label="ZippedPartitionsRDD2 [1145]\nsave at NativeMethodAccessorImpl.java:0"]; 1146 [label="MapPartitionsRDD [1146]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3538 { label="Exchange"; 1147 [label="MapPartitionsRDD [1147]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1141->1142; 1143->1144; 1142->1145; 1144->1145; 1145->1146; 1146->1147; }

1140,1141

1070,1143

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
920	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:29:14	1 s	661/661			189.5 MB	31.4 MB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

920

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:29:14

1 s

661/661

189.5 MB

31.4 MB

Skipped Stages (41)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
919	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
918	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/735
917	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/717
916	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
915	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
914	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
913	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/872
912	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
911	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/80
910	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/90
909	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
908	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
907	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
906	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/85
905	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
904	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/97
903	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
902	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/87
901	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/845
900	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
899	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
898	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
897	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/780
896	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
895	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
894	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/757
893	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
892	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
891	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
890	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
889	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
888	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
887	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
886	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
885	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
884	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
883	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
882	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
881	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
880	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
879	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433