paxus_etl - Details for Job 190

Details for Job 190

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 38

Enable zooming

digraph G { subgraph clusterstage_840 { label="Stage 840 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_841 { label="Stage 841 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2974 { label="InMemoryTableScan"; 943 [label="MapPartitionsRDD [943]\nsave at NativeMethodAccessorImpl.java:0"]; 944 [label="MapPartitionsRDD [944]\nsave at NativeMethodAccessorImpl.java:0"]; 945 [label="MapPartitionsRDD [945]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2969 { label="WholeStageCodegen (21)"; 946 [label="MapPartitionsRDD [946]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2968 { label="Exchange"; 947 [label="MapPartitionsRDD [947]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->943; 943->944; 944->945; 945->946; 946->947; }

56,57

digraph G { subgraph clusterstage_842 { label="Stage 842 (skipped)"; subgraph cluster2906 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 905 [label="FileScanRDD [905]\nsave at NativeMethodAccessorImpl.java:0"]; 906 [label="MapPartitionsRDD [906]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2902 { label="WholeStageCodegen (8)"; 907 [label="MapPartitionsRDD [907]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2901 { label="Exchange"; 908 [label="MapPartitionsRDD [908]\nsave at NativeMethodAccessorImpl.java:0"]; } } 905->906; 906->907; 907->908; }

digraph G { subgraph clusterstage_843 { label="Stage 843 (skipped)"; subgraph cluster2957 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryconfigtable"; 934 [label="FileScanRDD [934]\nsave at NativeMethodAccessorImpl.java:0"]; 935 [label="MapPartitionsRDD [935]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2953 { label="WholeStageCodegen (18)"; 936 [label="MapPartitionsRDD [936]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2952 { label="Exchange"; 937 [label="MapPartitionsRDD [937]\nsave at NativeMethodAccessorImpl.java:0"]; } } 934->935; 935->936; 936->937; }

digraph G { subgraph clusterstage_844 { label="Stage 844 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2965 { label="InMemoryTableScan"; 938 [label="MapPartitionsRDD [938]\nsave at NativeMethodAccessorImpl.java:0"]; 939 [label="MapPartitionsRDD [939]\nsave at NativeMethodAccessorImpl.java:0"]; 940 [label="MapPartitionsRDD [940]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2960 { label="WholeStageCodegen (19)"; 941 [label="MapPartitionsRDD [941]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2959 { label="Exchange"; 942 [label="MapPartitionsRDD [942]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->938; 938->939; 939->940; 940->941; 941->942; }

56,57

digraph G { subgraph clusterstage_845 { label="Stage 845 (skipped)"; subgraph cluster2891 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryconfigtable"; 896 [label="FileScanRDD [896]\nsave at NativeMethodAccessorImpl.java:0"]; 897 [label="MapPartitionsRDD [897]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2887 { label="WholeStageCodegen (6)"; 898 [label="MapPartitionsRDD [898]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2886 { label="Exchange"; 899 [label="MapPartitionsRDD [899]\nsave at NativeMethodAccessorImpl.java:0"]; } } 896->897; 897->898; 898->899; }

digraph G { subgraph clusterstage_846 { label="Stage 846 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2941 { label="InMemoryTableScan"; 924 [label="MapPartitionsRDD [924]\nsave at NativeMethodAccessorImpl.java:0"]; 925 [label="MapPartitionsRDD [925]\nsave at NativeMethodAccessorImpl.java:0"]; 926 [label="MapPartitionsRDD [926]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2936 { label="WholeStageCodegen (15)"; 927 [label="MapPartitionsRDD [927]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2935 { label="Exchange"; 928 [label="MapPartitionsRDD [928]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->924; 924->925; 925->926; 926->927; 927->928; }

56,57

digraph G { subgraph clusterstage_847 { label="Stage 847 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2932 { label="InMemoryTableScan"; 919 [label="MapPartitionsRDD [919]\nsave at NativeMethodAccessorImpl.java:0"]; 920 [label="MapPartitionsRDD [920]\nsave at NativeMethodAccessorImpl.java:0"]; 921 [label="MapPartitionsRDD [921]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2927 { label="WholeStageCodegen (13)"; 922 [label="MapPartitionsRDD [922]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2926 { label="Exchange"; 923 [label="MapPartitionsRDD [923]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->919; 919->920; 920->921; 921->922; 922->923; }

56,57

digraph G { subgraph clusterstage_848 { label="Stage 848 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2923 { label="InMemoryTableScan"; 914 [label="MapPartitionsRDD [914]\nsave at NativeMethodAccessorImpl.java:0"]; 915 [label="MapPartitionsRDD [915]\nsave at NativeMethodAccessorImpl.java:0"]; 916 [label="MapPartitionsRDD [916]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2918 { label="WholeStageCodegen (11)"; 917 [label="MapPartitionsRDD [917]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2917 { label="Exchange"; 918 [label="MapPartitionsRDD [918]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->914; 914->915; 915->916; 916->917; 917->918; }

56,57

digraph G { subgraph clusterstage_849 { label="Stage 849 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2899 { label="InMemoryTableScan"; 900 [label="MapPartitionsRDD [900]\nsave at NativeMethodAccessorImpl.java:0"]; 901 [label="MapPartitionsRDD [901]\nsave at NativeMethodAccessorImpl.java:0"]; 902 [label="MapPartitionsRDD [902]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2894 { label="WholeStageCodegen (7)"; 903 [label="MapPartitionsRDD [903]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2893 { label="Exchange"; 904 [label="MapPartitionsRDD [904]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->900; 900->901; 901->902; 902->903; 903->904; }

56,57

digraph G { subgraph clusterstage_850 { label="Stage 850 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2884 { label="InMemoryTableScan"; 891 [label="MapPartitionsRDD [891]\nsave at NativeMethodAccessorImpl.java:0"]; 892 [label="MapPartitionsRDD [892]\nsave at NativeMethodAccessorImpl.java:0"]; 893 [label="MapPartitionsRDD [893]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2879 { label="WholeStageCodegen (5)"; 894 [label="MapPartitionsRDD [894]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2878 { label="Exchange"; 895 [label="MapPartitionsRDD [895]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->891; 891->892; 892->893; 893->894; 894->895; }

56,57

digraph G { subgraph clusterstage_851 { label="Stage 851 (skipped)"; subgraph cluster2876 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 887 [label="FileScanRDD [887]\nsave at NativeMethodAccessorImpl.java:0"]; 888 [label="MapPartitionsRDD [888]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2872 { label="WholeStageCodegen (4)"; 889 [label="MapPartitionsRDD [889]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2871 { label="Exchange"; 890 [label="MapPartitionsRDD [890]\nsave at NativeMethodAccessorImpl.java:0"]; } } 887->888; 888->889; 889->890; }

digraph G { subgraph clusterstage_852 { label="Stage 852 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2863 { label="InMemoryTableScan"; 877 [label="MapPartitionsRDD [877]\nsave at NativeMethodAccessorImpl.java:0"]; 878 [label="MapPartitionsRDD [878]\nsave at NativeMethodAccessorImpl.java:0"]; 879 [label="MapPartitionsRDD [879]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2858 { label="WholeStageCodegen (2)"; 880 [label="MapPartitionsRDD [880]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2857 { label="Exchange"; 881 [label="MapPartitionsRDD [881]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->877; 877->878; 878->879; 879->880; 880->881; }

56,57

digraph G { subgraph clusterstage_853 { label="Stage 853 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2869 { label="InMemoryTableScan"; 882 [label="MapPartitionsRDD [882]\nsave at NativeMethodAccessorImpl.java:0"]; 883 [label="MapPartitionsRDD [883]\nsave at NativeMethodAccessorImpl.java:0"]; 884 [label="MapPartitionsRDD [884]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2866 { label="WholeStageCodegen (3)"; 885 [label="MapPartitionsRDD [885]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2865 { label="Exchange"; 886 [label="MapPartitionsRDD [886]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->882; 882->883; 883->884; 884->885; 885->886; }

56,57

digraph G { subgraph clusterstage_854 { label="Stage 854 (skipped)"; subgraph cluster2855 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.usagetable"; 873 [label="FileScanRDD [873]\nsave at NativeMethodAccessorImpl.java:0"]; 874 [label="MapPartitionsRDD [874]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2851 { label="WholeStageCodegen (1)"; 875 [label="MapPartitionsRDD [875]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2850 { label="Exchange"; 876 [label="MapPartitionsRDD [876]\nsave at NativeMethodAccessorImpl.java:0"]; } } 873->874; 874->875; 875->876; }

digraph G { subgraph clusterstage_855 { label="Stage 855 (skipped)"; subgraph cluster3035 { label="Exchange"; 968 [label="ShuffledRowRDD [968]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3038 { label="Exchange"; 969 [label="ShuffledRowRDD [969]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3026 { label="WholeStageCodegen (30)"; 970 [label="ZippedPartitionsRDD2 [970]\nsave at NativeMethodAccessorImpl.java:0"]; 971 [label="MapPartitionsRDD [971]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3044 { label="Exchange"; 972 [label="ShuffledRowRDD [972]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3040 { label="WholeStageCodegen (31)"; 973 [label="MapPartitionsRDD [973]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3023 { label="WholeStageCodegen (32)"; 974 [label="ZippedPartitionsRDD2 [974]\nsave at NativeMethodAccessorImpl.java:0"]; 975 [label="MapPartitionsRDD [975]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3022 { label="Exchange"; 976 [label="MapPartitionsRDD [976]\nsave at NativeMethodAccessorImpl.java:0"]; } } 968->970; 969->970; 970->971; 972->973; 971->974; 973->974; 974->975; 975->976; }

876,968

881,969

886,972

digraph G { subgraph clusterstage_856 { label="Stage 856 (skipped)"; subgraph cluster3235 { label="Exchange"; 1027 [label="ShuffledRowRDD [1027]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3238 { label="Exchange"; 1028 [label="ShuffledRowRDD [1028]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3226 { label="WholeStageCodegen (57)"; 1029 [label="ZippedPartitionsRDD2 [1029]\nsave at NativeMethodAccessorImpl.java:0"]; 1030 [label="MapPartitionsRDD [1030]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3225 { label="Window"; 1031 [label="MapPartitionsRDD [1031]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3221 { label="WholeStageCodegen (58)"; 1032 [label="MapPartitionsRDD [1032]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3220 { label="Exchange"; 1033 [label="MapPartitionsRDD [1033]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1027->1029; 1028->1029; 1029->1030; 1030->1031; 1031->1032; 1032->1033; }

890,1027

895,1028

digraph G { subgraph clusterstage_857 { label="Stage 857 (skipped)"; subgraph cluster3380 { label="AQEShuffleRead"; 1071 [label="ShuffledRowRDD [1071]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3377 { label="WholeStageCodegen (71)"; 1072 [label="MapPartitionsRDD [1072]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3385 { label="AQEShuffleRead"; 1073 [label="ShuffledRowRDD [1073]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3382 { label="WholeStageCodegen (72)"; 1074 [label="MapPartitionsRDD [1074]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3373 { label="WholeStageCodegen (73)"; 1075 [label="ZippedPartitionsRDD2 [1075]\nsave at NativeMethodAccessorImpl.java:0"]; 1076 [label="MapPartitionsRDD [1076]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3372 { label="Exchange"; 1077 [label="MapPartitionsRDD [1077]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1071->1072; 1073->1074; 1072->1075; 1074->1075; 1075->1076; 1076->1077; }

976,1071

1033,1073

digraph G { subgraph clusterstage_858 { label="Stage 858 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3068 { label="WholeStageCodegen (33)"; 978 [label="MapPartitionsRDD [978]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3078 { label="Exchange"; 979 [label="ShuffledRowRDD [979]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3074 { label="WholeStageCodegen (34)"; 980 [label="MapPartitionsRDD [980]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3061 { label="WholeStageCodegen (35)"; 981 [label="ZippedPartitionsRDD2 [981]\nsave at NativeMethodAccessorImpl.java:0"]; 982 [label="MapPartitionsRDD [982]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3060 { label="Window"; 983 [label="MapPartitionsRDD [983]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3056 { label="WholeStageCodegen (36)"; 984 [label="MapPartitionsRDD [984]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3055 { label="Exchange"; 985 [label="MapPartitionsRDD [985]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->978; 979->980; 978->981; 980->981; 981->982; 982->983; 983->984; 984->985; }

899,977

904,979

digraph G { subgraph clusterstage_859 { label="Stage 859 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2914 { label="InMemoryTableScan"; 909 [label="MapPartitionsRDD [909]\nsave at NativeMethodAccessorImpl.java:0"]; 910 [label="MapPartitionsRDD [910]\nsave at NativeMethodAccessorImpl.java:0"]; 911 [label="MapPartitionsRDD [911]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2909 { label="WholeStageCodegen (9)"; 912 [label="MapPartitionsRDD [912]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2908 { label="Exchange"; 913 [label="MapPartitionsRDD [913]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->909; 909->910; 910->911; 911->912; 912->913; }

56,57

digraph G { subgraph clusterstage_860 { label="Stage 860 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3264 { label="Exchange"; 1035 [label="ShuffledRowRDD [1035]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3252 { label="WholeStageCodegen (59)"; 1036 [label="ZippedPartitionsRDD2 [1036]\nsave at NativeMethodAccessorImpl.java:0"]; 1037 [label="MapPartitionsRDD [1037]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3251 { label="Window"; 1038 [label="MapPartitionsRDD [1038]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3247 { label="WholeStageCodegen (60)"; 1039 [label="MapPartitionsRDD [1039]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3246 { label="Exchange"; 1040 [label="MapPartitionsRDD [1040]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1036; 1035->1036; 1036->1037; 1037->1038; 1038->1039; 1039->1040; }

908,1034

913,1035

digraph G { subgraph clusterstage_861 { label="Stage 861 (skipped)"; subgraph cluster3402 { label="AQEShuffleRead"; 1078 [label="ShuffledRowRDD [1078]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3399 { label="WholeStageCodegen (74)"; 1079 [label="MapPartitionsRDD [1079]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3407 { label="AQEShuffleRead"; 1080 [label="ShuffledRowRDD [1080]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3404 { label="WholeStageCodegen (75)"; 1081 [label="MapPartitionsRDD [1081]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3395 { label="WholeStageCodegen (76)"; 1082 [label="ZippedPartitionsRDD2 [1082]\nsave at NativeMethodAccessorImpl.java:0"]; 1083 [label="MapPartitionsRDD [1083]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3394 { label="Exchange"; 1084 [label="MapPartitionsRDD [1084]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1078->1079; 1080->1081; 1079->1082; 1081->1082; 1082->1083; 1083->1084; }

1077,1078

985,1080

digraph G { subgraph clusterstage_862 { label="Stage 862 (skipped)"; subgraph cluster3418 { label="AQEShuffleRead"; 1085 [label="ShuffledRowRDD [1085]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3415 { label="WholeStageCodegen (77)"; 1086 [label="MapPartitionsRDD [1086]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3423 { label="AQEShuffleRead"; 1087 [label="ShuffledRowRDD [1087]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3420 { label="WholeStageCodegen (78)"; 1088 [label="MapPartitionsRDD [1088]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3411 { label="WholeStageCodegen (79)"; 1089 [label="ZippedPartitionsRDD2 [1089]\nsave at NativeMethodAccessorImpl.java:0"]; 1090 [label="MapPartitionsRDD [1090]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3410 { label="Exchange"; 1091 [label="MapPartitionsRDD [1091]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1085->1086; 1087->1088; 1086->1089; 1088->1089; 1089->1090; 1090->1091; }

1084,1085

1040,1087

digraph G { subgraph clusterstage_863 { label="Stage 863 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3093 { label="WholeStageCodegen (37)"; 986 [label="MapPartitionsRDD [986]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3104 { label="Exchange"; 987 [label="ShuffledRowRDD [987]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3100 { label="WholeStageCodegen (38)"; 988 [label="MapPartitionsRDD [988]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3086 { label="WholeStageCodegen (39)"; 989 [label="ZippedPartitionsRDD2 [989]\nsave at NativeMethodAccessorImpl.java:0"]; 990 [label="MapPartitionsRDD [990]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3085 { label="Window"; 991 [label="MapPartitionsRDD [991]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3081 { label="WholeStageCodegen (40)"; 992 [label="MapPartitionsRDD [992]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3080 { label="Exchange"; 993 [label="MapPartitionsRDD [993]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->986; 987->988; 986->989; 988->989; 989->990; 990->991; 991->992; 992->993; }

899,977

918,987

digraph G { subgraph clusterstage_864 { label="Stage 864 (skipped)"; subgraph cluster3434 { label="AQEShuffleRead"; 1092 [label="ShuffledRowRDD [1092]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3431 { label="WholeStageCodegen (80)"; 1093 [label="MapPartitionsRDD [1093]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3439 { label="AQEShuffleRead"; 1094 [label="ShuffledRowRDD [1094]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3436 { label="WholeStageCodegen (81)"; 1095 [label="MapPartitionsRDD [1095]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3427 { label="WholeStageCodegen (82)"; 1096 [label="ZippedPartitionsRDD2 [1096]\nsave at NativeMethodAccessorImpl.java:0"]; 1097 [label="MapPartitionsRDD [1097]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3426 { label="Exchange"; 1098 [label="MapPartitionsRDD [1098]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1092->1093; 1094->1095; 1093->1096; 1095->1096; 1096->1097; 1097->1098; }

1091,1092

993,1094

digraph G { subgraph clusterstage_865 { label="Stage 865 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3285 { label="Exchange"; 1041 [label="ShuffledRowRDD [1041]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3272 { label="WholeStageCodegen (61)"; 1042 [label="ZippedPartitionsRDD2 [1042]\nsave at NativeMethodAccessorImpl.java:0"]; 1043 [label="MapPartitionsRDD [1043]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3271 { label="Window"; 1044 [label="MapPartitionsRDD [1044]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3267 { label="WholeStageCodegen (62)"; 1045 [label="MapPartitionsRDD [1045]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3266 { label="Exchange"; 1046 [label="MapPartitionsRDD [1046]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1042; 1041->1042; 1042->1043; 1043->1044; 1044->1045; 1045->1046; }

908,1034

923,1041

digraph G { subgraph clusterstage_866 { label="Stage 866 (skipped)"; subgraph cluster3450 { label="AQEShuffleRead"; 1099 [label="ShuffledRowRDD [1099]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3447 { label="WholeStageCodegen (83)"; 1100 [label="MapPartitionsRDD [1100]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3455 { label="AQEShuffleRead"; 1101 [label="ShuffledRowRDD [1101]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3452 { label="WholeStageCodegen (84)"; 1102 [label="MapPartitionsRDD [1102]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3443 { label="WholeStageCodegen (85)"; 1103 [label="ZippedPartitionsRDD2 [1103]\nsave at NativeMethodAccessorImpl.java:0"]; 1104 [label="MapPartitionsRDD [1104]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3442 { label="Exchange"; 1105 [label="MapPartitionsRDD [1105]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1099->1100; 1101->1102; 1100->1103; 1102->1103; 1103->1104; 1104->1105; }

1098,1099

1046,1101

digraph G { subgraph clusterstage_867 { label="Stage 867 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3119 { label="WholeStageCodegen (41)"; 994 [label="MapPartitionsRDD [994]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3130 { label="Exchange"; 995 [label="ShuffledRowRDD [995]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3126 { label="WholeStageCodegen (42)"; 996 [label="MapPartitionsRDD [996]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3112 { label="WholeStageCodegen (43)"; 997 [label="ZippedPartitionsRDD2 [997]\nsave at NativeMethodAccessorImpl.java:0"]; 998 [label="MapPartitionsRDD [998]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3111 { label="Window"; 999 [label="MapPartitionsRDD [999]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3107 { label="WholeStageCodegen (44)"; 1000 [label="MapPartitionsRDD [1000]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3106 { label="Exchange"; 1001 [label="MapPartitionsRDD [1001]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->994; 995->996; 994->997; 996->997; 997->998; 998->999; 999->1000; 1000->1001; }

899,977

928,995

digraph G { subgraph clusterstage_868 { label="Stage 868 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2950 { label="InMemoryTableScan"; 929 [label="MapPartitionsRDD [929]\nsave at NativeMethodAccessorImpl.java:0"]; 930 [label="MapPartitionsRDD [930]\nsave at NativeMethodAccessorImpl.java:0"]; 931 [label="MapPartitionsRDD [931]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2945 { label="WholeStageCodegen (17)"; 932 [label="MapPartitionsRDD [932]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2944 { label="Exchange"; 933 [label="MapPartitionsRDD [933]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->929; 929->930; 930->931; 931->932; 932->933; }

56,57

digraph G { subgraph clusterstage_869 { label="Stage 869 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3306 { label="Exchange"; 1047 [label="ShuffledRowRDD [1047]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3293 { label="WholeStageCodegen (63)"; 1048 [label="ZippedPartitionsRDD2 [1048]\nsave at NativeMethodAccessorImpl.java:0"]; 1049 [label="MapPartitionsRDD [1049]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3292 { label="Window"; 1050 [label="MapPartitionsRDD [1050]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3288 { label="WholeStageCodegen (64)"; 1051 [label="MapPartitionsRDD [1051]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3287 { label="Exchange"; 1052 [label="MapPartitionsRDD [1052]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1048; 1047->1048; 1048->1049; 1049->1050; 1050->1051; 1051->1052; }

908,1034

933,1047

digraph G { subgraph clusterstage_870 { label="Stage 870 (skipped)"; subgraph cluster3466 { label="AQEShuffleRead"; 1106 [label="ShuffledRowRDD [1106]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3463 { label="WholeStageCodegen (86)"; 1107 [label="MapPartitionsRDD [1107]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3471 { label="AQEShuffleRead"; 1108 [label="ShuffledRowRDD [1108]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3468 { label="WholeStageCodegen (87)"; 1109 [label="MapPartitionsRDD [1109]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3459 { label="WholeStageCodegen (88)"; 1110 [label="ZippedPartitionsRDD2 [1110]\nsave at NativeMethodAccessorImpl.java:0"]; 1111 [label="MapPartitionsRDD [1111]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3458 { label="Exchange"; 1112 [label="MapPartitionsRDD [1112]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1106->1107; 1108->1109; 1107->1110; 1109->1110; 1110->1111; 1111->1112; }

1105,1106

1001,1108

digraph G { subgraph clusterstage_871 { label="Stage 871 (skipped)"; subgraph cluster3482 { label="AQEShuffleRead"; 1113 [label="ShuffledRowRDD [1113]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3479 { label="WholeStageCodegen (89)"; 1114 [label="MapPartitionsRDD [1114]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3487 { label="AQEShuffleRead"; 1115 [label="ShuffledRowRDD [1115]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3484 { label="WholeStageCodegen (90)"; 1116 [label="MapPartitionsRDD [1116]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3475 { label="WholeStageCodegen (91)"; 1117 [label="ZippedPartitionsRDD2 [1117]\nsave at NativeMethodAccessorImpl.java:0"]; 1118 [label="MapPartitionsRDD [1118]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3474 { label="Exchange"; 1119 [label="MapPartitionsRDD [1119]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1113->1114; 1115->1116; 1114->1117; 1116->1117; 1117->1118; 1118->1119; }

1112,1113

1052,1115

digraph G { subgraph clusterstage_872 { label="Stage 872 (skipped)"; subgraph cluster3153 { label="Exchange"; 1002 [label="ShuffledRowRDD [1002]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3149 { label="WholeStageCodegen (45)"; 1003 [label="MapPartitionsRDD [1003]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3159 { label="Exchange"; 1004 [label="ShuffledRowRDD [1004]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3155 { label="WholeStageCodegen (46)"; 1005 [label="MapPartitionsRDD [1005]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3142 { label="WholeStageCodegen (47)"; 1006 [label="ZippedPartitionsRDD2 [1006]\nsave at NativeMethodAccessorImpl.java:0"]; 1007 [label="MapPartitionsRDD [1007]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3141 { label="Window"; 1008 [label="MapPartitionsRDD [1008]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3137 { label="WholeStageCodegen (48)"; 1009 [label="MapPartitionsRDD [1009]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3136 { label="Exchange"; 1010 [label="MapPartitionsRDD [1010]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1002->1003; 1004->1005; 1003->1006; 1005->1006; 1006->1007; 1007->1008; 1008->1009; 1009->1010; }

937,1002

942,1004

digraph G { subgraph clusterstage_873 { label="Stage 873 (skipped)"; subgraph cluster3498 { label="AQEShuffleRead"; 1120 [label="ShuffledRowRDD [1120]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3495 { label="WholeStageCodegen (92)"; 1121 [label="MapPartitionsRDD [1121]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3503 { label="AQEShuffleRead"; 1122 [label="ShuffledRowRDD [1122]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3500 { label="WholeStageCodegen (93)"; 1123 [label="MapPartitionsRDD [1123]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3491 { label="WholeStageCodegen (94)"; 1124 [label="ZippedPartitionsRDD2 [1124]\nsave at NativeMethodAccessorImpl.java:0"]; 1125 [label="MapPartitionsRDD [1125]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3490 { label="Exchange"; 1126 [label="MapPartitionsRDD [1126]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1120->1121; 1122->1123; 1121->1124; 1123->1124; 1124->1125; 1125->1126; }

1119,1120

1010,1122

digraph G { subgraph clusterstage_874 { label="Stage 874 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3327 { label="Exchange"; 1053 [label="ShuffledRowRDD [1053]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3314 { label="WholeStageCodegen (65)"; 1054 [label="ZippedPartitionsRDD2 [1054]\nsave at NativeMethodAccessorImpl.java:0"]; 1055 [label="MapPartitionsRDD [1055]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3313 { label="Window"; 1056 [label="MapPartitionsRDD [1056]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3309 { label="WholeStageCodegen (66)"; 1057 [label="MapPartitionsRDD [1057]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3308 { label="Exchange"; 1058 [label="MapPartitionsRDD [1058]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1054; 1053->1054; 1054->1055; 1055->1056; 1056->1057; 1057->1058; }

908,1034

947,1053

digraph G { subgraph clusterstage_875 { label="Stage 875 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2983 { label="InMemoryTableScan"; 948 [label="MapPartitionsRDD [948]\nsave at NativeMethodAccessorImpl.java:0"]; 949 [label="MapPartitionsRDD [949]\nsave at NativeMethodAccessorImpl.java:0"]; 950 [label="MapPartitionsRDD [950]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2978 { label="WholeStageCodegen (23)"; 951 [label="MapPartitionsRDD [951]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2977 { label="Exchange"; 952 [label="MapPartitionsRDD [952]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->948; 948->949; 949->950; 950->951; 951->952; }

56,57

digraph G { subgraph clusterstage_876 { label="Stage 876 (skipped)"; subgraph cluster3153 { label="Exchange"; 1002 [label="ShuffledRowRDD [1002]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3174 { label="WholeStageCodegen (49)"; 1011 [label="MapPartitionsRDD [1011]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3185 { label="Exchange"; 1012 [label="ShuffledRowRDD [1012]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3181 { label="WholeStageCodegen (50)"; 1013 [label="MapPartitionsRDD [1013]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3167 { label="WholeStageCodegen (51)"; 1014 [label="ZippedPartitionsRDD2 [1014]\nsave at NativeMethodAccessorImpl.java:0"]; 1015 [label="MapPartitionsRDD [1015]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3166 { label="Window"; 1016 [label="MapPartitionsRDD [1016]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3162 { label="WholeStageCodegen (52)"; 1017 [label="MapPartitionsRDD [1017]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3161 { label="Exchange"; 1018 [label="MapPartitionsRDD [1018]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1002->1011; 1012->1013; 1011->1014; 1013->1014; 1014->1015; 1015->1016; 1016->1017; 1017->1018; }

937,1002

952,1012

digraph G { subgraph clusterstage_877 { label="Stage 877 (skipped)"; subgraph cluster3514 { label="AQEShuffleRead"; 1127 [label="ShuffledRowRDD [1127]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3511 { label="WholeStageCodegen (95)"; 1128 [label="MapPartitionsRDD [1128]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3519 { label="AQEShuffleRead"; 1129 [label="ShuffledRowRDD [1129]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3516 { label="WholeStageCodegen (96)"; 1130 [label="MapPartitionsRDD [1130]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3507 { label="WholeStageCodegen (97)"; 1131 [label="ZippedPartitionsRDD2 [1131]\nsave at NativeMethodAccessorImpl.java:0"]; 1132 [label="MapPartitionsRDD [1132]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3506 { label="Exchange"; 1133 [label="MapPartitionsRDD [1133]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1127->1128; 1129->1130; 1128->1131; 1130->1131; 1131->1132; 1132->1133; }

1126,1127

1058,1129

digraph G { subgraph clusterstage_878 { label="Stage 878"; subgraph cluster3530 { label="AQEShuffleRead"; 1134 [label="ShuffledRowRDD [1134]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3527 { label="WholeStageCodegen (98)"; 1135 [label="MapPartitionsRDD [1135]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3535 { label="AQEShuffleRead"; 1136 [label="ShuffledRowRDD [1136]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3532 { label="WholeStageCodegen (99)"; 1137 [label="MapPartitionsRDD [1137]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3523 { label="WholeStageCodegen (100)"; 1138 [label="ZippedPartitionsRDD2 [1138]\nsave at NativeMethodAccessorImpl.java:0"]; 1139 [label="MapPartitionsRDD [1139]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3522 { label="Exchange"; 1140 [label="MapPartitionsRDD [1140]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1134->1135; 1136->1137; 1135->1138; 1137->1138; 1138->1139; 1139->1140; }

1133,1134

1018,1136

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
878	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:29:13	1.0 s	735/735			160.7 MB	34.9 MB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

878

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:29:13

1.0 s

735/735

160.7 MB

34.9 MB

Skipped Stages (38)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
877	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/717
876	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
875	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
874	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
873	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/872
872	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
871	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/80
870	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/90
869	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
868	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
867	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
866	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/85
865	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
864	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/97
863	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
862	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/87
861	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/845
860	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
859	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
858	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
857	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/780
856	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
855	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
854	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/757
853	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
852	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
851	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
850	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
849	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
848	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
847	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
846	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
845	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
844	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
843	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
842	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
841	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
840	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433