paxus_etl - Details for Job 114

Details for Job 114

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 24

Enable zooming

digraph G { subgraph clusterstage_305 { label="Stage 305 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_306 { label="Stage 306 (skipped)"; subgraph cluster1101 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.mediatable"; 405 [label="FileScanRDD [405]\nsave at NativeMethodAccessorImpl.java:0"]; 406 [label="MapPartitionsRDD [406]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1097 { label="WholeStageCodegen (21)"; 407 [label="MapPartitionsRDD [407]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1096 { label="Exchange"; 408 [label="MapPartitionsRDD [408]\nsave at NativeMethodAccessorImpl.java:0"]; } } 405->406; 406->407; 407->408; }

digraph G { subgraph clusterstage_307 { label="Stage 307 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1109 { label="InMemoryTableScan"; 409 [label="MapPartitionsRDD [409]\nsave at NativeMethodAccessorImpl.java:0"]; 410 [label="MapPartitionsRDD [410]\nsave at NativeMethodAccessorImpl.java:0"]; 411 [label="MapPartitionsRDD [411]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1104 { label="WholeStageCodegen (22)"; 412 [label="MapPartitionsRDD [412]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1103 { label="Exchange"; 413 [label="MapPartitionsRDD [413]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->409; 409->410; 410->411; 411->412; 412->413; }

56,57

digraph G { subgraph clusterstage_308 { label="Stage 308 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1127 { label="InMemoryTableScan"; 419 [label="MapPartitionsRDD [419]\nsave at NativeMethodAccessorImpl.java:0"]; 420 [label="MapPartitionsRDD [420]\nsave at NativeMethodAccessorImpl.java:0"]; 421 [label="MapPartitionsRDD [421]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1122 { label="WholeStageCodegen (26)"; 422 [label="MapPartitionsRDD [422]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1121 { label="Exchange"; 423 [label="MapPartitionsRDD [423]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->419; 419->420; 420->421; 421->422; 422->423; }

56,57

digraph G { subgraph clusterstage_309 { label="Stage 309 (skipped)"; subgraph cluster1017 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.audiodescriptiontable"; 354 [label="FileScanRDD [354]\nsave at NativeMethodAccessorImpl.java:0"]; 355 [label="MapPartitionsRDD [355]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1013 { label="WholeStageCodegen (8)"; 356 [label="MapPartitionsRDD [356]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1012 { label="Exchange"; 357 [label="MapPartitionsRDD [357]\nsave at NativeMethodAccessorImpl.java:0"]; } } 354->355; 355->356; 356->357; }

digraph G { subgraph clusterstage_310 { label="Stage 310 (skipped)"; subgraph cluster1068 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.videodescriptiontable"; 386 [label="FileScanRDD [386]\nsave at NativeMethodAccessorImpl.java:0"]; 387 [label="MapPartitionsRDD [387]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1064 { label="WholeStageCodegen (15)"; 388 [label="MapPartitionsRDD [388]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1063 { label="Exchange"; 389 [label="MapPartitionsRDD [389]\nsave at NativeMethodAccessorImpl.java:0"]; } } 386->387; 387->388; 388->389; }

digraph G { subgraph clusterstage_311 { label="Stage 311 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1118 { label="InMemoryTableScan"; 414 [label="MapPartitionsRDD [414]\nsave at NativeMethodAccessorImpl.java:0"]; 415 [label="MapPartitionsRDD [415]\nsave at NativeMethodAccessorImpl.java:0"]; 416 [label="MapPartitionsRDD [416]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1113 { label="WholeStageCodegen (24)"; 417 [label="MapPartitionsRDD [417]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1112 { label="Exchange"; 418 [label="MapPartitionsRDD [418]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->414; 414->415; 415->416; 416->417; 417->418; }

56,57

digraph G { subgraph clusterstage_312 { label="Stage 312 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1076 { label="InMemoryTableScan"; 390 [label="MapPartitionsRDD [390]\nsave at NativeMethodAccessorImpl.java:0"]; 391 [label="MapPartitionsRDD [391]\nsave at NativeMethodAccessorImpl.java:0"]; 392 [label="MapPartitionsRDD [392]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1071 { label="WholeStageCodegen (16)"; 393 [label="MapPartitionsRDD [393]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1070 { label="Exchange"; 394 [label="MapPartitionsRDD [394]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->390; 390->391; 391->392; 392->393; 393->394; }

56,57

digraph G { subgraph clusterstage_313 { label="Stage 313 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1085 { label="InMemoryTableScan"; 395 [label="MapPartitionsRDD [395]\nsave at NativeMethodAccessorImpl.java:0"]; 396 [label="MapPartitionsRDD [396]\nsave at NativeMethodAccessorImpl.java:0"]; 397 [label="MapPartitionsRDD [397]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1080 { label="WholeStageCodegen (18)"; 398 [label="MapPartitionsRDD [398]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1079 { label="Exchange"; 399 [label="MapPartitionsRDD [399]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->395; 395->396; 396->397; 397->398; 398->399; }

56,57

digraph G { subgraph clusterstage_314 { label="Stage 314 (skipped)"; subgraph cluster1032 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.aggregatetable"; 363 [label="FileScanRDD [363]\nsave at NativeMethodAccessorImpl.java:0"]; 364 [label="MapPartitionsRDD [364]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1028 { label="WholeStageCodegen (10)"; 365 [label="MapPartitionsRDD [365]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1027 { label="Exchange"; 366 [label="MapPartitionsRDD [366]\nsave at NativeMethodAccessorImpl.java:0"]; } } 363->364; 364->365; 365->366; }

digraph G { subgraph clusterstage_315 { label="Stage 315 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1094 { label="InMemoryTableScan"; 400 [label="MapPartitionsRDD [400]\nsave at NativeMethodAccessorImpl.java:0"]; 401 [label="MapPartitionsRDD [401]\nsave at NativeMethodAccessorImpl.java:0"]; 402 [label="MapPartitionsRDD [402]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1089 { label="WholeStageCodegen (20)"; 403 [label="MapPartitionsRDD [403]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1088 { label="Exchange"; 404 [label="MapPartitionsRDD [404]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->400; 400->401; 401->402; 402->403; 403->404; }

56,57

digraph G { subgraph clusterstage_316 { label="Stage 316 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1040 { label="InMemoryTableScan"; 367 [label="MapPartitionsRDD [367]\nsave at NativeMethodAccessorImpl.java:0"]; 368 [label="MapPartitionsRDD [368]\nsave at NativeMethodAccessorImpl.java:0"]; 369 [label="MapPartitionsRDD [369]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1035 { label="WholeStageCodegen (11)"; 370 [label="MapPartitionsRDD [370]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1034 { label="Exchange"; 371 [label="MapPartitionsRDD [371]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->367; 367->368; 368->369; 369->370; 370->371; }

56,57

digraph G { subgraph clusterstage_317 { label="Stage 317 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1046 { label="InMemoryTableScan"; 372 [label="MapPartitionsRDD [372]\nsave at NativeMethodAccessorImpl.java:0"]; 373 [label="MapPartitionsRDD [373]\nsave at NativeMethodAccessorImpl.java:0"]; 374 [label="MapPartitionsRDD [374]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1043 { label="WholeStageCodegen (12)"; 375 [label="MapPartitionsRDD [375]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1042 { label="Exchange"; 376 [label="MapPartitionsRDD [376]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->372; 372->373; 373->374; 374->375; 375->376; }

56,57

digraph G { subgraph clusterstage_318 { label="Stage 318 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1061 { label="InMemoryTableScan"; 381 [label="MapPartitionsRDD [381]\nsave at NativeMethodAccessorImpl.java:0"]; 382 [label="MapPartitionsRDD [382]\nsave at NativeMethodAccessorImpl.java:0"]; 383 [label="MapPartitionsRDD [383]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1056 { label="WholeStageCodegen (14)"; 384 [label="MapPartitionsRDD [384]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1055 { label="Exchange"; 385 [label="MapPartitionsRDD [385]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->381; 381->382; 382->383; 383->384; 384->385; }

56,57

digraph G { subgraph clusterstage_319 { label="Stage 319 (skipped)"; subgraph cluster1053 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.mediatable"; 377 [label="FileScanRDD [377]\nsave at NativeMethodAccessorImpl.java:0"]; 378 [label="MapPartitionsRDD [378]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1049 { label="WholeStageCodegen (13)"; 379 [label="MapPartitionsRDD [379]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1048 { label="Exchange"; 380 [label="MapPartitionsRDD [380]\nsave at NativeMethodAccessorImpl.java:0"]; } } 377->378; 378->379; 379->380; }

digraph G { subgraph clusterstage_320 { label="Stage 320 (skipped)"; subgraph cluster1477 { label="Exchange"; 553 [label="ShuffledRowRDD [553]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1480 { label="Exchange"; 554 [label="ShuffledRowRDD [554]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1469 { label="WholeStageCodegen (69)"; 555 [label="ZippedPartitionsRDD2 [555]\nsave at NativeMethodAccessorImpl.java:0"]; 556 [label="MapPartitionsRDD [556]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1468 { label="Exchange"; 557 [label="MapPartitionsRDD [557]\nsave at NativeMethodAccessorImpl.java:0"]; } } 553->555; 554->555; 555->556; 556->557; }

380,553

385,554

digraph G { subgraph clusterstage_321 { label="Stage 321 (skipped)"; subgraph cluster1354 { label="Exchange"; 512 [label="ShuffledRowRDD [512]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1382 { label="Exchange"; 519 [label="ShuffledRowRDD [519]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1370 { label="WholeStageCodegen (55)"; 520 [label="ZippedPartitionsRDD2 [520]\nsave at NativeMethodAccessorImpl.java:0"]; 521 [label="MapPartitionsRDD [521]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1369 { label="Window"; 522 [label="MapPartitionsRDD [522]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1364 { label="WholeStageCodegen (56)"; 523 [label="MapPartitionsRDD [523]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1388 { label="Exchange"; 524 [label="ShuffledRowRDD [524]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1384 { label="WholeStageCodegen (57)"; 525 [label="MapPartitionsRDD [525]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1360 { label="WholeStageCodegen (58)"; 526 [label="ZippedPartitionsRDD2 [526]\nsave at NativeMethodAccessorImpl.java:0"]; 527 [label="MapPartitionsRDD [527]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1359 { label="Exchange"; 528 [label="MapPartitionsRDD [528]\nsave at NativeMethodAccessorImpl.java:0"]; } } 512->520; 519->520; 520->521; 521->522; 522->523; 524->525; 523->526; 525->526; 526->527; 527->528; }

366,512

371,519

376,524

digraph G { subgraph clusterstage_322 { label="Stage 322 (skipped)"; subgraph cluster1354 { label="Exchange"; 512 [label="ShuffledRowRDD [512]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1357 { label="Exchange"; 513 [label="ShuffledRowRDD [513]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1344 { label="WholeStageCodegen (53)"; 514 [label="ZippedPartitionsRDD2 [514]\nsave at NativeMethodAccessorImpl.java:0"]; 515 [label="MapPartitionsRDD [515]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1343 { label="Window"; 516 [label="MapPartitionsRDD [516]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1339 { label="WholeStageCodegen (54)"; 517 [label="MapPartitionsRDD [517]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1338 { label="Exchange"; 518 [label="MapPartitionsRDD [518]\nsave at NativeMethodAccessorImpl.java:0"]; } } 512->514; 513->514; 514->515; 515->516; 516->517; 517->518; }

366,512

404,513

digraph G { subgraph clusterstage_323 { label="Stage 323 (skipped)"; subgraph cluster1307 { label="Exchange"; 499 [label="ShuffledRowRDD [499]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1332 { label="Exchange"; 506 [label="ShuffledRowRDD [506]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1319 { label="WholeStageCodegen (51)"; 507 [label="ZippedPartitionsRDD2 [507]\nsave at NativeMethodAccessorImpl.java:0"]; 508 [label="MapPartitionsRDD [508]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1318 { label="Window"; 509 [label="MapPartitionsRDD [509]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1314 { label="WholeStageCodegen (52)"; 510 [label="MapPartitionsRDD [510]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1313 { label="Exchange"; 511 [label="MapPartitionsRDD [511]\nsave at NativeMethodAccessorImpl.java:0"]; } } 499->507; 506->507; 507->508; 508->509; 509->510; 510->511; }

357,499

399,506

digraph G { subgraph clusterstage_324 { label="Stage 324 (skipped)"; subgraph cluster1632 { label="Exchange"; 602 [label="ShuffledRowRDD [602]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1655 { label="Exchange"; 609 [label="ShuffledRowRDD [609]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1643 { label="WholeStageCodegen (89)"; 610 [label="ZippedPartitionsRDD2 [610]\nsave at NativeMethodAccessorImpl.java:0"]; 611 [label="MapPartitionsRDD [611]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1642 { label="Window"; 612 [label="MapPartitionsRDD [612]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1638 { label="WholeStageCodegen (90)"; 613 [label="MapPartitionsRDD [613]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1637 { label="Exchange"; 614 [label="MapPartitionsRDD [614]\nsave at NativeMethodAccessorImpl.java:0"]; } } 602->610; 609->610; 610->611; 611->612; 612->613; 613->614; }

389,602

394,609

digraph G { subgraph clusterstage_325 { label="Stage 325 (skipped)"; subgraph cluster1704 { label="Exchange"; 621 [label="ShuffledRowRDD [621]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1700 { label="WholeStageCodegen (93)"; 622 [label="MapPartitionsRDD [622]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1710 { label="Exchange"; 623 [label="ShuffledRowRDD [623]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1706 { label="WholeStageCodegen (94)"; 624 [label="MapPartitionsRDD [624]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1696 { label="WholeStageCodegen (95)"; 625 [label="ZippedPartitionsRDD2 [625]\nsave at NativeMethodAccessorImpl.java:0"]; 626 [label="MapPartitionsRDD [626]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1716 { label="Exchange"; 627 [label="ShuffledRowRDD [627]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1712 { label="WholeStageCodegen (96)"; 628 [label="MapPartitionsRDD [628]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1692 { label="WholeStageCodegen (97)"; 629 [label="ZippedPartitionsRDD2 [629]\nsave at NativeMethodAccessorImpl.java:0"]; 630 [label="MapPartitionsRDD [630]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1722 { label="Exchange"; 631 [label="ShuffledRowRDD [631]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1718 { label="WholeStageCodegen (98)"; 632 [label="MapPartitionsRDD [632]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1688 { label="WholeStageCodegen (99)"; 633 [label="ZippedPartitionsRDD2 [633]\nsave at NativeMethodAccessorImpl.java:0"]; 634 [label="MapPartitionsRDD [634]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1728 { label="Exchange"; 635 [label="ShuffledRowRDD [635]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1724 { label="WholeStageCodegen (100)"; 636 [label="MapPartitionsRDD [636]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1684 { label="WholeStageCodegen (101)"; 637 [label="ZippedPartitionsRDD2 [637]\nsave at NativeMethodAccessorImpl.java:0"]; 638 [label="MapPartitionsRDD [638]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1683 { label="Exchange"; 639 [label="MapPartitionsRDD [639]\nsave at NativeMethodAccessorImpl.java:0"]; } } 621->622; 623->624; 622->625; 624->625; 625->626; 627->628; 626->629; 628->629; 629->630; 631->632; 630->633; 632->633; 633->634; 635->636; 634->637; 636->637; 637->638; 638->639; }

528,621

557,623

614,627

511,631

518,635

digraph G { subgraph clusterstage_326 { label="Stage 326 (skipped)"; subgraph cluster1632 { label="Exchange"; 602 [label="ShuffledRowRDD [602]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1635 { label="Exchange"; 603 [label="ShuffledRowRDD [603]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1622 { label="WholeStageCodegen (87)"; 604 [label="ZippedPartitionsRDD2 [604]\nsave at NativeMethodAccessorImpl.java:0"]; 605 [label="MapPartitionsRDD [605]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1621 { label="Window"; 606 [label="MapPartitionsRDD [606]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1617 { label="WholeStageCodegen (88)"; 607 [label="MapPartitionsRDD [607]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1616 { label="Exchange"; 608 [label="MapPartitionsRDD [608]\nsave at NativeMethodAccessorImpl.java:0"]; } } 602->604; 603->604; 604->605; 605->606; 606->607; 607->608; }

389,602

418,603

digraph G { subgraph clusterstage_327 { label="Stage 327 (skipped)"; subgraph cluster1307 { label="Exchange"; 499 [label="ShuffledRowRDD [499]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1411 { label="Exchange"; 529 [label="ShuffledRowRDD [529]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1398 { label="WholeStageCodegen (59)"; 530 [label="ZippedPartitionsRDD2 [530]\nsave at NativeMethodAccessorImpl.java:0"]; 531 [label="MapPartitionsRDD [531]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1397 { label="Window"; 532 [label="MapPartitionsRDD [532]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1393 { label="WholeStageCodegen (60)"; 533 [label="MapPartitionsRDD [533]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1392 { label="Exchange"; 534 [label="MapPartitionsRDD [534]\nsave at NativeMethodAccessorImpl.java:0"]; } } 499->530; 529->530; 530->531; 531->532; 532->533; 533->534; }

357,499

423,529

digraph G { subgraph clusterstage_328 { label="Stage 328 (skipped)"; subgraph cluster1566 { label="Exchange"; 583 [label="ShuffledRowRDD [583]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1569 { label="Exchange"; 584 [label="ShuffledRowRDD [584]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1558 { label="WholeStageCodegen (81)"; 585 [label="ZippedPartitionsRDD2 [585]\nsave at NativeMethodAccessorImpl.java:0"]; 586 [label="MapPartitionsRDD [586]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1557 { label="Exchange"; 587 [label="MapPartitionsRDD [587]\nsave at NativeMethodAccessorImpl.java:0"]; } } 583->585; 584->585; 585->586; 586->587; }

408,583

413,584

digraph G { subgraph clusterstage_329 { label="Stage 329"; subgraph cluster1748 { label="AQEShuffleRead"; 640 [label="ShuffledRowRDD [640]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1745 { label="WholeStageCodegen (102)"; 641 [label="MapPartitionsRDD [641]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1753 { label="AQEShuffleRead"; 642 [label="ShuffledRowRDD [642]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1750 { label="WholeStageCodegen (103)"; 643 [label="MapPartitionsRDD [643]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1741 { label="WholeStageCodegen (104)"; 644 [label="ZippedPartitionsRDD2 [644]\nsave at NativeMethodAccessorImpl.java:0"]; 645 [label="MapPartitionsRDD [645]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1758 { label="AQEShuffleRead"; 646 [label="ShuffledRowRDD [646]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1755 { label="WholeStageCodegen (105)"; 647 [label="MapPartitionsRDD [647]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1737 { label="WholeStageCodegen (106)"; 648 [label="ZippedPartitionsRDD2 [648]\nsave at NativeMethodAccessorImpl.java:0"]; 649 [label="MapPartitionsRDD [649]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1763 { label="AQEShuffleRead"; 650 [label="ShuffledRowRDD [650]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1760 { label="WholeStageCodegen (107)"; 651 [label="MapPartitionsRDD [651]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1732 { label="WholeStageCodegen (108)"; 652 [label="ZippedPartitionsRDD2 [652]\nsave at NativeMethodAccessorImpl.java:0"]; 653 [label="MapPartitionsRDD [653]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1731 { label="Exchange"; 654 [label="MapPartitionsRDD [654]\nsave at NativeMethodAccessorImpl.java:0"]; } } 640->641; 642->643; 641->644; 643->644; 644->645; 646->647; 645->648; 647->648; 648->649; 650->651; 649->652; 651->652; 652->653; 653->654; }

639,640

587,642

608,646

534,650

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
329	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:16:37	4.0 min	501/501			6.9 GB	3.9 GB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

329

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:16:37

4.0 min

501/501

6.9 GB

3.9 GB

Skipped Stages (24)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
328	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
327	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
326	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
325	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
324	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
323	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
322	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
321	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
320	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
319	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/639
318	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
317	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
316	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
315	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
314	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/649
313	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
312	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
311	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
310	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/642
309	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/651
308	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
307	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
306	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/639
305	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433