paxus_etl - Details for Job 116

Details for Job 116

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 57

Enable zooming

digraph G { subgraph clusterstage_356 { label="Stage 356 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_357 { label="Stage 357 (skipped)"; subgraph cluster1134 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.webcontentdescriptiontable"; 424 [label="FileScanRDD [424]\nsave at NativeMethodAccessorImpl.java:0"]; 425 [label="MapPartitionsRDD [425]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1130 { label="WholeStageCodegen (27)"; 426 [label="MapPartitionsRDD [426]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1129 { label="Exchange"; 427 [label="MapPartitionsRDD [427]\nsave at NativeMethodAccessorImpl.java:0"]; } } 424->425; 425->426; 426->427; }

digraph G { subgraph clusterstage_358 { label="Stage 358 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1142 { label="InMemoryTableScan"; 428 [label="MapPartitionsRDD [428]\nsave at NativeMethodAccessorImpl.java:0"]; 429 [label="MapPartitionsRDD [429]\nsave at NativeMethodAccessorImpl.java:0"]; 430 [label="MapPartitionsRDD [430]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1137 { label="WholeStageCodegen (28)"; 431 [label="MapPartitionsRDD [431]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1136 { label="Exchange"; 432 [label="MapPartitionsRDD [432]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->428; 428->429; 429->430; 430->431; 431->432; }

56,57

digraph G { subgraph clusterstage_359 { label="Stage 359 (skipped)"; subgraph cluster1017 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.audiodescriptiontable"; 354 [label="FileScanRDD [354]\nsave at NativeMethodAccessorImpl.java:0"]; 355 [label="MapPartitionsRDD [355]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1013 { label="WholeStageCodegen (8)"; 356 [label="MapPartitionsRDD [356]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1012 { label="Exchange"; 357 [label="MapPartitionsRDD [357]\nsave at NativeMethodAccessorImpl.java:0"]; } } 354->355; 355->356; 356->357; }

digraph G { subgraph clusterstage_360 { label="Stage 360 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1025 { label="InMemoryTableScan"; 358 [label="MapPartitionsRDD [358]\nsave at NativeMethodAccessorImpl.java:0"]; 359 [label="MapPartitionsRDD [359]\nsave at NativeMethodAccessorImpl.java:0"]; 360 [label="MapPartitionsRDD [360]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1020 { label="WholeStageCodegen (9)"; 361 [label="MapPartitionsRDD [361]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1019 { label="Exchange"; 362 [label="MapPartitionsRDD [362]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->358; 358->359; 359->360; 360->361; 361->362; }

56,57

digraph G { subgraph clusterstage_361 { label="Stage 361 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1211 { label="InMemoryTableScan"; 469 [label="MapPartitionsRDD [469]\nsave at NativeMethodAccessorImpl.java:0"]; 470 [label="MapPartitionsRDD [470]\nsave at NativeMethodAccessorImpl.java:0"]; 471 [label="MapPartitionsRDD [471]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1206 { label="WholeStageCodegen (38)"; 472 [label="MapPartitionsRDD [472]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1205 { label="Exchange"; 473 [label="MapPartitionsRDD [473]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->469; 469->470; 470->471; 471->472; 472->473; }

56,57

digraph G { subgraph clusterstage_362 { label="Stage 362 (skipped)"; subgraph cluster1203 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.videodescriptiontable"; 465 [label="FileScanRDD [465]\nsave at NativeMethodAccessorImpl.java:0"]; 466 [label="MapPartitionsRDD [466]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1199 { label="WholeStageCodegen (37)"; 467 [label="MapPartitionsRDD [467]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1198 { label="Exchange"; 468 [label="MapPartitionsRDD [468]\nsave at NativeMethodAccessorImpl.java:0"]; } } 465->466; 466->467; 467->468; }

digraph G { subgraph clusterstage_363 { label="Stage 363 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1010 { label="InMemoryTableScan"; 349 [label="MapPartitionsRDD [349]\nsave at NativeMethodAccessorImpl.java:0"]; 350 [label="MapPartitionsRDD [350]\nsave at NativeMethodAccessorImpl.java:0"]; 351 [label="MapPartitionsRDD [351]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1005 { label="WholeStageCodegen (7)"; 352 [label="MapPartitionsRDD [352]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1004 { label="Exchange"; 353 [label="MapPartitionsRDD [353]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->349; 349->350; 350->351; 351->352; 352->353; }

56,57

digraph G { subgraph clusterstage_364 { label="Stage 364 (skipped)"; subgraph cluster1002 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.ratingtable"; 345 [label="FileScanRDD [345]\nsave at NativeMethodAccessorImpl.java:0"]; 346 [label="MapPartitionsRDD [346]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster998 { label="WholeStageCodegen (6)"; 347 [label="MapPartitionsRDD [347]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster997 { label="Exchange"; 348 [label="MapPartitionsRDD [348]\nsave at NativeMethodAccessorImpl.java:0"]; } } 345->346; 346->347; 347->348; }

digraph G { subgraph clusterstage_365 { label="Stage 365 (skipped)"; subgraph cluster987 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.versiontable"; 336 [label="FileScanRDD [336]\nsave at NativeMethodAccessorImpl.java:0"]; 337 [label="MapPartitionsRDD [337]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster983 { label="WholeStageCodegen (4)"; 338 [label="MapPartitionsRDD [338]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster982 { label="Exchange"; 339 [label="MapPartitionsRDD [339]\nsave at NativeMethodAccessorImpl.java:0"]; } } 336->337; 337->338; 338->339; }

digraph G { subgraph clusterstage_366 { label="Stage 366 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster995 { label="InMemoryTableScan"; 340 [label="MapPartitionsRDD [340]\nsave at NativeMethodAccessorImpl.java:0"]; 341 [label="MapPartitionsRDD [341]\nsave at NativeMethodAccessorImpl.java:0"]; 342 [label="MapPartitionsRDD [342]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster990 { label="WholeStageCodegen (5)"; 343 [label="MapPartitionsRDD [343]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster989 { label="Exchange"; 344 [label="MapPartitionsRDD [344]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->340; 340->341; 341->342; 342->343; 343->344; }

56,57

digraph G { subgraph clusterstage_367 { label="Stage 367 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster974 { label="InMemoryTableScan"; 326 [label="MapPartitionsRDD [326]\nsave at NativeMethodAccessorImpl.java:0"]; 327 [label="MapPartitionsRDD [327]\nsave at NativeMethodAccessorImpl.java:0"]; 328 [label="MapPartitionsRDD [328]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster969 { label="WholeStageCodegen (2)"; 329 [label="MapPartitionsRDD [329]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster968 { label="Exchange"; 330 [label="MapPartitionsRDD [330]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->326; 326->327; 327->328; 328->329; 329->330; }

56,57

digraph G { subgraph clusterstage_368 { label="Stage 368 (skipped)"; subgraph cluster966 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.mediatable"; 322 [label="FileScanRDD [322]\nsave at NativeMethodAccessorImpl.java:0"]; 323 [label="MapPartitionsRDD [323]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster962 { label="WholeStageCodegen (1)"; 324 [label="MapPartitionsRDD [324]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster961 { label="Exchange"; 325 [label="MapPartitionsRDD [325]\nsave at NativeMethodAccessorImpl.java:0"]; } } 322->323; 323->324; 324->325; }

digraph G { subgraph clusterstage_369 { label="Stage 369 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster980 { label="InMemoryTableScan"; 331 [label="MapPartitionsRDD [331]\nsave at NativeMethodAccessorImpl.java:0"]; 332 [label="MapPartitionsRDD [332]\nsave at NativeMethodAccessorImpl.java:0"]; 333 [label="MapPartitionsRDD [333]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster977 { label="WholeStageCodegen (3)"; 334 [label="MapPartitionsRDD [334]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster976 { label="Exchange"; 335 [label="MapPartitionsRDD [335]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->331; 331->332; 332->333; 333->334; 334->335; }

56,57

digraph G { subgraph clusterstage_370 { label="Stage 370 (skipped)"; subgraph cluster1278 { label="Exchange"; 490 [label="ShuffledRowRDD [490]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1281 { label="Exchange"; 491 [label="ShuffledRowRDD [491]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1269 { label="WholeStageCodegen (46)"; 492 [label="ZippedPartitionsRDD2 [492]\nsave at NativeMethodAccessorImpl.java:0"]; 493 [label="MapPartitionsRDD [493]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1287 { label="Exchange"; 494 [label="ShuffledRowRDD [494]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1283 { label="WholeStageCodegen (47)"; 495 [label="MapPartitionsRDD [495]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1265 { label="WholeStageCodegen (48)"; 496 [label="ZippedPartitionsRDD2 [496]\nsave at NativeMethodAccessorImpl.java:0"]; 497 [label="MapPartitionsRDD [497]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1264 { label="Exchange"; 498 [label="MapPartitionsRDD [498]\nsave at NativeMethodAccessorImpl.java:0"]; } } 490->492; 491->492; 492->493; 494->495; 493->496; 495->496; 496->497; 497->498; }

325,490

330,491

335,494

digraph G { subgraph clusterstage_371 { label="Stage 371 (skipped)"; subgraph cluster1227 { label="AQEShuffleRead"; 474 [label="ShuffledRowRDD [474]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1224 { label="WholeStageCodegen (39)"; 475 [label="MapPartitionsRDD [475]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1232 { label="AQEShuffleRead"; 476 [label="ShuffledRowRDD [476]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1229 { label="WholeStageCodegen (40)"; 477 [label="MapPartitionsRDD [477]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1218 { label="WholeStageCodegen (41)"; 478 [label="ZippedPartitionsRDD2 [478]\nsave at NativeMethodAccessorImpl.java:0"]; 479 [label="MapPartitionsRDD [479]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1217 { label="Exchange"; 480 [label="MapPartitionsRDD [480]\nsave at NativeMethodAccessorImpl.java:0"]; } } 474->475; 476->477; 475->478; 477->478; 478->479; 479->480; }

339,474

344,476

digraph G { subgraph clusterstage_372 { label="Stage 372 (skipped)"; subgraph cluster1544 { label="AQEShuffleRead"; 576 [label="ShuffledRowRDD [576]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1541 { label="WholeStageCodegen (78)"; 577 [label="MapPartitionsRDD [577]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1549 { label="AQEShuffleRead"; 578 [label="ShuffledRowRDD [578]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1546 { label="WholeStageCodegen (79)"; 579 [label="MapPartitionsRDD [579]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1537 { label="WholeStageCodegen (80)"; 580 [label="ZippedPartitionsRDD2 [580]\nsave at NativeMethodAccessorImpl.java:0"]; 581 [label="MapPartitionsRDD [581]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1536 { label="Exchange"; 582 [label="MapPartitionsRDD [582]\nsave at NativeMethodAccessorImpl.java:0"]; } } 576->577; 578->579; 577->580; 579->580; 580->581; 581->582; }

498,576

480,578

digraph G { subgraph clusterstage_373 { label="Stage 373 (skipped)"; subgraph cluster1252 { label="AQEShuffleRead"; 481 [label="ShuffledRowRDD [481]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1249 { label="WholeStageCodegen (42)"; 482 [label="MapPartitionsRDD [482]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1257 { label="AQEShuffleRead"; 483 [label="ShuffledRowRDD [483]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1254 { label="WholeStageCodegen (43)"; 484 [label="MapPartitionsRDD [484]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1242 { label="WholeStageCodegen (44)"; 485 [label="ZippedPartitionsRDD2 [485]\nsave at NativeMethodAccessorImpl.java:0"]; 486 [label="MapPartitionsRDD [486]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1241 { label="Window"; 487 [label="MapPartitionsRDD [487]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1237 { label="WholeStageCodegen (45)"; 488 [label="MapPartitionsRDD [488]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1236 { label="Exchange"; 489 [label="MapPartitionsRDD [489]\nsave at NativeMethodAccessorImpl.java:0"]; } } 481->482; 483->484; 482->485; 484->485; 485->486; 486->487; 487->488; 488->489; }

348,481

353,483

digraph G { subgraph clusterstage_374 { label="Stage 374 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1172 { label="InMemoryTableScan"; 446 [label="MapPartitionsRDD [446]\nsave at NativeMethodAccessorImpl.java:0"]; 447 [label="MapPartitionsRDD [447]\nsave at NativeMethodAccessorImpl.java:0"]; 448 [label="MapPartitionsRDD [448]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1167 { label="WholeStageCodegen (32)"; 449 [label="MapPartitionsRDD [449]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1166 { label="Exchange"; 450 [label="MapPartitionsRDD [450]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->446; 446->447; 447->448; 448->449; 449->450; }

56,57

digraph G { subgraph clusterstage_375 { label="Stage 375 (skipped)"; subgraph cluster1164 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.appdescriptiontable"; 442 [label="FileScanRDD [442]\nsave at NativeMethodAccessorImpl.java:0"]; 443 [label="MapPartitionsRDD [443]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1160 { label="WholeStageCodegen (31)"; 444 [label="MapPartitionsRDD [444]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1159 { label="Exchange"; 445 [label="MapPartitionsRDD [445]\nsave at NativeMethodAccessorImpl.java:0"]; } } 442->443; 443->444; 444->445; }

digraph G { subgraph clusterstage_376 { label="Stage 376 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1157 { label="InMemoryTableScan"; 437 [label="MapPartitionsRDD [437]\nsave at NativeMethodAccessorImpl.java:0"]; 438 [label="MapPartitionsRDD [438]\nsave at NativeMethodAccessorImpl.java:0"]; 439 [label="MapPartitionsRDD [439]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1152 { label="WholeStageCodegen (30)"; 440 [label="MapPartitionsRDD [440]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1151 { label="Exchange"; 441 [label="MapPartitionsRDD [441]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->437; 437->438; 438->439; 439->440; 440->441; }

56,57

digraph G { subgraph clusterstage_377 { label="Stage 377 (skipped)"; subgraph cluster1149 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.gamedescriptiontable"; 433 [label="FileScanRDD [433]\nsave at NativeMethodAccessorImpl.java:0"]; 434 [label="MapPartitionsRDD [434]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1145 { label="WholeStageCodegen (29)"; 435 [label="MapPartitionsRDD [435]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1144 { label="Exchange"; 436 [label="MapPartitionsRDD [436]\nsave at NativeMethodAccessorImpl.java:0"]; } } 433->434; 434->435; 435->436; }

digraph G { subgraph clusterstage_378 { label="Stage 378 (skipped)"; subgraph cluster1101 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.mediatable"; 405 [label="FileScanRDD [405]\nsave at NativeMethodAccessorImpl.java:0"]; 406 [label="MapPartitionsRDD [406]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1097 { label="WholeStageCodegen (21)"; 407 [label="MapPartitionsRDD [407]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1096 { label="Exchange"; 408 [label="MapPartitionsRDD [408]\nsave at NativeMethodAccessorImpl.java:0"]; } } 405->406; 406->407; 407->408; }

digraph G { subgraph clusterstage_379 { label="Stage 379 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1109 { label="InMemoryTableScan"; 409 [label="MapPartitionsRDD [409]\nsave at NativeMethodAccessorImpl.java:0"]; 410 [label="MapPartitionsRDD [410]\nsave at NativeMethodAccessorImpl.java:0"]; 411 [label="MapPartitionsRDD [411]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1104 { label="WholeStageCodegen (22)"; 412 [label="MapPartitionsRDD [412]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1103 { label="Exchange"; 413 [label="MapPartitionsRDD [413]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->409; 409->410; 410->411; 411->412; 412->413; }

56,57

digraph G { subgraph clusterstage_380 { label="Stage 380 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1127 { label="InMemoryTableScan"; 419 [label="MapPartitionsRDD [419]\nsave at NativeMethodAccessorImpl.java:0"]; 420 [label="MapPartitionsRDD [420]\nsave at NativeMethodAccessorImpl.java:0"]; 421 [label="MapPartitionsRDD [421]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1122 { label="WholeStageCodegen (26)"; 422 [label="MapPartitionsRDD [422]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1121 { label="Exchange"; 423 [label="MapPartitionsRDD [423]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->419; 419->420; 420->421; 421->422; 422->423; }

56,57

digraph G { subgraph clusterstage_381 { label="Stage 381 (skipped)"; subgraph cluster1068 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.videodescriptiontable"; 386 [label="FileScanRDD [386]\nsave at NativeMethodAccessorImpl.java:0"]; 387 [label="MapPartitionsRDD [387]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1064 { label="WholeStageCodegen (15)"; 388 [label="MapPartitionsRDD [388]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1063 { label="Exchange"; 389 [label="MapPartitionsRDD [389]\nsave at NativeMethodAccessorImpl.java:0"]; } } 386->387; 387->388; 388->389; }

digraph G { subgraph clusterstage_382 { label="Stage 382 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1118 { label="InMemoryTableScan"; 414 [label="MapPartitionsRDD [414]\nsave at NativeMethodAccessorImpl.java:0"]; 415 [label="MapPartitionsRDD [415]\nsave at NativeMethodAccessorImpl.java:0"]; 416 [label="MapPartitionsRDD [416]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1113 { label="WholeStageCodegen (24)"; 417 [label="MapPartitionsRDD [417]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1112 { label="Exchange"; 418 [label="MapPartitionsRDD [418]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->414; 414->415; 415->416; 416->417; 417->418; }

56,57

digraph G { subgraph clusterstage_383 { label="Stage 383 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1076 { label="InMemoryTableScan"; 390 [label="MapPartitionsRDD [390]\nsave at NativeMethodAccessorImpl.java:0"]; 391 [label="MapPartitionsRDD [391]\nsave at NativeMethodAccessorImpl.java:0"]; 392 [label="MapPartitionsRDD [392]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1071 { label="WholeStageCodegen (16)"; 393 [label="MapPartitionsRDD [393]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1070 { label="Exchange"; 394 [label="MapPartitionsRDD [394]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->390; 390->391; 391->392; 392->393; 393->394; }

56,57

digraph G { subgraph clusterstage_384 { label="Stage 384 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1085 { label="InMemoryTableScan"; 395 [label="MapPartitionsRDD [395]\nsave at NativeMethodAccessorImpl.java:0"]; 396 [label="MapPartitionsRDD [396]\nsave at NativeMethodAccessorImpl.java:0"]; 397 [label="MapPartitionsRDD [397]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1080 { label="WholeStageCodegen (18)"; 398 [label="MapPartitionsRDD [398]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1079 { label="Exchange"; 399 [label="MapPartitionsRDD [399]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->395; 395->396; 396->397; 397->398; 398->399; }

56,57

digraph G { subgraph clusterstage_385 { label="Stage 385 (skipped)"; subgraph cluster1032 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.aggregatetable"; 363 [label="FileScanRDD [363]\nsave at NativeMethodAccessorImpl.java:0"]; 364 [label="MapPartitionsRDD [364]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1028 { label="WholeStageCodegen (10)"; 365 [label="MapPartitionsRDD [365]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1027 { label="Exchange"; 366 [label="MapPartitionsRDD [366]\nsave at NativeMethodAccessorImpl.java:0"]; } } 363->364; 364->365; 365->366; }

digraph G { subgraph clusterstage_386 { label="Stage 386 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1094 { label="InMemoryTableScan"; 400 [label="MapPartitionsRDD [400]\nsave at NativeMethodAccessorImpl.java:0"]; 401 [label="MapPartitionsRDD [401]\nsave at NativeMethodAccessorImpl.java:0"]; 402 [label="MapPartitionsRDD [402]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1089 { label="WholeStageCodegen (20)"; 403 [label="MapPartitionsRDD [403]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1088 { label="Exchange"; 404 [label="MapPartitionsRDD [404]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->400; 400->401; 401->402; 402->403; 403->404; }

56,57

digraph G { subgraph clusterstage_387 { label="Stage 387 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1040 { label="InMemoryTableScan"; 367 [label="MapPartitionsRDD [367]\nsave at NativeMethodAccessorImpl.java:0"]; 368 [label="MapPartitionsRDD [368]\nsave at NativeMethodAccessorImpl.java:0"]; 369 [label="MapPartitionsRDD [369]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1035 { label="WholeStageCodegen (11)"; 370 [label="MapPartitionsRDD [370]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1034 { label="Exchange"; 371 [label="MapPartitionsRDD [371]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->367; 367->368; 368->369; 369->370; 370->371; }

56,57

digraph G { subgraph clusterstage_388 { label="Stage 388 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1046 { label="InMemoryTableScan"; 372 [label="MapPartitionsRDD [372]\nsave at NativeMethodAccessorImpl.java:0"]; 373 [label="MapPartitionsRDD [373]\nsave at NativeMethodAccessorImpl.java:0"]; 374 [label="MapPartitionsRDD [374]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1043 { label="WholeStageCodegen (12)"; 375 [label="MapPartitionsRDD [375]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1042 { label="Exchange"; 376 [label="MapPartitionsRDD [376]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->372; 372->373; 373->374; 374->375; 375->376; }

56,57

digraph G { subgraph clusterstage_389 { label="Stage 389 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1061 { label="InMemoryTableScan"; 381 [label="MapPartitionsRDD [381]\nsave at NativeMethodAccessorImpl.java:0"]; 382 [label="MapPartitionsRDD [382]\nsave at NativeMethodAccessorImpl.java:0"]; 383 [label="MapPartitionsRDD [383]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1056 { label="WholeStageCodegen (14)"; 384 [label="MapPartitionsRDD [384]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1055 { label="Exchange"; 385 [label="MapPartitionsRDD [385]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->381; 381->382; 382->383; 383->384; 384->385; }

56,57

digraph G { subgraph clusterstage_390 { label="Stage 390 (skipped)"; subgraph cluster1053 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.mediatable"; 377 [label="FileScanRDD [377]\nsave at NativeMethodAccessorImpl.java:0"]; 378 [label="MapPartitionsRDD [378]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1049 { label="WholeStageCodegen (13)"; 379 [label="MapPartitionsRDD [379]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1048 { label="Exchange"; 380 [label="MapPartitionsRDD [380]\nsave at NativeMethodAccessorImpl.java:0"]; } } 377->378; 378->379; 379->380; }

digraph G { subgraph clusterstage_391 { label="Stage 391 (skipped)"; subgraph cluster1477 { label="Exchange"; 553 [label="ShuffledRowRDD [553]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1480 { label="Exchange"; 554 [label="ShuffledRowRDD [554]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1469 { label="WholeStageCodegen (69)"; 555 [label="ZippedPartitionsRDD2 [555]\nsave at NativeMethodAccessorImpl.java:0"]; 556 [label="MapPartitionsRDD [556]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1468 { label="Exchange"; 557 [label="MapPartitionsRDD [557]\nsave at NativeMethodAccessorImpl.java:0"]; } } 553->555; 554->555; 555->556; 556->557; }

380,553

385,554

digraph G { subgraph clusterstage_392 { label="Stage 392 (skipped)"; subgraph cluster1354 { label="Exchange"; 512 [label="ShuffledRowRDD [512]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1382 { label="Exchange"; 519 [label="ShuffledRowRDD [519]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1370 { label="WholeStageCodegen (55)"; 520 [label="ZippedPartitionsRDD2 [520]\nsave at NativeMethodAccessorImpl.java:0"]; 521 [label="MapPartitionsRDD [521]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1369 { label="Window"; 522 [label="MapPartitionsRDD [522]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1364 { label="WholeStageCodegen (56)"; 523 [label="MapPartitionsRDD [523]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1388 { label="Exchange"; 524 [label="ShuffledRowRDD [524]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1384 { label="WholeStageCodegen (57)"; 525 [label="MapPartitionsRDD [525]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1360 { label="WholeStageCodegen (58)"; 526 [label="ZippedPartitionsRDD2 [526]\nsave at NativeMethodAccessorImpl.java:0"]; 527 [label="MapPartitionsRDD [527]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1359 { label="Exchange"; 528 [label="MapPartitionsRDD [528]\nsave at NativeMethodAccessorImpl.java:0"]; } } 512->520; 519->520; 520->521; 521->522; 522->523; 524->525; 523->526; 525->526; 526->527; 527->528; }

366,512

371,519

376,524

digraph G { subgraph clusterstage_393 { label="Stage 393 (skipped)"; subgraph cluster1354 { label="Exchange"; 512 [label="ShuffledRowRDD [512]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1357 { label="Exchange"; 513 [label="ShuffledRowRDD [513]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1344 { label="WholeStageCodegen (53)"; 514 [label="ZippedPartitionsRDD2 [514]\nsave at NativeMethodAccessorImpl.java:0"]; 515 [label="MapPartitionsRDD [515]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1343 { label="Window"; 516 [label="MapPartitionsRDD [516]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1339 { label="WholeStageCodegen (54)"; 517 [label="MapPartitionsRDD [517]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1338 { label="Exchange"; 518 [label="MapPartitionsRDD [518]\nsave at NativeMethodAccessorImpl.java:0"]; } } 512->514; 513->514; 514->515; 515->516; 516->517; 517->518; }

366,512

404,513

digraph G { subgraph clusterstage_394 { label="Stage 394 (skipped)"; subgraph cluster1307 { label="Exchange"; 499 [label="ShuffledRowRDD [499]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1332 { label="Exchange"; 506 [label="ShuffledRowRDD [506]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1319 { label="WholeStageCodegen (51)"; 507 [label="ZippedPartitionsRDD2 [507]\nsave at NativeMethodAccessorImpl.java:0"]; 508 [label="MapPartitionsRDD [508]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1318 { label="Window"; 509 [label="MapPartitionsRDD [509]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1314 { label="WholeStageCodegen (52)"; 510 [label="MapPartitionsRDD [510]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1313 { label="Exchange"; 511 [label="MapPartitionsRDD [511]\nsave at NativeMethodAccessorImpl.java:0"]; } } 499->507; 506->507; 507->508; 508->509; 509->510; 510->511; }

357,499

399,506

digraph G { subgraph clusterstage_395 { label="Stage 395 (skipped)"; subgraph cluster1632 { label="Exchange"; 602 [label="ShuffledRowRDD [602]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1655 { label="Exchange"; 609 [label="ShuffledRowRDD [609]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1643 { label="WholeStageCodegen (89)"; 610 [label="ZippedPartitionsRDD2 [610]\nsave at NativeMethodAccessorImpl.java:0"]; 611 [label="MapPartitionsRDD [611]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1642 { label="Window"; 612 [label="MapPartitionsRDD [612]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1638 { label="WholeStageCodegen (90)"; 613 [label="MapPartitionsRDD [613]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1637 { label="Exchange"; 614 [label="MapPartitionsRDD [614]\nsave at NativeMethodAccessorImpl.java:0"]; } } 602->610; 609->610; 610->611; 611->612; 612->613; 613->614; }

389,602

394,609

digraph G { subgraph clusterstage_396 { label="Stage 396 (skipped)"; subgraph cluster1704 { label="Exchange"; 621 [label="ShuffledRowRDD [621]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1700 { label="WholeStageCodegen (93)"; 622 [label="MapPartitionsRDD [622]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1710 { label="Exchange"; 623 [label="ShuffledRowRDD [623]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1706 { label="WholeStageCodegen (94)"; 624 [label="MapPartitionsRDD [624]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1696 { label="WholeStageCodegen (95)"; 625 [label="ZippedPartitionsRDD2 [625]\nsave at NativeMethodAccessorImpl.java:0"]; 626 [label="MapPartitionsRDD [626]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1716 { label="Exchange"; 627 [label="ShuffledRowRDD [627]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1712 { label="WholeStageCodegen (96)"; 628 [label="MapPartitionsRDD [628]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1692 { label="WholeStageCodegen (97)"; 629 [label="ZippedPartitionsRDD2 [629]\nsave at NativeMethodAccessorImpl.java:0"]; 630 [label="MapPartitionsRDD [630]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1722 { label="Exchange"; 631 [label="ShuffledRowRDD [631]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1718 { label="WholeStageCodegen (98)"; 632 [label="MapPartitionsRDD [632]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1688 { label="WholeStageCodegen (99)"; 633 [label="ZippedPartitionsRDD2 [633]\nsave at NativeMethodAccessorImpl.java:0"]; 634 [label="MapPartitionsRDD [634]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1728 { label="Exchange"; 635 [label="ShuffledRowRDD [635]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1724 { label="WholeStageCodegen (100)"; 636 [label="MapPartitionsRDD [636]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1684 { label="WholeStageCodegen (101)"; 637 [label="ZippedPartitionsRDD2 [637]\nsave at NativeMethodAccessorImpl.java:0"]; 638 [label="MapPartitionsRDD [638]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1683 { label="Exchange"; 639 [label="MapPartitionsRDD [639]\nsave at NativeMethodAccessorImpl.java:0"]; } } 621->622; 623->624; 622->625; 624->625; 625->626; 627->628; 626->629; 628->629; 629->630; 631->632; 630->633; 632->633; 633->634; 635->636; 634->637; 636->637; 637->638; 638->639; }

528,621

557,623

614,627

511,631

518,635

digraph G { subgraph clusterstage_397 { label="Stage 397 (skipped)"; subgraph cluster1632 { label="Exchange"; 602 [label="ShuffledRowRDD [602]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1635 { label="Exchange"; 603 [label="ShuffledRowRDD [603]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1622 { label="WholeStageCodegen (87)"; 604 [label="ZippedPartitionsRDD2 [604]\nsave at NativeMethodAccessorImpl.java:0"]; 605 [label="MapPartitionsRDD [605]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1621 { label="Window"; 606 [label="MapPartitionsRDD [606]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1617 { label="WholeStageCodegen (88)"; 607 [label="MapPartitionsRDD [607]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1616 { label="Exchange"; 608 [label="MapPartitionsRDD [608]\nsave at NativeMethodAccessorImpl.java:0"]; } } 602->604; 603->604; 604->605; 605->606; 606->607; 607->608; }

389,602

418,603

digraph G { subgraph clusterstage_398 { label="Stage 398 (skipped)"; subgraph cluster1307 { label="Exchange"; 499 [label="ShuffledRowRDD [499]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1411 { label="Exchange"; 529 [label="ShuffledRowRDD [529]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1398 { label="WholeStageCodegen (59)"; 530 [label="ZippedPartitionsRDD2 [530]\nsave at NativeMethodAccessorImpl.java:0"]; 531 [label="MapPartitionsRDD [531]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1397 { label="Window"; 532 [label="MapPartitionsRDD [532]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1393 { label="WholeStageCodegen (60)"; 533 [label="MapPartitionsRDD [533]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1392 { label="Exchange"; 534 [label="MapPartitionsRDD [534]\nsave at NativeMethodAccessorImpl.java:0"]; } } 499->530; 529->530; 530->531; 531->532; 532->533; 533->534; }

357,499

423,529

digraph G { subgraph clusterstage_399 { label="Stage 399 (skipped)"; subgraph cluster1566 { label="Exchange"; 583 [label="ShuffledRowRDD [583]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1569 { label="Exchange"; 584 [label="ShuffledRowRDD [584]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1558 { label="WholeStageCodegen (81)"; 585 [label="ZippedPartitionsRDD2 [585]\nsave at NativeMethodAccessorImpl.java:0"]; 586 [label="MapPartitionsRDD [586]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1557 { label="Exchange"; 587 [label="MapPartitionsRDD [587]\nsave at NativeMethodAccessorImpl.java:0"]; } } 583->585; 584->585; 585->586; 586->587; }

408,583

413,584

digraph G { subgraph clusterstage_400 { label="Stage 400 (skipped)"; subgraph cluster1748 { label="AQEShuffleRead"; 640 [label="ShuffledRowRDD [640]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1745 { label="WholeStageCodegen (102)"; 641 [label="MapPartitionsRDD [641]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1753 { label="AQEShuffleRead"; 642 [label="ShuffledRowRDD [642]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1750 { label="WholeStageCodegen (103)"; 643 [label="MapPartitionsRDD [643]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1741 { label="WholeStageCodegen (104)"; 644 [label="ZippedPartitionsRDD2 [644]\nsave at NativeMethodAccessorImpl.java:0"]; 645 [label="MapPartitionsRDD [645]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1758 { label="AQEShuffleRead"; 646 [label="ShuffledRowRDD [646]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1755 { label="WholeStageCodegen (105)"; 647 [label="MapPartitionsRDD [647]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1737 { label="WholeStageCodegen (106)"; 648 [label="ZippedPartitionsRDD2 [648]\nsave at NativeMethodAccessorImpl.java:0"]; 649 [label="MapPartitionsRDD [649]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1763 { label="AQEShuffleRead"; 650 [label="ShuffledRowRDD [650]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1760 { label="WholeStageCodegen (107)"; 651 [label="MapPartitionsRDD [651]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1732 { label="WholeStageCodegen (108)"; 652 [label="ZippedPartitionsRDD2 [652]\nsave at NativeMethodAccessorImpl.java:0"]; 653 [label="MapPartitionsRDD [653]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1731 { label="Exchange"; 654 [label="MapPartitionsRDD [654]\nsave at NativeMethodAccessorImpl.java:0"]; } } 640->641; 642->643; 641->644; 643->644; 644->645; 646->647; 645->648; 647->648; 648->649; 650->651; 649->652; 651->652; 652->653; 653->654; }

639,640

587,642

608,646

534,650

digraph G { subgraph clusterstage_401 { label="Stage 401 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1196 { label="InMemoryTableScan"; 460 [label="MapPartitionsRDD [460]\nsave at NativeMethodAccessorImpl.java:0"]; 461 [label="MapPartitionsRDD [461]\nsave at NativeMethodAccessorImpl.java:0"]; 462 [label="MapPartitionsRDD [462]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1191 { label="WholeStageCodegen (36)"; 463 [label="MapPartitionsRDD [463]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1190 { label="Exchange"; 464 [label="MapPartitionsRDD [464]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->460; 460->461; 461->462; 462->463; 463->464; }

56,57

digraph G { subgraph clusterstage_402 { label="Stage 402 (skipped)"; subgraph cluster1179 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.audiobookdescriptiontable"; 451 [label="FileScanRDD [451]\nsave at NativeMethodAccessorImpl.java:0"]; 452 [label="MapPartitionsRDD [452]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1175 { label="WholeStageCodegen (33)"; 453 [label="MapPartitionsRDD [453]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1174 { label="Exchange"; 454 [label="MapPartitionsRDD [454]\nsave at NativeMethodAccessorImpl.java:0"]; } } 451->452; 452->453; 453->454; }

digraph G { subgraph clusterstage_403 { label="Stage 403 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1187 { label="InMemoryTableScan"; 455 [label="MapPartitionsRDD [455]\nsave at NativeMethodAccessorImpl.java:0"]; 456 [label="MapPartitionsRDD [456]\nsave at NativeMethodAccessorImpl.java:0"]; 457 [label="MapPartitionsRDD [457]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1182 { label="WholeStageCodegen (34)"; 458 [label="MapPartitionsRDD [458]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1181 { label="Exchange"; 459 [label="MapPartitionsRDD [459]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->455; 455->456; 456->457; 457->458; 458->459; }

56,57

digraph G { subgraph clusterstage_404 { label="Stage 404 (skipped)"; subgraph cluster1499 { label="AQEShuffleRead"; 558 [label="ShuffledRowRDD [558]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1496 { label="WholeStageCodegen (70)"; 559 [label="MapPartitionsRDD [559]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1504 { label="AQEShuffleRead"; 560 [label="ShuffledRowRDD [560]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1501 { label="WholeStageCodegen (71)"; 561 [label="MapPartitionsRDD [561]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1489 { label="WholeStageCodegen (72)"; 562 [label="ZippedPartitionsRDD2 [562]\nsave at NativeMethodAccessorImpl.java:0"]; 563 [label="MapPartitionsRDD [563]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1488 { label="Window"; 564 [label="MapPartitionsRDD [564]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1484 { label="WholeStageCodegen (73)"; 565 [label="MapPartitionsRDD [565]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1483 { label="Exchange"; 566 [label="MapPartitionsRDD [566]\nsave at NativeMethodAccessorImpl.java:0"]; } } 558->559; 560->561; 559->562; 561->562; 562->563; 563->564; 564->565; 565->566; }

454,558

459,560

digraph G { subgraph clusterstage_405 { label="Stage 405 (skipped)"; subgraph cluster1632 { label="Exchange"; 602 [label="ShuffledRowRDD [602]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1676 { label="Exchange"; 615 [label="ShuffledRowRDD [615]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1663 { label="WholeStageCodegen (91)"; 616 [label="ZippedPartitionsRDD2 [616]\nsave at NativeMethodAccessorImpl.java:0"]; 617 [label="MapPartitionsRDD [617]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1662 { label="Window"; 618 [label="MapPartitionsRDD [618]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1658 { label="WholeStageCodegen (92)"; 619 [label="MapPartitionsRDD [619]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1657 { label="Exchange"; 620 [label="MapPartitionsRDD [620]\nsave at NativeMethodAccessorImpl.java:0"]; } } 602->616; 615->616; 616->617; 617->618; 618->619; 619->620; }

389,602

464,615

digraph G { subgraph clusterstage_406 { label="Stage 406 (skipped)"; subgraph cluster1772 { label="Exchange"; 655 [label="ShuffledRowRDD [655]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1768 { label="WholeStageCodegen (109)"; 656 [label="MapPartitionsRDD [656]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1767 { label="Exchange"; 657 [label="MapPartitionsRDD [657]\nsave at NativeMethodAccessorImpl.java:0"]; } } 655->656; 656->657; }

654,655

digraph G { subgraph clusterstage_407 { label="Stage 407 (skipped)"; subgraph cluster1524 { label="Exchange"; 567 [label="ShuffledRowRDD [567]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1520 { label="WholeStageCodegen (74)"; 568 [label="MapPartitionsRDD [568]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1530 { label="Exchange"; 569 [label="ShuffledRowRDD [569]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1526 { label="WholeStageCodegen (75)"; 570 [label="MapPartitionsRDD [570]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1513 { label="WholeStageCodegen (76)"; 571 [label="ZippedPartitionsRDD2 [571]\nsave at NativeMethodAccessorImpl.java:0"]; 572 [label="MapPartitionsRDD [572]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1512 { label="Window"; 573 [label="MapPartitionsRDD [573]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1508 { label="WholeStageCodegen (77)"; 574 [label="MapPartitionsRDD [574]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1507 { label="Exchange"; 575 [label="MapPartitionsRDD [575]\nsave at NativeMethodAccessorImpl.java:0"]; } } 567->568; 569->570; 568->571; 570->571; 571->572; 572->573; 573->574; 574->575; }

436,567

441,569

digraph G { subgraph clusterstage_408 { label="Stage 408 (skipped)"; subgraph cluster1458 { label="AQEShuffleRead"; 544 [label="ShuffledRowRDD [544]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1455 { label="WholeStageCodegen (65)"; 545 [label="MapPartitionsRDD [545]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1463 { label="AQEShuffleRead"; 546 [label="ShuffledRowRDD [546]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1460 { label="WholeStageCodegen (66)"; 547 [label="MapPartitionsRDD [547]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1448 { label="WholeStageCodegen (67)"; 548 [label="ZippedPartitionsRDD2 [548]\nsave at NativeMethodAccessorImpl.java:0"]; 549 [label="MapPartitionsRDD [549]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1447 { label="Window"; 550 [label="MapPartitionsRDD [550]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1443 { label="WholeStageCodegen (68)"; 551 [label="MapPartitionsRDD [551]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1442 { label="Exchange"; 552 [label="MapPartitionsRDD [552]\nsave at NativeMethodAccessorImpl.java:0"]; } } 544->545; 546->547; 545->548; 547->548; 548->549; 549->550; 550->551; 551->552; }

445,544

450,546

digraph G { subgraph clusterstage_409 { label="Stage 409 (skipped)"; subgraph cluster1585 { label="AQEShuffleRead"; 588 [label="ShuffledRowRDD [588]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1582 { label="WholeStageCodegen (82)"; 589 [label="MapPartitionsRDD [589]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1590 { label="AQEShuffleRead"; 590 [label="ShuffledRowRDD [590]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1587 { label="WholeStageCodegen (83)"; 591 [label="MapPartitionsRDD [591]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1578 { label="WholeStageCodegen (84)"; 592 [label="ZippedPartitionsRDD2 [592]\nsave at NativeMethodAccessorImpl.java:0"]; 593 [label="MapPartitionsRDD [593]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1577 { label="Exchange"; 594 [label="MapPartitionsRDD [594]\nsave at NativeMethodAccessorImpl.java:0"]; } } 588->589; 590->591; 589->592; 591->592; 592->593; 593->594; }

582,588

489,590

digraph G { subgraph clusterstage_410 { label="Stage 410 (skipped)"; subgraph cluster1608 { label="Exchange"; 595 [label="ShuffledRowRDD [595]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1611 { label="Exchange"; 596 [label="ShuffledRowRDD [596]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1599 { label="WholeStageCodegen (85)"; 597 [label="ZippedPartitionsRDD2 [597]\nsave at NativeMethodAccessorImpl.java:0"]; 598 [label="MapPartitionsRDD [598]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1598 { label="Window"; 599 [label="MapPartitionsRDD [599]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1594 { label="WholeStageCodegen (86)"; 600 [label="MapPartitionsRDD [600]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1593 { label="Exchange"; 601 [label="MapPartitionsRDD [601]\nsave at NativeMethodAccessorImpl.java:0"]; } } 595->597; 596->597; 597->598; 598->599; 599->600; 600->601; }

468,595

473,596

digraph G { subgraph clusterstage_411 { label="Stage 411 (skipped)"; subgraph cluster1307 { label="Exchange"; 499 [label="ShuffledRowRDD [499]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1310 { label="Exchange"; 500 [label="ShuffledRowRDD [500]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1298 { label="WholeStageCodegen (49)"; 501 [label="ZippedPartitionsRDD2 [501]\nsave at NativeMethodAccessorImpl.java:0"]; 502 [label="MapPartitionsRDD [502]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1297 { label="Window"; 503 [label="MapPartitionsRDD [503]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1293 { label="WholeStageCodegen (50)"; 504 [label="MapPartitionsRDD [504]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1292 { label="Exchange"; 505 [label="MapPartitionsRDD [505]\nsave at NativeMethodAccessorImpl.java:0"]; } } 499->501; 500->501; 501->502; 502->503; 503->504; 504->505; }

357,499

362,500

digraph G { subgraph clusterstage_412 { label="Stage 412 (skipped)"; subgraph cluster1432 { label="AQEShuffleRead"; 535 [label="ShuffledRowRDD [535]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1429 { label="WholeStageCodegen (61)"; 536 [label="MapPartitionsRDD [536]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1437 { label="AQEShuffleRead"; 537 [label="ShuffledRowRDD [537]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1434 { label="WholeStageCodegen (62)"; 538 [label="MapPartitionsRDD [538]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1422 { label="WholeStageCodegen (63)"; 539 [label="ZippedPartitionsRDD2 [539]\nsave at NativeMethodAccessorImpl.java:0"]; 540 [label="MapPartitionsRDD [540]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1421 { label="Window"; 541 [label="MapPartitionsRDD [541]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1417 { label="WholeStageCodegen (64)"; 542 [label="MapPartitionsRDD [542]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1416 { label="Exchange"; 543 [label="MapPartitionsRDD [543]\nsave at NativeMethodAccessorImpl.java:0"]; } } 535->536; 537->538; 536->539; 538->539; 539->540; 540->541; 541->542; 542->543; }

427,535

432,537

digraph G { subgraph clusterstage_413 { label="Stage 413"; subgraph cluster1812 { label="Exchange"; 658 [label="ShuffledRowRDD [658]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1808 { label="WholeStageCodegen (110)"; 659 [label="MapPartitionsRDD [659]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1818 { label="Exchange"; 660 [label="ShuffledRowRDD [660]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1814 { label="WholeStageCodegen (111)"; 661 [label="MapPartitionsRDD [661]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1804 { label="WholeStageCodegen (112)"; 662 [label="ZippedPartitionsRDD2 [662]\nsave at NativeMethodAccessorImpl.java:0"]; 663 [label="MapPartitionsRDD [663]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1824 { label="Exchange"; 664 [label="ShuffledRowRDD [664]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1820 { label="WholeStageCodegen (113)"; 665 [label="MapPartitionsRDD [665]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1800 { label="WholeStageCodegen (114)"; 666 [label="ZippedPartitionsRDD2 [666]\nsave at NativeMethodAccessorImpl.java:0"]; 667 [label="MapPartitionsRDD [667]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1830 { label="Exchange"; 668 [label="ShuffledRowRDD [668]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1826 { label="WholeStageCodegen (115)"; 669 [label="MapPartitionsRDD [669]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1796 { label="WholeStageCodegen (116)"; 670 [label="ZippedPartitionsRDD2 [670]\nsave at NativeMethodAccessorImpl.java:0"]; 671 [label="MapPartitionsRDD [671]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1836 { label="Exchange"; 672 [label="ShuffledRowRDD [672]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1832 { label="WholeStageCodegen (117)"; 673 [label="MapPartitionsRDD [673]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1792 { label="WholeStageCodegen (118)"; 674 [label="ZippedPartitionsRDD2 [674]\nsave at NativeMethodAccessorImpl.java:0"]; 675 [label="MapPartitionsRDD [675]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1842 { label="Exchange"; 676 [label="ShuffledRowRDD [676]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1838 { label="WholeStageCodegen (119)"; 677 [label="MapPartitionsRDD [677]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1788 { label="WholeStageCodegen (120)"; 678 [label="ZippedPartitionsRDD2 [678]\nsave at NativeMethodAccessorImpl.java:0"]; 679 [label="MapPartitionsRDD [679]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1848 { label="Exchange"; 680 [label="ShuffledRowRDD [680]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1844 { label="WholeStageCodegen (121)"; 681 [label="MapPartitionsRDD [681]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1784 { label="WholeStageCodegen (122)"; 682 [label="ZippedPartitionsRDD2 [682]\nsave at NativeMethodAccessorImpl.java:0"]; 683 [label="MapPartitionsRDD [683]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1854 { label="Exchange"; 684 [label="ShuffledRowRDD [684]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1850 { label="WholeStageCodegen (123)"; 685 [label="MapPartitionsRDD [685]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1780 { label="WholeStageCodegen (124)"; 686 [label="ZippedPartitionsRDD2 [686]\nsave at NativeMethodAccessorImpl.java:0"]; 687 [label="MapPartitionsRDD [687]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1860 { label="Exchange"; 688 [label="ShuffledRowRDD [688]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1856 { label="WholeStageCodegen (125)"; 689 [label="MapPartitionsRDD [689]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1776 { label="WholeStageCodegen (126)"; 690 [label="ZippedPartitionsRDD2 [690]\nsave at NativeMethodAccessorImpl.java:0"]; 691 [label="MapPartitionsRDD [691]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1775 { label="Exchange"; 692 [label="MapPartitionsRDD [692]\nsave at NativeMethodAccessorImpl.java:0"]; 693 [label="MapPartitionsRDD [693]\nsave at NativeMethodAccessorImpl.java:0"]; } } 658->659; 660->661; 659->662; 661->662; 662->663; 664->665; 663->666; 665->666; 666->667; 668->669; 667->670; 669->670; 670->671; 672->673; 671->674; 673->674; 674->675; 676->677; 675->678; 677->678; 678->679; 680->681; 679->682; 681->682; 682->683; 684->685; 683->686; 685->686; 686->687; 688->689; 687->690; 689->690; 690->691; 691->692; 692->693; }

594,658

505,660

657,664

543,668

575,672

552,676

566,680

620,684

601,688

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
413	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:21:07	23 s	1000/1000			18.5 GB	21.9 GB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

413

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:21:07

23 s

1000/1000

18.5 GB

21.9 GB

Skipped Stages (57)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
412	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/3
411	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
410	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
409	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/821
408	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/28
407	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
406	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
405	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
404	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/893
403	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
402	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/753
401	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
400	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/501
399	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
398	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
397	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
396	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
395	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
394	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
393	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
392	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
391	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
390	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/639
389	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
388	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
387	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
386	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
385	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/649
384	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
383	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
382	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
381	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/642
380	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
379	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
378	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/639
377	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/682
376	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
375	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/676
374	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
373	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/500
372	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/782
371	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/109
370	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
369	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
368	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/639
367	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
366	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
365	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/680
364	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
363	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
362	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/642
361	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
360	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
359	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/651
358	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
357	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/709
356	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433