paxus_etl - Details for Job 113

Details for Job 113

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 16

Enable zooming

digraph G { subgraph clusterstage_186 { label="Stage 186 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_283 { label="Stage 283 (skipped)"; subgraph cluster1068 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.videodescriptiontable"; 386 [label="FileScanRDD [386]\nsave at NativeMethodAccessorImpl.java:0"]; 387 [label="MapPartitionsRDD [387]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1064 { label="WholeStageCodegen (15)"; 388 [label="MapPartitionsRDD [388]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1063 { label="Exchange"; 389 [label="MapPartitionsRDD [389]\nsave at NativeMethodAccessorImpl.java:0"]; } } 386->387; 387->388; 388->389; }

digraph G { subgraph clusterstage_290 { label="Stage 290 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1076 { label="InMemoryTableScan"; 390 [label="MapPartitionsRDD [390]\nsave at NativeMethodAccessorImpl.java:0"]; 391 [label="MapPartitionsRDD [391]\nsave at NativeMethodAccessorImpl.java:0"]; 392 [label="MapPartitionsRDD [392]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1071 { label="WholeStageCodegen (16)"; 393 [label="MapPartitionsRDD [393]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1070 { label="Exchange"; 394 [label="MapPartitionsRDD [394]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->390; 390->391; 391->392; 392->393; 393->394; }

56,57

digraph G { subgraph clusterstage_291 { label="Stage 291 (skipped)"; subgraph cluster1017 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.audiodescriptiontable"; 354 [label="FileScanRDD [354]\nsave at NativeMethodAccessorImpl.java:0"]; 355 [label="MapPartitionsRDD [355]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1013 { label="WholeStageCodegen (8)"; 356 [label="MapPartitionsRDD [356]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1012 { label="Exchange"; 357 [label="MapPartitionsRDD [357]\nsave at NativeMethodAccessorImpl.java:0"]; } } 354->355; 355->356; 356->357; }

digraph G { subgraph clusterstage_292 { label="Stage 292 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1085 { label="InMemoryTableScan"; 395 [label="MapPartitionsRDD [395]\nsave at NativeMethodAccessorImpl.java:0"]; 396 [label="MapPartitionsRDD [396]\nsave at NativeMethodAccessorImpl.java:0"]; 397 [label="MapPartitionsRDD [397]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1080 { label="WholeStageCodegen (18)"; 398 [label="MapPartitionsRDD [398]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1079 { label="Exchange"; 399 [label="MapPartitionsRDD [399]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->395; 395->396; 396->397; 397->398; 398->399; }

56,57

digraph G { subgraph clusterstage_293 { label="Stage 293 (skipped)"; subgraph cluster1032 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.aggregatetable"; 363 [label="FileScanRDD [363]\nsave at NativeMethodAccessorImpl.java:0"]; 364 [label="MapPartitionsRDD [364]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1028 { label="WholeStageCodegen (10)"; 365 [label="MapPartitionsRDD [365]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1027 { label="Exchange"; 366 [label="MapPartitionsRDD [366]\nsave at NativeMethodAccessorImpl.java:0"]; } } 363->364; 364->365; 365->366; }

digraph G { subgraph clusterstage_294 { label="Stage 294 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1094 { label="InMemoryTableScan"; 400 [label="MapPartitionsRDD [400]\nsave at NativeMethodAccessorImpl.java:0"]; 401 [label="MapPartitionsRDD [401]\nsave at NativeMethodAccessorImpl.java:0"]; 402 [label="MapPartitionsRDD [402]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1089 { label="WholeStageCodegen (20)"; 403 [label="MapPartitionsRDD [403]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1088 { label="Exchange"; 404 [label="MapPartitionsRDD [404]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->400; 400->401; 401->402; 402->403; 403->404; }

56,57

digraph G { subgraph clusterstage_295 { label="Stage 295 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1040 { label="InMemoryTableScan"; 367 [label="MapPartitionsRDD [367]\nsave at NativeMethodAccessorImpl.java:0"]; 368 [label="MapPartitionsRDD [368]\nsave at NativeMethodAccessorImpl.java:0"]; 369 [label="MapPartitionsRDD [369]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1035 { label="WholeStageCodegen (11)"; 370 [label="MapPartitionsRDD [370]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1034 { label="Exchange"; 371 [label="MapPartitionsRDD [371]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->367; 367->368; 368->369; 369->370; 370->371; }

56,57

digraph G { subgraph clusterstage_296 { label="Stage 296 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1046 { label="InMemoryTableScan"; 372 [label="MapPartitionsRDD [372]\nsave at NativeMethodAccessorImpl.java:0"]; 373 [label="MapPartitionsRDD [373]\nsave at NativeMethodAccessorImpl.java:0"]; 374 [label="MapPartitionsRDD [374]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1043 { label="WholeStageCodegen (12)"; 375 [label="MapPartitionsRDD [375]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1042 { label="Exchange"; 376 [label="MapPartitionsRDD [376]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->372; 372->373; 373->374; 374->375; 375->376; }

56,57

digraph G { subgraph clusterstage_297 { label="Stage 297 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1061 { label="InMemoryTableScan"; 381 [label="MapPartitionsRDD [381]\nsave at NativeMethodAccessorImpl.java:0"]; 382 [label="MapPartitionsRDD [382]\nsave at NativeMethodAccessorImpl.java:0"]; 383 [label="MapPartitionsRDD [383]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1056 { label="WholeStageCodegen (14)"; 384 [label="MapPartitionsRDD [384]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1055 { label="Exchange"; 385 [label="MapPartitionsRDD [385]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->381; 381->382; 382->383; 383->384; 384->385; }

56,57

digraph G { subgraph clusterstage_298 { label="Stage 298 (skipped)"; subgraph cluster1053 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.mediatable"; 377 [label="FileScanRDD [377]\nsave at NativeMethodAccessorImpl.java:0"]; 378 [label="MapPartitionsRDD [378]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1049 { label="WholeStageCodegen (13)"; 379 [label="MapPartitionsRDD [379]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1048 { label="Exchange"; 380 [label="MapPartitionsRDD [380]\nsave at NativeMethodAccessorImpl.java:0"]; } } 377->378; 378->379; 379->380; }

digraph G { subgraph clusterstage_299 { label="Stage 299 (skipped)"; subgraph cluster1477 { label="Exchange"; 553 [label="ShuffledRowRDD [553]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1480 { label="Exchange"; 554 [label="ShuffledRowRDD [554]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1469 { label="WholeStageCodegen (69)"; 555 [label="ZippedPartitionsRDD2 [555]\nsave at NativeMethodAccessorImpl.java:0"]; 556 [label="MapPartitionsRDD [556]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1468 { label="Exchange"; 557 [label="MapPartitionsRDD [557]\nsave at NativeMethodAccessorImpl.java:0"]; } } 553->555; 554->555; 555->556; 556->557; }

380,553

385,554

digraph G { subgraph clusterstage_300 { label="Stage 300 (skipped)"; subgraph cluster1354 { label="Exchange"; 512 [label="ShuffledRowRDD [512]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1382 { label="Exchange"; 519 [label="ShuffledRowRDD [519]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1370 { label="WholeStageCodegen (55)"; 520 [label="ZippedPartitionsRDD2 [520]\nsave at NativeMethodAccessorImpl.java:0"]; 521 [label="MapPartitionsRDD [521]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1369 { label="Window"; 522 [label="MapPartitionsRDD [522]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1364 { label="WholeStageCodegen (56)"; 523 [label="MapPartitionsRDD [523]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1388 { label="Exchange"; 524 [label="ShuffledRowRDD [524]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1384 { label="WholeStageCodegen (57)"; 525 [label="MapPartitionsRDD [525]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1360 { label="WholeStageCodegen (58)"; 526 [label="ZippedPartitionsRDD2 [526]\nsave at NativeMethodAccessorImpl.java:0"]; 527 [label="MapPartitionsRDD [527]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1359 { label="Exchange"; 528 [label="MapPartitionsRDD [528]\nsave at NativeMethodAccessorImpl.java:0"]; } } 512->520; 519->520; 520->521; 521->522; 522->523; 524->525; 523->526; 525->526; 526->527; 527->528; }

366,512

371,519

376,524

digraph G { subgraph clusterstage_301 { label="Stage 301 (skipped)"; subgraph cluster1354 { label="Exchange"; 512 [label="ShuffledRowRDD [512]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1357 { label="Exchange"; 513 [label="ShuffledRowRDD [513]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1344 { label="WholeStageCodegen (53)"; 514 [label="ZippedPartitionsRDD2 [514]\nsave at NativeMethodAccessorImpl.java:0"]; 515 [label="MapPartitionsRDD [515]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1343 { label="Window"; 516 [label="MapPartitionsRDD [516]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1339 { label="WholeStageCodegen (54)"; 517 [label="MapPartitionsRDD [517]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1338 { label="Exchange"; 518 [label="MapPartitionsRDD [518]\nsave at NativeMethodAccessorImpl.java:0"]; } } 512->514; 513->514; 514->515; 515->516; 516->517; 517->518; }

366,512

404,513

digraph G { subgraph clusterstage_302 { label="Stage 302 (skipped)"; subgraph cluster1307 { label="Exchange"; 499 [label="ShuffledRowRDD [499]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1332 { label="Exchange"; 506 [label="ShuffledRowRDD [506]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1319 { label="WholeStageCodegen (51)"; 507 [label="ZippedPartitionsRDD2 [507]\nsave at NativeMethodAccessorImpl.java:0"]; 508 [label="MapPartitionsRDD [508]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1318 { label="Window"; 509 [label="MapPartitionsRDD [509]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1314 { label="WholeStageCodegen (52)"; 510 [label="MapPartitionsRDD [510]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1313 { label="Exchange"; 511 [label="MapPartitionsRDD [511]\nsave at NativeMethodAccessorImpl.java:0"]; } } 499->507; 506->507; 507->508; 508->509; 509->510; 510->511; }

357,499

399,506

digraph G { subgraph clusterstage_303 { label="Stage 303 (skipped)"; subgraph cluster1632 { label="Exchange"; 602 [label="ShuffledRowRDD [602]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1655 { label="Exchange"; 609 [label="ShuffledRowRDD [609]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1643 { label="WholeStageCodegen (89)"; 610 [label="ZippedPartitionsRDD2 [610]\nsave at NativeMethodAccessorImpl.java:0"]; 611 [label="MapPartitionsRDD [611]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1642 { label="Window"; 612 [label="MapPartitionsRDD [612]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1638 { label="WholeStageCodegen (90)"; 613 [label="MapPartitionsRDD [613]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1637 { label="Exchange"; 614 [label="MapPartitionsRDD [614]\nsave at NativeMethodAccessorImpl.java:0"]; } } 602->610; 609->610; 610->611; 611->612; 612->613; 613->614; }

389,602

394,609

digraph G { subgraph clusterstage_304 { label="Stage 304"; subgraph cluster1704 { label="Exchange"; 621 [label="ShuffledRowRDD [621]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1700 { label="WholeStageCodegen (93)"; 622 [label="MapPartitionsRDD [622]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1710 { label="Exchange"; 623 [label="ShuffledRowRDD [623]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1706 { label="WholeStageCodegen (94)"; 624 [label="MapPartitionsRDD [624]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1696 { label="WholeStageCodegen (95)"; 625 [label="ZippedPartitionsRDD2 [625]\nsave at NativeMethodAccessorImpl.java:0"]; 626 [label="MapPartitionsRDD [626]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1716 { label="Exchange"; 627 [label="ShuffledRowRDD [627]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1712 { label="WholeStageCodegen (96)"; 628 [label="MapPartitionsRDD [628]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1692 { label="WholeStageCodegen (97)"; 629 [label="ZippedPartitionsRDD2 [629]\nsave at NativeMethodAccessorImpl.java:0"]; 630 [label="MapPartitionsRDD [630]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1722 { label="Exchange"; 631 [label="ShuffledRowRDD [631]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1718 { label="WholeStageCodegen (98)"; 632 [label="MapPartitionsRDD [632]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1688 { label="WholeStageCodegen (99)"; 633 [label="ZippedPartitionsRDD2 [633]\nsave at NativeMethodAccessorImpl.java:0"]; 634 [label="MapPartitionsRDD [634]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1728 { label="Exchange"; 635 [label="ShuffledRowRDD [635]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1724 { label="WholeStageCodegen (100)"; 636 [label="MapPartitionsRDD [636]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1684 { label="WholeStageCodegen (101)"; 637 [label="ZippedPartitionsRDD2 [637]\nsave at NativeMethodAccessorImpl.java:0"]; 638 [label="MapPartitionsRDD [638]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1683 { label="Exchange"; 639 [label="MapPartitionsRDD [639]\nsave at NativeMethodAccessorImpl.java:0"]; } } 621->622; 623->624; 622->625; 624->625; 625->626; 627->628; 626->629; 628->629; 629->630; 631->632; 630->633; 632->633; 633->634; 635->636; 634->637; 636->637; 637->638; 638->639; }

528,621

557,623

614,627

511,631

518,635

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
304	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:16:30	4 s	1000/1000			7.3 GB	1756.0 MB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

304

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:16:30

4 s

1000/1000

7.3 GB

1756.0 MB

Skipped Stages (16)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
303	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
302	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
301	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
300	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
299	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
298	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/639
297	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
296	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
295	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
294	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
293	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/649
292	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
291	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/651
290	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
283	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/642
186	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433