paxus_etl - Details for Job 108

Details for Job 108

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 12

Enable zooming

digraph G { subgraph clusterstage_186 { label="Stage 186"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_268 { label="Stage 268 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1010 { label="InMemoryTableScan"; 349 [label="MapPartitionsRDD [349]\nsave at NativeMethodAccessorImpl.java:0"]; 350 [label="MapPartitionsRDD [350]\nsave at NativeMethodAccessorImpl.java:0"]; 351 [label="MapPartitionsRDD [351]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1005 { label="WholeStageCodegen (7)"; 352 [label="MapPartitionsRDD [352]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1004 { label="Exchange"; 353 [label="MapPartitionsRDD [353]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->349; 349->350; 350->351; 351->352; 352->353; }

56,57

digraph G { subgraph clusterstage_269 { label="Stage 269 (skipped)"; subgraph cluster1002 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.ratingtable"; 345 [label="FileScanRDD [345]\nsave at NativeMethodAccessorImpl.java:0"]; 346 [label="MapPartitionsRDD [346]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster998 { label="WholeStageCodegen (6)"; 347 [label="MapPartitionsRDD [347]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster997 { label="Exchange"; 348 [label="MapPartitionsRDD [348]\nsave at NativeMethodAccessorImpl.java:0"]; } } 345->346; 346->347; 347->348; }

digraph G { subgraph clusterstage_270 { label="Stage 270 (skipped)"; subgraph cluster987 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.versiontable"; 336 [label="FileScanRDD [336]\nsave at NativeMethodAccessorImpl.java:0"]; 337 [label="MapPartitionsRDD [337]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster983 { label="WholeStageCodegen (4)"; 338 [label="MapPartitionsRDD [338]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster982 { label="Exchange"; 339 [label="MapPartitionsRDD [339]\nsave at NativeMethodAccessorImpl.java:0"]; } } 336->337; 337->338; 338->339; }

digraph G { subgraph clusterstage_271 { label="Stage 271 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster995 { label="InMemoryTableScan"; 340 [label="MapPartitionsRDD [340]\nsave at NativeMethodAccessorImpl.java:0"]; 341 [label="MapPartitionsRDD [341]\nsave at NativeMethodAccessorImpl.java:0"]; 342 [label="MapPartitionsRDD [342]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster990 { label="WholeStageCodegen (5)"; 343 [label="MapPartitionsRDD [343]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster989 { label="Exchange"; 344 [label="MapPartitionsRDD [344]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->340; 340->341; 341->342; 342->343; 343->344; }

56,57

digraph G { subgraph clusterstage_272 { label="Stage 272 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster974 { label="InMemoryTableScan"; 326 [label="MapPartitionsRDD [326]\nsave at NativeMethodAccessorImpl.java:0"]; 327 [label="MapPartitionsRDD [327]\nsave at NativeMethodAccessorImpl.java:0"]; 328 [label="MapPartitionsRDD [328]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster969 { label="WholeStageCodegen (2)"; 329 [label="MapPartitionsRDD [329]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster968 { label="Exchange"; 330 [label="MapPartitionsRDD [330]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->326; 326->327; 327->328; 328->329; 329->330; }

56,57

digraph G { subgraph clusterstage_273 { label="Stage 273 (skipped)"; subgraph cluster966 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.mediatable"; 322 [label="FileScanRDD [322]\nsave at NativeMethodAccessorImpl.java:0"]; 323 [label="MapPartitionsRDD [323]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster962 { label="WholeStageCodegen (1)"; 324 [label="MapPartitionsRDD [324]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster961 { label="Exchange"; 325 [label="MapPartitionsRDD [325]\nsave at NativeMethodAccessorImpl.java:0"]; } } 322->323; 323->324; 324->325; }

digraph G { subgraph clusterstage_274 { label="Stage 274 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster980 { label="InMemoryTableScan"; 331 [label="MapPartitionsRDD [331]\nsave at NativeMethodAccessorImpl.java:0"]; 332 [label="MapPartitionsRDD [332]\nsave at NativeMethodAccessorImpl.java:0"]; 333 [label="MapPartitionsRDD [333]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster977 { label="WholeStageCodegen (3)"; 334 [label="MapPartitionsRDD [334]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster976 { label="Exchange"; 335 [label="MapPartitionsRDD [335]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->331; 331->332; 332->333; 333->334; 334->335; }

56,57

digraph G { subgraph clusterstage_275 { label="Stage 275 (skipped)"; subgraph cluster1278 { label="Exchange"; 490 [label="ShuffledRowRDD [490]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1281 { label="Exchange"; 491 [label="ShuffledRowRDD [491]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1269 { label="WholeStageCodegen (46)"; 492 [label="ZippedPartitionsRDD2 [492]\nsave at NativeMethodAccessorImpl.java:0"]; 493 [label="MapPartitionsRDD [493]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1287 { label="Exchange"; 494 [label="ShuffledRowRDD [494]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1283 { label="WholeStageCodegen (47)"; 495 [label="MapPartitionsRDD [495]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1265 { label="WholeStageCodegen (48)"; 496 [label="ZippedPartitionsRDD2 [496]\nsave at NativeMethodAccessorImpl.java:0"]; 497 [label="MapPartitionsRDD [497]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1264 { label="Exchange"; 498 [label="MapPartitionsRDD [498]\nsave at NativeMethodAccessorImpl.java:0"]; } } 490->492; 491->492; 492->493; 494->495; 493->496; 495->496; 496->497; 497->498; }

325,490

330,491

335,494

digraph G { subgraph clusterstage_276 { label="Stage 276 (skipped)"; subgraph cluster1227 { label="AQEShuffleRead"; 474 [label="ShuffledRowRDD [474]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1224 { label="WholeStageCodegen (39)"; 475 [label="MapPartitionsRDD [475]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1232 { label="AQEShuffleRead"; 476 [label="ShuffledRowRDD [476]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1229 { label="WholeStageCodegen (40)"; 477 [label="MapPartitionsRDD [477]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1218 { label="WholeStageCodegen (41)"; 478 [label="ZippedPartitionsRDD2 [478]\nsave at NativeMethodAccessorImpl.java:0"]; 479 [label="MapPartitionsRDD [479]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1217 { label="Exchange"; 480 [label="MapPartitionsRDD [480]\nsave at NativeMethodAccessorImpl.java:0"]; } } 474->475; 476->477; 475->478; 477->478; 478->479; 479->480; }

339,474

344,476

digraph G { subgraph clusterstage_277 { label="Stage 277 (skipped)"; subgraph cluster1544 { label="AQEShuffleRead"; 576 [label="ShuffledRowRDD [576]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1541 { label="WholeStageCodegen (78)"; 577 [label="MapPartitionsRDD [577]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1549 { label="AQEShuffleRead"; 578 [label="ShuffledRowRDD [578]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1546 { label="WholeStageCodegen (79)"; 579 [label="MapPartitionsRDD [579]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1537 { label="WholeStageCodegen (80)"; 580 [label="ZippedPartitionsRDD2 [580]\nsave at NativeMethodAccessorImpl.java:0"]; 581 [label="MapPartitionsRDD [581]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1536 { label="Exchange"; 582 [label="MapPartitionsRDD [582]\nsave at NativeMethodAccessorImpl.java:0"]; } } 576->577; 578->579; 577->580; 579->580; 580->581; 581->582; }

498,576

480,578

digraph G { subgraph clusterstage_278 { label="Stage 278 (skipped)"; subgraph cluster1252 { label="AQEShuffleRead"; 481 [label="ShuffledRowRDD [481]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1249 { label="WholeStageCodegen (42)"; 482 [label="MapPartitionsRDD [482]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1257 { label="AQEShuffleRead"; 483 [label="ShuffledRowRDD [483]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1254 { label="WholeStageCodegen (43)"; 484 [label="MapPartitionsRDD [484]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1242 { label="WholeStageCodegen (44)"; 485 [label="ZippedPartitionsRDD2 [485]\nsave at NativeMethodAccessorImpl.java:0"]; 486 [label="MapPartitionsRDD [486]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1241 { label="Window"; 487 [label="MapPartitionsRDD [487]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1237 { label="WholeStageCodegen (45)"; 488 [label="MapPartitionsRDD [488]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1236 { label="Exchange"; 489 [label="MapPartitionsRDD [489]\nsave at NativeMethodAccessorImpl.java:0"]; } } 481->482; 483->484; 482->485; 484->485; 485->486; 486->487; 487->488; 488->489; }

348,481

353,483

digraph G { subgraph clusterstage_279 { label="Stage 279"; subgraph cluster1585 { label="AQEShuffleRead"; 588 [label="ShuffledRowRDD [588]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1582 { label="WholeStageCodegen (82)"; 589 [label="MapPartitionsRDD [589]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1590 { label="AQEShuffleRead"; 590 [label="ShuffledRowRDD [590]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1587 { label="WholeStageCodegen (83)"; 591 [label="MapPartitionsRDD [591]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1578 { label="WholeStageCodegen (84)"; 592 [label="ZippedPartitionsRDD2 [592]\nsave at NativeMethodAccessorImpl.java:0"]; 593 [label="MapPartitionsRDD [593]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1577 { label="Exchange"; 594 [label="MapPartitionsRDD [594]\nsave at NativeMethodAccessorImpl.java:0"]; } } 588->589; 590->591; 589->592; 591->592; 592->593; 593->594; }

582,588

489,590

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
279	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:15:15	5 s	821/821			2.0 GB	6.3 GB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

279

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:15:15

5 s

821/821

2.0 GB

6.3 GB

Skipped Stages (12)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
278	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/500
277	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/782
276	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/109
275	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
274	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
273	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/639
272	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
271	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
270	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/680
269	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
268	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
186	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433