paxus_etl - Details for Job 125

Details for Job 125

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 8

Enable zooming

digraph G { subgraph clusterstage_488 { label="Stage 488 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_489 { label="Stage 489 (skipped)"; subgraph cluster2273 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.soundtracktable"; 697 [label="FileScanRDD [697]\nsave at NativeMethodAccessorImpl.java:0"]; 698 [label="MapPartitionsRDD [698]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2269 { label="WholeStageCodegen (1)"; 699 [label="MapPartitionsRDD [699]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2268 { label="Exchange"; 700 [label="MapPartitionsRDD [700]\nsave at NativeMethodAccessorImpl.java:0"]; } } 697->698; 698->699; 699->700; }

digraph G { subgraph clusterstage_490 { label="Stage 490 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2287 { label="InMemoryTableScan"; 706 [label="MapPartitionsRDD [706]\nsave at NativeMethodAccessorImpl.java:0"]; 707 [label="MapPartitionsRDD [707]\nsave at NativeMethodAccessorImpl.java:0"]; 708 [label="MapPartitionsRDD [708]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2284 { label="WholeStageCodegen (3)"; 709 [label="MapPartitionsRDD [709]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2283 { label="Exchange"; 710 [label="MapPartitionsRDD [710]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->706; 706->707; 707->708; 708->709; 709->710; }

56,57

digraph G { subgraph clusterstage_491 { label="Stage 491 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2281 { label="InMemoryTableScan"; 701 [label="MapPartitionsRDD [701]\nsave at NativeMethodAccessorImpl.java:0"]; 702 [label="MapPartitionsRDD [702]\nsave at NativeMethodAccessorImpl.java:0"]; 703 [label="MapPartitionsRDD [703]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2276 { label="WholeStageCodegen (2)"; 704 [label="MapPartitionsRDD [704]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2275 { label="Exchange"; 705 [label="MapPartitionsRDD [705]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->701; 701->702; 702->703; 703->704; 704->705; }

56,57

digraph G { subgraph clusterstage_492 { label="Stage 492 (skipped)"; subgraph cluster2294 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.languagetable"; 711 [label="FileScanRDD [711]\nsave at NativeMethodAccessorImpl.java:0"]; 712 [label="MapPartitionsRDD [712]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2290 { label="WholeStageCodegen (4)"; 713 [label="MapPartitionsRDD [713]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2289 { label="Exchange"; 714 [label="MapPartitionsRDD [714]\nsave at NativeMethodAccessorImpl.java:0"]; } } 711->712; 712->713; 713->714; }

digraph G { subgraph clusterstage_493 { label="Stage 493 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2302 { label="InMemoryTableScan"; 715 [label="MapPartitionsRDD [715]\nsave at NativeMethodAccessorImpl.java:0"]; 716 [label="MapPartitionsRDD [716]\nsave at NativeMethodAccessorImpl.java:0"]; 717 [label="MapPartitionsRDD [717]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2297 { label="WholeStageCodegen (5)"; 718 [label="MapPartitionsRDD [718]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2296 { label="Exchange"; 719 [label="MapPartitionsRDD [719]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->715; 715->716; 716->717; 717->718; 718->719; }

56,57

digraph G { subgraph clusterstage_494 { label="Stage 494 (skipped)"; subgraph cluster2325 { label="Exchange"; 720 [label="ShuffledRowRDD [720]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2321 { label="WholeStageCodegen (6)"; 721 [label="MapPartitionsRDD [721]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2331 { label="Exchange"; 722 [label="ShuffledRowRDD [722]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2327 { label="WholeStageCodegen (7)"; 723 [label="MapPartitionsRDD [723]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2314 { label="WholeStageCodegen (8)"; 724 [label="ZippedPartitionsRDD2 [724]\nsave at NativeMethodAccessorImpl.java:0"]; 725 [label="MapPartitionsRDD [725]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2313 { label="Window"; 726 [label="MapPartitionsRDD [726]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2309 { label="WholeStageCodegen (9)"; 727 [label="MapPartitionsRDD [727]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2308 { label="Exchange"; 728 [label="MapPartitionsRDD [728]\nsave at NativeMethodAccessorImpl.java:0"]; } } 720->721; 722->723; 721->724; 723->724; 724->725; 725->726; 726->727; 727->728; }

714,720

719,722

digraph G { subgraph clusterstage_495 { label="Stage 495 (skipped)"; subgraph cluster2349 { label="Exchange"; 729 [label="ShuffledRowRDD [729]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2352 { label="Exchange"; 730 [label="ShuffledRowRDD [730]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2340 { label="WholeStageCodegen (10)"; 731 [label="ZippedPartitionsRDD2 [731]\nsave at NativeMethodAccessorImpl.java:0"]; 732 [label="MapPartitionsRDD [732]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2358 { label="Exchange"; 733 [label="ShuffledRowRDD [733]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2354 { label="WholeStageCodegen (11)"; 734 [label="MapPartitionsRDD [734]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2336 { label="WholeStageCodegen (12)"; 735 [label="ZippedPartitionsRDD2 [735]\nsave at NativeMethodAccessorImpl.java:0"]; 736 [label="MapPartitionsRDD [736]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2335 { label="Exchange"; 737 [label="MapPartitionsRDD [737]\nsave at NativeMethodAccessorImpl.java:0"]; } } 729->731; 730->731; 731->732; 733->734; 732->735; 734->735; 735->736; 736->737; }

700,729

705,730

710,733

digraph G { subgraph clusterstage_496 { label="Stage 496"; subgraph cluster2369 { label="AQEShuffleRead"; 738 [label="ShuffledRowRDD [738]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2366 { label="WholeStageCodegen (13)"; 739 [label="MapPartitionsRDD [739]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2374 { label="AQEShuffleRead"; 740 [label="ShuffledRowRDD [740]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2371 { label="WholeStageCodegen (14)"; 741 [label="MapPartitionsRDD [741]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2362 { label="WholeStageCodegen (15)"; 742 [label="ZippedPartitionsRDD2 [742]\nsave at NativeMethodAccessorImpl.java:0"]; 743 [label="MapPartitionsRDD [743]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2361 { label="Exchange"; 744 [label="MapPartitionsRDD [744]\nsave at NativeMethodAccessorImpl.java:0"]; 745 [label="MapPartitionsRDD [745]\nsave at NativeMethodAccessorImpl.java:0"]; } } 738->739; 740->741; 739->742; 741->742; 742->743; 743->744; 744->745; }

737,738

728,740

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
496	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:25:29	1 s	822/822			499.2 MB	1791.1 MB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

496

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:25:29

1 s

822/822

499.2 MB

1791.1 MB

Skipped Stages (8)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
495	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
494	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
493	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
492	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/705
491	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
490	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
489	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/659
488	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433