paxus_etl - Details for Job 181

Details for Job 181

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 8

Enable zooming

digraph G { subgraph clusterstage_621 { label="Stage 621"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_635 { label="Stage 635 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2884 { label="InMemoryTableScan"; 891 [label="MapPartitionsRDD [891]\nsave at NativeMethodAccessorImpl.java:0"]; 892 [label="MapPartitionsRDD [892]\nsave at NativeMethodAccessorImpl.java:0"]; 893 [label="MapPartitionsRDD [893]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2879 { label="WholeStageCodegen (5)"; 894 [label="MapPartitionsRDD [894]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2878 { label="Exchange"; 895 [label="MapPartitionsRDD [895]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->891; 891->892; 892->893; 893->894; 894->895; }

56,57

digraph G { subgraph clusterstage_636 { label="Stage 636 (skipped)"; subgraph cluster2876 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 887 [label="FileScanRDD [887]\nsave at NativeMethodAccessorImpl.java:0"]; 888 [label="MapPartitionsRDD [888]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2872 { label="WholeStageCodegen (4)"; 889 [label="MapPartitionsRDD [889]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2871 { label="Exchange"; 890 [label="MapPartitionsRDD [890]\nsave at NativeMethodAccessorImpl.java:0"]; } } 887->888; 888->889; 889->890; }

digraph G { subgraph clusterstage_637 { label="Stage 637 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2863 { label="InMemoryTableScan"; 877 [label="MapPartitionsRDD [877]\nsave at NativeMethodAccessorImpl.java:0"]; 878 [label="MapPartitionsRDD [878]\nsave at NativeMethodAccessorImpl.java:0"]; 879 [label="MapPartitionsRDD [879]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2858 { label="WholeStageCodegen (2)"; 880 [label="MapPartitionsRDD [880]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2857 { label="Exchange"; 881 [label="MapPartitionsRDD [881]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->877; 877->878; 878->879; 879->880; 880->881; }

56,57

digraph G { subgraph clusterstage_638 { label="Stage 638 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2869 { label="InMemoryTableScan"; 882 [label="MapPartitionsRDD [882]\nsave at NativeMethodAccessorImpl.java:0"]; 883 [label="MapPartitionsRDD [883]\nsave at NativeMethodAccessorImpl.java:0"]; 884 [label="MapPartitionsRDD [884]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2866 { label="WholeStageCodegen (3)"; 885 [label="MapPartitionsRDD [885]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2865 { label="Exchange"; 886 [label="MapPartitionsRDD [886]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->882; 882->883; 883->884; 884->885; 885->886; }

56,57

digraph G { subgraph clusterstage_639 { label="Stage 639 (skipped)"; subgraph cluster2855 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.usagetable"; 873 [label="FileScanRDD [873]\nsave at NativeMethodAccessorImpl.java:0"]; 874 [label="MapPartitionsRDD [874]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2851 { label="WholeStageCodegen (1)"; 875 [label="MapPartitionsRDD [875]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2850 { label="Exchange"; 876 [label="MapPartitionsRDD [876]\nsave at NativeMethodAccessorImpl.java:0"]; } } 873->874; 874->875; 875->876; }

digraph G { subgraph clusterstage_640 { label="Stage 640 (skipped)"; subgraph cluster3035 { label="Exchange"; 968 [label="ShuffledRowRDD [968]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3038 { label="Exchange"; 969 [label="ShuffledRowRDD [969]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3026 { label="WholeStageCodegen (30)"; 970 [label="ZippedPartitionsRDD2 [970]\nsave at NativeMethodAccessorImpl.java:0"]; 971 [label="MapPartitionsRDD [971]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3044 { label="Exchange"; 972 [label="ShuffledRowRDD [972]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3040 { label="WholeStageCodegen (31)"; 973 [label="MapPartitionsRDD [973]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3023 { label="WholeStageCodegen (32)"; 974 [label="ZippedPartitionsRDD2 [974]\nsave at NativeMethodAccessorImpl.java:0"]; 975 [label="MapPartitionsRDD [975]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3022 { label="Exchange"; 976 [label="MapPartitionsRDD [976]\nsave at NativeMethodAccessorImpl.java:0"]; } } 968->970; 969->970; 970->971; 972->973; 971->974; 973->974; 974->975; 975->976; }

876,968

881,969

886,972

digraph G { subgraph clusterstage_641 { label="Stage 641 (skipped)"; subgraph cluster3235 { label="Exchange"; 1027 [label="ShuffledRowRDD [1027]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3238 { label="Exchange"; 1028 [label="ShuffledRowRDD [1028]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3226 { label="WholeStageCodegen (57)"; 1029 [label="ZippedPartitionsRDD2 [1029]\nsave at NativeMethodAccessorImpl.java:0"]; 1030 [label="MapPartitionsRDD [1030]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3225 { label="Window"; 1031 [label="MapPartitionsRDD [1031]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3221 { label="WholeStageCodegen (58)"; 1032 [label="MapPartitionsRDD [1032]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3220 { label="Exchange"; 1033 [label="MapPartitionsRDD [1033]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1027->1029; 1028->1029; 1029->1030; 1030->1031; 1031->1032; 1032->1033; }

890,1027

895,1028

digraph G { subgraph clusterstage_642 { label="Stage 642"; subgraph cluster3380 { label="AQEShuffleRead"; 1071 [label="ShuffledRowRDD [1071]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3377 { label="WholeStageCodegen (71)"; 1072 [label="MapPartitionsRDD [1072]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3385 { label="AQEShuffleRead"; 1073 [label="ShuffledRowRDD [1073]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3382 { label="WholeStageCodegen (72)"; 1074 [label="MapPartitionsRDD [1074]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3373 { label="WholeStageCodegen (73)"; 1075 [label="ZippedPartitionsRDD2 [1075]\nsave at NativeMethodAccessorImpl.java:0"]; 1076 [label="MapPartitionsRDD [1076]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3372 { label="Exchange"; 1077 [label="MapPartitionsRDD [1077]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1071->1072; 1073->1074; 1072->1075; 1074->1075; 1075->1076; 1076->1077; }

976,1071

1033,1073

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
642	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:28:39	1 s	780/780			210.0 MB	74.6 MB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

642

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:28:39

1 s

780/780

210.0 MB

74.6 MB

Skipped Stages (8)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
641	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
640	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
639	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/757
638	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
637	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
636	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
635	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
621	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433