mpu-usecase4 - Details for Job 11

Details for Job 11

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 10

Enable zooming

digraph G { subgraph clusterstage_40 { label="Stage 40 (skipped)"; subgraph cluster41 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_media_data"; 20 [label="FileScanRDD [20]\nsave at NativeMethodAccessorImpl.java:0"]; 21 [label="MapPartitionsRDD [21]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster37 { label="WholeStageCodegen (3)"; 22 [label="MapPartitionsRDD [22]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster46 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_media_data"; 23 [label="FileScanRDD [23]\nsave at NativeMethodAccessorImpl.java:0"]; 24 [label="MapPartitionsRDD [24]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster42 { label="WholeStageCodegen (4)"; 25 [label="MapPartitionsRDD [25]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster36 { label="Union"; 26 [label="UnionRDD [26]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster35 { label="Exchange"; 27 [label="MapPartitionsRDD [27]\nsave at NativeMethodAccessorImpl.java:0"]; } } 20->21; 21->22; 23->24; 24->25; 22->26; 25->26; 26->27; }

digraph G { subgraph clusterstage_41 { label="Stage 41 (skipped)"; subgraph cluster28 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_usage_data"; 12 [label="FileScanRDD [12]\nsave at NativeMethodAccessorImpl.java:0"]; 13 [label="MapPartitionsRDD [13]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster24 { label="WholeStageCodegen (1)"; 14 [label="MapPartitionsRDD [14]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster33 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_usage_data"; 15 [label="FileScanRDD [15]\nsave at NativeMethodAccessorImpl.java:0"]; 16 [label="MapPartitionsRDD [16]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster29 { label="WholeStageCodegen (2)"; 17 [label="MapPartitionsRDD [17]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster23 { label="Union"; 18 [label="UnionRDD [18]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster22 { label="Exchange"; 19 [label="MapPartitionsRDD [19]\nsave at NativeMethodAccessorImpl.java:0"]; } } 12->13; 13->14; 15->16; 16->17; 14->18; 17->18; 18->19; }

digraph G { subgraph clusterstage_42 { label="Stage 42 (skipped)"; subgraph cluster54 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_category_data"; 28 [label="FileScanRDD [28]\nsave at NativeMethodAccessorImpl.java:0"]; 29 [label="MapPartitionsRDD [29]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster50 { label="WholeStageCodegen (5)"; 30 [label="MapPartitionsRDD [30]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster59 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_category_data"; 31 [label="FileScanRDD [31]\nsave at NativeMethodAccessorImpl.java:0"]; 32 [label="MapPartitionsRDD [32]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster55 { label="WholeStageCodegen (6)"; 33 [label="MapPartitionsRDD [33]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster49 { label="Union"; 34 [label="UnionRDD [34]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster48 { label="Exchange"; 35 [label="MapPartitionsRDD [35]\nsave at NativeMethodAccessorImpl.java:0"]; } } 28->29; 29->30; 31->32; 32->33; 30->34; 33->34; 34->35; }

digraph G { subgraph clusterstage_43 { label="Stage 43 (skipped)"; subgraph cluster97 { label="Exchange"; 48 [label="ShuffledRowRDD [48]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster93 { label="WholeStageCodegen (10)"; 49 [label="MapPartitionsRDD [49]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster103 { label="Exchange"; 50 [label="ShuffledRowRDD [50]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster99 { label="WholeStageCodegen (11)"; 51 [label="MapPartitionsRDD [51]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster89 { label="WholeStageCodegen (12)"; 52 [label="ZippedPartitionsRDD2 [52]\nsave at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster88 { label="Exchange"; 54 [label="MapPartitionsRDD [54]\nsave at NativeMethodAccessorImpl.java:0"]; } } 48->49; 50->51; 49->52; 51->52; 52->53; 53->54; }

19,48

27,50

digraph G { subgraph clusterstage_44 { label="Stage 44 (skipped)"; subgraph cluster115 { label="Exchange"; 55 [label="ShuffledRowRDD [55]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster111 { label="WholeStageCodegen (13)"; 56 [label="MapPartitionsRDD [56]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster121 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster117 { label="WholeStageCodegen (14)"; 58 [label="MapPartitionsRDD [58]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster107 { label="WholeStageCodegen (15)"; 59 [label="ZippedPartitionsRDD2 [59]\nsave at NativeMethodAccessorImpl.java:0"]; 60 [label="MapPartitionsRDD [60]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster106 { label="Exchange"; 61 [label="MapPartitionsRDD [61]\nsave at NativeMethodAccessorImpl.java:0"]; } } 55->56; 57->58; 56->59; 58->59; 59->60; 60->61; }

54,55

35,57

digraph G { subgraph clusterstage_45 { label="Stage 45 (skipped)"; subgraph cluster68 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_category_data"; 36 [label="FileScanRDD [36]\nsave at NativeMethodAccessorImpl.java:0"]; 37 [label="MapPartitionsRDD [37]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster63 { label="WholeStageCodegen (7)"; 38 [label="MapPartitionsRDD [38]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster74 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_category_data"; 39 [label="FileScanRDD [39]\nsave at NativeMethodAccessorImpl.java:0"]; 40 [label="MapPartitionsRDD [40]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster69 { label="WholeStageCodegen (8)"; 41 [label="MapPartitionsRDD [41]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster62 { label="Union"; 42 [label="UnionRDD [42]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster61 { label="Exchange"; 43 [label="MapPartitionsRDD [43]\nsave at NativeMethodAccessorImpl.java:0"]; } } 36->37; 37->38; 39->40; 40->41; 38->42; 41->42; 42->43; }

digraph G { subgraph clusterstage_46 { label="Stage 46 (skipped)"; subgraph cluster81 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.mma_media_data"; 44 [label="FileScanRDD [44]\nsave at NativeMethodAccessorImpl.java:0"]; 45 [label="MapPartitionsRDD [45]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster77 { label="WholeStageCodegen (9)"; 46 [label="MapPartitionsRDD [46]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster76 { label="Exchange"; 47 [label="MapPartitionsRDD [47]\nsave at NativeMethodAccessorImpl.java:0"]; } } 44->45; 45->46; 46->47; }

digraph G { subgraph clusterstage_47 { label="Stage 47 (skipped)"; subgraph cluster133 { label="Exchange"; 62 [label="ShuffledRowRDD [62]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster129 { label="WholeStageCodegen (16)"; 63 [label="MapPartitionsRDD [63]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster139 { label="Exchange"; 64 [label="ShuffledRowRDD [64]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster135 { label="WholeStageCodegen (17)"; 65 [label="MapPartitionsRDD [65]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster125 { label="WholeStageCodegen (18)"; 66 [label="ZippedPartitionsRDD2 [66]\nsave at NativeMethodAccessorImpl.java:0"]; 67 [label="MapPartitionsRDD [67]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster124 { label="Exchange"; 68 [label="MapPartitionsRDD [68]\nsave at NativeMethodAccessorImpl.java:0"]; } } 62->63; 64->65; 63->66; 65->66; 66->67; 67->68; }

61,62

43,64

digraph G { subgraph clusterstage_48 { label="Stage 48 (skipped)"; subgraph cluster152 { label="AQEShuffleRead"; 69 [label="ShuffledRowRDD [69]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster149 { label="WholeStageCodegen (19)"; 70 [label="MapPartitionsRDD [70]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster157 { label="AQEShuffleRead"; 71 [label="ShuffledRowRDD [71]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster154 { label="WholeStageCodegen (20)"; 72 [label="MapPartitionsRDD [72]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster143 { label="WholeStageCodegen (21)"; 73 [label="ZippedPartitionsRDD2 [73]\nsave at NativeMethodAccessorImpl.java:0"]; 74 [label="MapPartitionsRDD [74]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster142 { label="Exchange"; 75 [label="MapPartitionsRDD [75]\nsave at NativeMethodAccessorImpl.java:0"]; 76 [label="MapPartitionsRDD [76]\nsave at NativeMethodAccessorImpl.java:0"]; } } 69->70; 71->72; 70->73; 72->73; 73->74; 74->75; 75->76; }

68,69

47,71

digraph G { subgraph clusterstage_49 { label="Stage 49 (skipped)"; subgraph cluster165 { label="Exchange"; 77 [label="ShuffledRowRDD [77]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster161 { label="WholeStageCodegen (22)"; 78 [label="MapPartitionsRDD [78]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster160 { label="Exchange"; 79 [label="MapPartitionsRDD [79]\nsave at NativeMethodAccessorImpl.java:0"]; } } 77->78; 78->79; }

76,77

digraph G { subgraph clusterstage_50 { label="Stage 50"; subgraph cluster171 { label="Exchange"; 80 [label="ShuffledRowRDD [80]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster169 { label="HashAggregate"; 81 [label="MapPartitionsRDD [81]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster168 { label="Exchange"; 82 [label="MapPartitionsRDD [82]\nsave at NativeMethodAccessorImpl.java:0"]; 83 [label="MapPartitionsRDD [83]\nsave at NativeMethodAccessorImpl.java:0"]; } } 80->81; 81->82; 82->83; }

79,80

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
50	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/05 09:18:02	7 s	600/600			28.1 GB	31.7 GB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/05 09:18:02

7 s

600/600

28.1 GB

31.7 GB

Skipped Stages (10)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
49	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/400
48	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/279
47	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
46	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/400
45	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/445
44	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
43	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
42	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/445
41	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/826
40	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/807