mpu-usecase4 - Details for Job 9

Details for Job 9

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 8

Enable zooming

digraph G { subgraph clusterstage_21 { label="Stage 21 (skipped)"; subgraph cluster41 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_media_data"; 20 [label="FileScanRDD [20]\nsave at NativeMethodAccessorImpl.java:0"]; 21 [label="MapPartitionsRDD [21]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster37 { label="WholeStageCodegen (3)"; 22 [label="MapPartitionsRDD [22]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster46 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_media_data"; 23 [label="FileScanRDD [23]\nsave at NativeMethodAccessorImpl.java:0"]; 24 [label="MapPartitionsRDD [24]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster42 { label="WholeStageCodegen (4)"; 25 [label="MapPartitionsRDD [25]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster36 { label="Union"; 26 [label="UnionRDD [26]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster35 { label="Exchange"; 27 [label="MapPartitionsRDD [27]\nsave at NativeMethodAccessorImpl.java:0"]; } } 20->21; 21->22; 23->24; 24->25; 22->26; 25->26; 26->27; }

digraph G { subgraph clusterstage_22 { label="Stage 22 (skipped)"; subgraph cluster28 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_usage_data"; 12 [label="FileScanRDD [12]\nsave at NativeMethodAccessorImpl.java:0"]; 13 [label="MapPartitionsRDD [13]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster24 { label="WholeStageCodegen (1)"; 14 [label="MapPartitionsRDD [14]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster33 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_usage_data"; 15 [label="FileScanRDD [15]\nsave at NativeMethodAccessorImpl.java:0"]; 16 [label="MapPartitionsRDD [16]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster29 { label="WholeStageCodegen (2)"; 17 [label="MapPartitionsRDD [17]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster23 { label="Union"; 18 [label="UnionRDD [18]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster22 { label="Exchange"; 19 [label="MapPartitionsRDD [19]\nsave at NativeMethodAccessorImpl.java:0"]; } } 12->13; 13->14; 15->16; 16->17; 14->18; 17->18; 18->19; }

digraph G { subgraph clusterstage_23 { label="Stage 23 (skipped)"; subgraph cluster54 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_category_data"; 28 [label="FileScanRDD [28]\nsave at NativeMethodAccessorImpl.java:0"]; 29 [label="MapPartitionsRDD [29]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster50 { label="WholeStageCodegen (5)"; 30 [label="MapPartitionsRDD [30]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster59 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_category_data"; 31 [label="FileScanRDD [31]\nsave at NativeMethodAccessorImpl.java:0"]; 32 [label="MapPartitionsRDD [32]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster55 { label="WholeStageCodegen (6)"; 33 [label="MapPartitionsRDD [33]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster49 { label="Union"; 34 [label="UnionRDD [34]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster48 { label="Exchange"; 35 [label="MapPartitionsRDD [35]\nsave at NativeMethodAccessorImpl.java:0"]; } } 28->29; 29->30; 31->32; 32->33; 30->34; 33->34; 34->35; }

digraph G { subgraph clusterstage_24 { label="Stage 24 (skipped)"; subgraph cluster97 { label="Exchange"; 48 [label="ShuffledRowRDD [48]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster93 { label="WholeStageCodegen (10)"; 49 [label="MapPartitionsRDD [49]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster103 { label="Exchange"; 50 [label="ShuffledRowRDD [50]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster99 { label="WholeStageCodegen (11)"; 51 [label="MapPartitionsRDD [51]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster89 { label="WholeStageCodegen (12)"; 52 [label="ZippedPartitionsRDD2 [52]\nsave at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster88 { label="Exchange"; 54 [label="MapPartitionsRDD [54]\nsave at NativeMethodAccessorImpl.java:0"]; } } 48->49; 50->51; 49->52; 51->52; 52->53; 53->54; }

19,48

27,50

digraph G { subgraph clusterstage_25 { label="Stage 25 (skipped)"; subgraph cluster115 { label="Exchange"; 55 [label="ShuffledRowRDD [55]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster111 { label="WholeStageCodegen (13)"; 56 [label="MapPartitionsRDD [56]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster121 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster117 { label="WholeStageCodegen (14)"; 58 [label="MapPartitionsRDD [58]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster107 { label="WholeStageCodegen (15)"; 59 [label="ZippedPartitionsRDD2 [59]\nsave at NativeMethodAccessorImpl.java:0"]; 60 [label="MapPartitionsRDD [60]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster106 { label="Exchange"; 61 [label="MapPartitionsRDD [61]\nsave at NativeMethodAccessorImpl.java:0"]; } } 55->56; 57->58; 56->59; 58->59; 59->60; 60->61; }

54,55

35,57

digraph G { subgraph clusterstage_26 { label="Stage 26 (skipped)"; subgraph cluster68 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_category_data"; 36 [label="FileScanRDD [36]\nsave at NativeMethodAccessorImpl.java:0"]; 37 [label="MapPartitionsRDD [37]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster63 { label="WholeStageCodegen (7)"; 38 [label="MapPartitionsRDD [38]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster74 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_category_data"; 39 [label="FileScanRDD [39]\nsave at NativeMethodAccessorImpl.java:0"]; 40 [label="MapPartitionsRDD [40]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster69 { label="WholeStageCodegen (8)"; 41 [label="MapPartitionsRDD [41]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster62 { label="Union"; 42 [label="UnionRDD [42]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster61 { label="Exchange"; 43 [label="MapPartitionsRDD [43]\nsave at NativeMethodAccessorImpl.java:0"]; } } 36->37; 37->38; 39->40; 40->41; 38->42; 41->42; 42->43; }

digraph G { subgraph clusterstage_27 { label="Stage 27 (skipped)"; subgraph cluster81 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.mma_media_data"; 44 [label="FileScanRDD [44]\nsave at NativeMethodAccessorImpl.java:0"]; 45 [label="MapPartitionsRDD [45]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster77 { label="WholeStageCodegen (9)"; 46 [label="MapPartitionsRDD [46]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster76 { label="Exchange"; 47 [label="MapPartitionsRDD [47]\nsave at NativeMethodAccessorImpl.java:0"]; } } 44->45; 45->46; 46->47; }

digraph G { subgraph clusterstage_28 { label="Stage 28 (skipped)"; subgraph cluster133 { label="Exchange"; 62 [label="ShuffledRowRDD [62]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster129 { label="WholeStageCodegen (16)"; 63 [label="MapPartitionsRDD [63]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster139 { label="Exchange"; 64 [label="ShuffledRowRDD [64]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster135 { label="WholeStageCodegen (17)"; 65 [label="MapPartitionsRDD [65]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster125 { label="WholeStageCodegen (18)"; 66 [label="ZippedPartitionsRDD2 [66]\nsave at NativeMethodAccessorImpl.java:0"]; 67 [label="MapPartitionsRDD [67]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster124 { label="Exchange"; 68 [label="MapPartitionsRDD [68]\nsave at NativeMethodAccessorImpl.java:0"]; } } 62->63; 64->65; 63->66; 65->66; 66->67; 67->68; }

61,62

43,64

digraph G { subgraph clusterstage_29 { label="Stage 29"; subgraph cluster152 { label="AQEShuffleRead"; 69 [label="ShuffledRowRDD [69]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster149 { label="WholeStageCodegen (19)"; 70 [label="MapPartitionsRDD [70]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster157 { label="AQEShuffleRead"; 71 [label="ShuffledRowRDD [71]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster154 { label="WholeStageCodegen (20)"; 72 [label="MapPartitionsRDD [72]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster143 { label="WholeStageCodegen (21)"; 73 [label="ZippedPartitionsRDD2 [73]\nsave at NativeMethodAccessorImpl.java:0"]; 74 [label="MapPartitionsRDD [74]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster142 { label="Exchange"; 75 [label="MapPartitionsRDD [75]\nsave at NativeMethodAccessorImpl.java:0"]; 76 [label="MapPartitionsRDD [76]\nsave at NativeMethodAccessorImpl.java:0"]; } } 69->70; 71->72; 70->73; 72->73; 73->74; 74->75; 75->76; }

68,69

47,71

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
29	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/05 04:37:30	4.7 h	279/279			5.8 GB	21.7 GB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/05 04:37:30

4.7 h

279/279

5.8 GB

21.7 GB

Skipped Stages (8)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
28	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
27	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/400
26	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/445
25	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
24	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
23	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/445
22	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/826
21	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/807