mpu-usecase4 - Details for Job 8

Details for Job 8

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 6

Enable zooming

digraph G { subgraph clusterstage_14 { label="Stage 14 (skipped)"; subgraph cluster41 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_media_data"; 20 [label="FileScanRDD [20]\nsave at NativeMethodAccessorImpl.java:0"]; 21 [label="MapPartitionsRDD [21]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster37 { label="WholeStageCodegen (3)"; 22 [label="MapPartitionsRDD [22]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster46 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_media_data"; 23 [label="FileScanRDD [23]\nsave at NativeMethodAccessorImpl.java:0"]; 24 [label="MapPartitionsRDD [24]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster42 { label="WholeStageCodegen (4)"; 25 [label="MapPartitionsRDD [25]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster36 { label="Union"; 26 [label="UnionRDD [26]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster35 { label="Exchange"; 27 [label="MapPartitionsRDD [27]\nsave at NativeMethodAccessorImpl.java:0"]; } } 20->21; 21->22; 23->24; 24->25; 22->26; 25->26; 26->27; }

digraph G { subgraph clusterstage_15 { label="Stage 15 (skipped)"; subgraph cluster28 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_usage_data"; 12 [label="FileScanRDD [12]\nsave at NativeMethodAccessorImpl.java:0"]; 13 [label="MapPartitionsRDD [13]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster24 { label="WholeStageCodegen (1)"; 14 [label="MapPartitionsRDD [14]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster33 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_usage_data"; 15 [label="FileScanRDD [15]\nsave at NativeMethodAccessorImpl.java:0"]; 16 [label="MapPartitionsRDD [16]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster29 { label="WholeStageCodegen (2)"; 17 [label="MapPartitionsRDD [17]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster23 { label="Union"; 18 [label="UnionRDD [18]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster22 { label="Exchange"; 19 [label="MapPartitionsRDD [19]\nsave at NativeMethodAccessorImpl.java:0"]; } } 12->13; 13->14; 15->16; 16->17; 14->18; 17->18; 18->19; }

digraph G { subgraph clusterstage_16 { label="Stage 16 (skipped)"; subgraph cluster54 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_category_data"; 28 [label="FileScanRDD [28]\nsave at NativeMethodAccessorImpl.java:0"]; 29 [label="MapPartitionsRDD [29]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster50 { label="WholeStageCodegen (5)"; 30 [label="MapPartitionsRDD [30]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster59 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_category_data"; 31 [label="FileScanRDD [31]\nsave at NativeMethodAccessorImpl.java:0"]; 32 [label="MapPartitionsRDD [32]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster55 { label="WholeStageCodegen (6)"; 33 [label="MapPartitionsRDD [33]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster49 { label="Union"; 34 [label="UnionRDD [34]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster48 { label="Exchange"; 35 [label="MapPartitionsRDD [35]\nsave at NativeMethodAccessorImpl.java:0"]; } } 28->29; 29->30; 31->32; 32->33; 30->34; 33->34; 34->35; }

digraph G { subgraph clusterstage_17 { label="Stage 17 (skipped)"; subgraph cluster97 { label="Exchange"; 48 [label="ShuffledRowRDD [48]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster93 { label="WholeStageCodegen (10)"; 49 [label="MapPartitionsRDD [49]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster103 { label="Exchange"; 50 [label="ShuffledRowRDD [50]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster99 { label="WholeStageCodegen (11)"; 51 [label="MapPartitionsRDD [51]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster89 { label="WholeStageCodegen (12)"; 52 [label="ZippedPartitionsRDD2 [52]\nsave at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster88 { label="Exchange"; 54 [label="MapPartitionsRDD [54]\nsave at NativeMethodAccessorImpl.java:0"]; } } 48->49; 50->51; 49->52; 51->52; 52->53; 53->54; }

19,48

27,50

digraph G { subgraph clusterstage_18 { label="Stage 18 (skipped)"; subgraph cluster115 { label="Exchange"; 55 [label="ShuffledRowRDD [55]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster111 { label="WholeStageCodegen (13)"; 56 [label="MapPartitionsRDD [56]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster121 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster117 { label="WholeStageCodegen (14)"; 58 [label="MapPartitionsRDD [58]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster107 { label="WholeStageCodegen (15)"; 59 [label="ZippedPartitionsRDD2 [59]\nsave at NativeMethodAccessorImpl.java:0"]; 60 [label="MapPartitionsRDD [60]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster106 { label="Exchange"; 61 [label="MapPartitionsRDD [61]\nsave at NativeMethodAccessorImpl.java:0"]; } } 55->56; 57->58; 56->59; 58->59; 59->60; 60->61; }

54,55

35,57

digraph G { subgraph clusterstage_19 { label="Stage 19 (skipped)"; subgraph cluster68 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_category_data"; 36 [label="FileScanRDD [36]\nsave at NativeMethodAccessorImpl.java:0"]; 37 [label="MapPartitionsRDD [37]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster63 { label="WholeStageCodegen (7)"; 38 [label="MapPartitionsRDD [38]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster74 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_category_data"; 39 [label="FileScanRDD [39]\nsave at NativeMethodAccessorImpl.java:0"]; 40 [label="MapPartitionsRDD [40]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster69 { label="WholeStageCodegen (8)"; 41 [label="MapPartitionsRDD [41]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster62 { label="Union"; 42 [label="UnionRDD [42]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster61 { label="Exchange"; 43 [label="MapPartitionsRDD [43]\nsave at NativeMethodAccessorImpl.java:0"]; } } 36->37; 37->38; 39->40; 40->41; 38->42; 41->42; 42->43; }

digraph G { subgraph clusterstage_20 { label="Stage 20"; subgraph cluster133 { label="Exchange"; 62 [label="ShuffledRowRDD [62]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster129 { label="WholeStageCodegen (16)"; 63 [label="MapPartitionsRDD [63]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster139 { label="Exchange"; 64 [label="ShuffledRowRDD [64]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster135 { label="WholeStageCodegen (17)"; 65 [label="MapPartitionsRDD [65]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster125 { label="WholeStageCodegen (18)"; 66 [label="ZippedPartitionsRDD2 [66]\nsave at NativeMethodAccessorImpl.java:0"]; 67 [label="MapPartitionsRDD [67]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster124 { label="Exchange"; 68 [label="MapPartitionsRDD [68]\nsave at NativeMethodAccessorImpl.java:0"]; } } 62->63; 64->65; 63->66; 65->66; 66->67; 67->68; }

61,62

43,64

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
20	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/05 04:37:20	10 s	600/600			5.9 GB	5.5 GB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/05 04:37:20

10 s

600/600

5.9 GB

5.5 GB

Skipped Stages (6)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
19	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/445
18	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
17	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
16	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/445
15	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/826
14	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/807