mpu-usecase4 - Details for Job 7

Details for Job 7

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_9 { label="Stage 9 (skipped)"; subgraph cluster41 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_media_data"; 20 [label="FileScanRDD [20]\nsave at NativeMethodAccessorImpl.java:0"]; 21 [label="MapPartitionsRDD [21]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster37 { label="WholeStageCodegen (3)"; 22 [label="MapPartitionsRDD [22]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster46 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_media_data"; 23 [label="FileScanRDD [23]\nsave at NativeMethodAccessorImpl.java:0"]; 24 [label="MapPartitionsRDD [24]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster42 { label="WholeStageCodegen (4)"; 25 [label="MapPartitionsRDD [25]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster36 { label="Union"; 26 [label="UnionRDD [26]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster35 { label="Exchange"; 27 [label="MapPartitionsRDD [27]\nsave at NativeMethodAccessorImpl.java:0"]; } } 20->21; 21->22; 23->24; 24->25; 22->26; 25->26; 26->27; }

digraph G { subgraph clusterstage_10 { label="Stage 10 (skipped)"; subgraph cluster28 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_usage_data"; 12 [label="FileScanRDD [12]\nsave at NativeMethodAccessorImpl.java:0"]; 13 [label="MapPartitionsRDD [13]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster24 { label="WholeStageCodegen (1)"; 14 [label="MapPartitionsRDD [14]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster33 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_usage_data"; 15 [label="FileScanRDD [15]\nsave at NativeMethodAccessorImpl.java:0"]; 16 [label="MapPartitionsRDD [16]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster29 { label="WholeStageCodegen (2)"; 17 [label="MapPartitionsRDD [17]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster23 { label="Union"; 18 [label="UnionRDD [18]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster22 { label="Exchange"; 19 [label="MapPartitionsRDD [19]\nsave at NativeMethodAccessorImpl.java:0"]; } } 12->13; 13->14; 15->16; 16->17; 14->18; 17->18; 18->19; }

digraph G { subgraph clusterstage_11 { label="Stage 11 (skipped)"; subgraph cluster54 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_category_data"; 28 [label="FileScanRDD [28]\nsave at NativeMethodAccessorImpl.java:0"]; 29 [label="MapPartitionsRDD [29]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster50 { label="WholeStageCodegen (5)"; 30 [label="MapPartitionsRDD [30]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster59 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_category_data"; 31 [label="FileScanRDD [31]\nsave at NativeMethodAccessorImpl.java:0"]; 32 [label="MapPartitionsRDD [32]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster55 { label="WholeStageCodegen (6)"; 33 [label="MapPartitionsRDD [33]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster49 { label="Union"; 34 [label="UnionRDD [34]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster48 { label="Exchange"; 35 [label="MapPartitionsRDD [35]\nsave at NativeMethodAccessorImpl.java:0"]; } } 28->29; 29->30; 31->32; 32->33; 30->34; 33->34; 34->35; }

digraph G { subgraph clusterstage_12 { label="Stage 12 (skipped)"; subgraph cluster97 { label="Exchange"; 48 [label="ShuffledRowRDD [48]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster93 { label="WholeStageCodegen (10)"; 49 [label="MapPartitionsRDD [49]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster103 { label="Exchange"; 50 [label="ShuffledRowRDD [50]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster99 { label="WholeStageCodegen (11)"; 51 [label="MapPartitionsRDD [51]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster89 { label="WholeStageCodegen (12)"; 52 [label="ZippedPartitionsRDD2 [52]\nsave at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster88 { label="Exchange"; 54 [label="MapPartitionsRDD [54]\nsave at NativeMethodAccessorImpl.java:0"]; } } 48->49; 50->51; 49->52; 51->52; 52->53; 53->54; }

19,48

27,50

digraph G { subgraph clusterstage_13 { label="Stage 13"; subgraph cluster115 { label="Exchange"; 55 [label="ShuffledRowRDD [55]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster111 { label="WholeStageCodegen (13)"; 56 [label="MapPartitionsRDD [56]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster121 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster117 { label="WholeStageCodegen (14)"; 58 [label="MapPartitionsRDD [58]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster107 { label="WholeStageCodegen (15)"; 59 [label="ZippedPartitionsRDD2 [59]\nsave at NativeMethodAccessorImpl.java:0"]; 60 [label="MapPartitionsRDD [60]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster106 { label="Exchange"; 61 [label="MapPartitionsRDD [61]\nsave at NativeMethodAccessorImpl.java:0"]; } } 55->56; 57->58; 56->59; 58->59; 59->60; 60->61; }

54,55

35,57

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
13	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/05 04:37:09	10 s	600/600			5.8 GB	5.8 GB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/05 04:37:09

10 s

600/600

5.8 GB

Skipped Stages (4)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
12	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
11	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/445
10	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/826
9	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/807