mpu-usecase4 - Details for Job 6

Details for Job 6

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 2

Enable zooming

digraph G { subgraph clusterstage_6 { label="Stage 6 (skipped)"; subgraph cluster41 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_media_data"; 20 [label="FileScanRDD [20]\nsave at NativeMethodAccessorImpl.java:0"]; 21 [label="MapPartitionsRDD [21]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster37 { label="WholeStageCodegen (3)"; 22 [label="MapPartitionsRDD [22]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster46 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_media_data"; 23 [label="FileScanRDD [23]\nsave at NativeMethodAccessorImpl.java:0"]; 24 [label="MapPartitionsRDD [24]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster42 { label="WholeStageCodegen (4)"; 25 [label="MapPartitionsRDD [25]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster36 { label="Union"; 26 [label="UnionRDD [26]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster35 { label="Exchange"; 27 [label="MapPartitionsRDD [27]\nsave at NativeMethodAccessorImpl.java:0"]; } } 20->21; 21->22; 23->24; 24->25; 22->26; 25->26; 26->27; }

digraph G { subgraph clusterstage_7 { label="Stage 7 (skipped)"; subgraph cluster28 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_usage_data"; 12 [label="FileScanRDD [12]\nsave at NativeMethodAccessorImpl.java:0"]; 13 [label="MapPartitionsRDD [13]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster24 { label="WholeStageCodegen (1)"; 14 [label="MapPartitionsRDD [14]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster33 { label="Scan parquet pac_dp_datapipeline_dev_database_v2.offload_rack_usage_data"; 15 [label="FileScanRDD [15]\nsave at NativeMethodAccessorImpl.java:0"]; 16 [label="MapPartitionsRDD [16]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster29 { label="WholeStageCodegen (2)"; 17 [label="MapPartitionsRDD [17]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster23 { label="Union"; 18 [label="UnionRDD [18]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster22 { label="Exchange"; 19 [label="MapPartitionsRDD [19]\nsave at NativeMethodAccessorImpl.java:0"]; } } 12->13; 13->14; 15->16; 16->17; 14->18; 17->18; 18->19; }

digraph G { subgraph clusterstage_8 { label="Stage 8"; subgraph cluster97 { label="Exchange"; 48 [label="ShuffledRowRDD [48]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster93 { label="WholeStageCodegen (10)"; 49 [label="MapPartitionsRDD [49]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster103 { label="Exchange"; 50 [label="ShuffledRowRDD [50]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster99 { label="WholeStageCodegen (11)"; 51 [label="MapPartitionsRDD [51]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster89 { label="WholeStageCodegen (12)"; 52 [label="ZippedPartitionsRDD2 [52]\nsave at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster88 { label="Exchange"; 54 [label="MapPartitionsRDD [54]\nsave at NativeMethodAccessorImpl.java:0"]; } } 48->49; 50->51; 49->52; 51->52; 52->53; 53->54; }

19,48

27,50

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
8	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/05 04:18:00	19 min	600/600			17.6 GB	5.7 GB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/05 04:18:00

19 min

600/600

17.6 GB

5.7 GB

Skipped Stages (2)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
7	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/826
6	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/807

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

Unknown

0/826

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

Unknown

0/807