paxus_etl - Details for Job 135

Details for Job 135

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 9

Enable zooming

digraph G { subgraph clusterstage_531 { label="Stage 531 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_532 { label="Stage 532 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2471 { label="InMemoryTableScan"; 767 [label="MapPartitionsRDD [767]\nsave at NativeMethodAccessorImpl.java:0"]; 768 [label="MapPartitionsRDD [768]\nsave at NativeMethodAccessorImpl.java:0"]; 769 [label="MapPartitionsRDD [769]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2466 { label="WholeStageCodegen (5)"; 770 [label="MapPartitionsRDD [770]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2465 { label="Exchange"; 771 [label="MapPartitionsRDD [771]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->767; 767->768; 768->769; 769->770; 770->771; }

56,57

digraph G { subgraph clusterstage_533 { label="Stage 533 (skipped)"; subgraph cluster2463 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.languagetable"; 763 [label="FileScanRDD [763]\nsave at NativeMethodAccessorImpl.java:0"]; 764 [label="MapPartitionsRDD [764]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2459 { label="WholeStageCodegen (4)"; 765 [label="MapPartitionsRDD [765]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2458 { label="Exchange"; 766 [label="MapPartitionsRDD [766]\nsave at NativeMethodAccessorImpl.java:0"]; } } 763->764; 764->765; 765->766; }

digraph G { subgraph clusterstage_534 { label="Stage 534 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2450 { label="InMemoryTableScan"; 753 [label="MapPartitionsRDD [753]\nsave at NativeMethodAccessorImpl.java:0"]; 754 [label="MapPartitionsRDD [754]\nsave at NativeMethodAccessorImpl.java:0"]; 755 [label="MapPartitionsRDD [755]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2445 { label="WholeStageCodegen (2)"; 756 [label="MapPartitionsRDD [756]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2444 { label="Exchange"; 757 [label="MapPartitionsRDD [757]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->753; 753->754; 754->755; 755->756; 756->757; }

56,57

digraph G { subgraph clusterstage_535 { label="Stage 535 (skipped)"; subgraph cluster2442 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.ccsubtitlestable"; 749 [label="FileScanRDD [749]\nsave at NativeMethodAccessorImpl.java:0"]; 750 [label="MapPartitionsRDD [750]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2438 { label="WholeStageCodegen (1)"; 751 [label="MapPartitionsRDD [751]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2437 { label="Exchange"; 752 [label="MapPartitionsRDD [752]\nsave at NativeMethodAccessorImpl.java:0"]; } } 749->750; 750->751; 751->752; }

digraph G { subgraph clusterstage_536 { label="Stage 536 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2456 { label="InMemoryTableScan"; 758 [label="MapPartitionsRDD [758]\nsave at NativeMethodAccessorImpl.java:0"]; 759 [label="MapPartitionsRDD [759]\nsave at NativeMethodAccessorImpl.java:0"]; 760 [label="MapPartitionsRDD [760]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2453 { label="WholeStageCodegen (3)"; 761 [label="MapPartitionsRDD [761]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2452 { label="Exchange"; 762 [label="MapPartitionsRDD [762]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->758; 758->759; 759->760; 760->761; 761->762; }

56,57

digraph G { subgraph clusterstage_537 { label="Stage 537 (skipped)"; subgraph cluster2492 { label="AQEShuffleRead"; 772 [label="ShuffledRowRDD [772]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2489 { label="WholeStageCodegen (6)"; 773 [label="MapPartitionsRDD [773]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2497 { label="AQEShuffleRead"; 774 [label="ShuffledRowRDD [774]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2494 { label="WholeStageCodegen (7)"; 775 [label="MapPartitionsRDD [775]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2482 { label="WholeStageCodegen (8)"; 776 [label="ZippedPartitionsRDD2 [776]\nsave at NativeMethodAccessorImpl.java:0"]; 777 [label="MapPartitionsRDD [777]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2502 { label="AQEShuffleRead"; 778 [label="ShuffledRowRDD [778]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2499 { label="WholeStageCodegen (9)"; 779 [label="MapPartitionsRDD [779]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2478 { label="WholeStageCodegen (10)"; 780 [label="ZippedPartitionsRDD2 [780]\nsave at NativeMethodAccessorImpl.java:0"]; 781 [label="MapPartitionsRDD [781]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2477 { label="Exchange"; 782 [label="MapPartitionsRDD [782]\nsave at NativeMethodAccessorImpl.java:0"]; } } 772->773; 774->775; 773->776; 775->776; 776->777; 778->779; 777->780; 779->780; 780->781; 781->782; }

752,772

757,774

762,778

digraph G { subgraph clusterstage_538 { label="Stage 538 (skipped)"; subgraph cluster2523 { label="Exchange"; 783 [label="ShuffledRowRDD [783]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2519 { label="WholeStageCodegen (11)"; 784 [label="MapPartitionsRDD [784]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2529 { label="Exchange"; 785 [label="ShuffledRowRDD [785]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2525 { label="WholeStageCodegen (12)"; 786 [label="MapPartitionsRDD [786]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2512 { label="WholeStageCodegen (13)"; 787 [label="ZippedPartitionsRDD2 [787]\nsave at NativeMethodAccessorImpl.java:0"]; 788 [label="MapPartitionsRDD [788]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2511 { label="Window"; 789 [label="MapPartitionsRDD [789]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2507 { label="WholeStageCodegen (14)"; 790 [label="MapPartitionsRDD [790]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2506 { label="Exchange"; 791 [label="MapPartitionsRDD [791]\nsave at NativeMethodAccessorImpl.java:0"]; } } 783->784; 785->786; 784->787; 786->787; 787->788; 788->789; 789->790; 790->791; }

766,783

771,785

digraph G { subgraph clusterstage_539 { label="Stage 539 (skipped)"; subgraph cluster2540 { label="AQEShuffleRead"; 792 [label="ShuffledRowRDD [792]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2537 { label="WholeStageCodegen (15)"; 793 [label="MapPartitionsRDD [793]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2545 { label="AQEShuffleRead"; 794 [label="ShuffledRowRDD [794]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2542 { label="WholeStageCodegen (16)"; 795 [label="MapPartitionsRDD [795]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2533 { label="WholeStageCodegen (17)"; 796 [label="ZippedPartitionsRDD2 [796]\nsave at NativeMethodAccessorImpl.java:0"]; 797 [label="MapPartitionsRDD [797]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2532 { label="Exchange"; 798 [label="MapPartitionsRDD [798]\nsave at NativeMethodAccessorImpl.java:0"]; 799 [label="MapPartitionsRDD [799]\nsave at NativeMethodAccessorImpl.java:0"]; } } 792->793; 794->795; 793->796; 795->796; 796->797; 797->798; 798->799; }

782,792

791,794

digraph G { subgraph clusterstage_540 { label="Stage 540"; subgraph cluster2548 { label="Exchange"; 800 [label="ShuffledRowRDD [800]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2434 { label="Project"; 801 [label="MapPartitionsRDD [801]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2433 { label="Sort"; 802 [label="MapPartitionsRDD [802]\nsave at NativeMethodAccessorImpl.java:0"]; } } 800->801; 801->802; }

799,800

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
540	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:26:20	0.6 s	10/10		605.5 KB	1132.0 KB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

540

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:26:20

0.6 s

10/10

605.5 KB

1132.0 KB

Skipped Stages (9)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
539	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/64
538	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
537	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/557
536	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
535	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/690
534	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
533	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/705
532	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
531	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433