paxus_etl - Details for Job 146

Details for Job 146

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 11

Enable zooming

digraph G { subgraph clusterstage_566 { label="Stage 566 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_567 { label="Stage 567 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2653 { label="InMemoryTableScan"; 826 [label="MapPartitionsRDD [826]\nsave at NativeMethodAccessorImpl.java:0"]; 827 [label="MapPartitionsRDD [827]\nsave at NativeMethodAccessorImpl.java:0"]; 828 [label="MapPartitionsRDD [828]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2648 { label="WholeStageCodegen (7)"; 829 [label="MapPartitionsRDD [829]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2647 { label="Exchange"; 830 [label="MapPartitionsRDD [830]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->826; 826->827; 827->828; 828->829; 829->830; }

56,57

digraph G { subgraph clusterstage_568 { label="Stage 568 (skipped)"; subgraph cluster2636 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 817 [label="FileScanRDD [817]\nsave at NativeMethodAccessorImpl.java:0"]; 818 [label="MapPartitionsRDD [818]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2632 { label="WholeStageCodegen (4)"; 819 [label="MapPartitionsRDD [819]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2631 { label="Exchange"; 820 [label="MapPartitionsRDD [820]\nsave at NativeMethodAccessorImpl.java:0"]; } } 817->818; 818->819; 819->820; }

digraph G { subgraph clusterstage_569 { label="Stage 569 (skipped)"; subgraph cluster2714 { label="Exchange"; 844 [label="ShuffledRowRDD [844]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2738 { label="Exchange"; 851 [label="ShuffledRowRDD [851]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2725 { label="WholeStageCodegen (16)"; 852 [label="ZippedPartitionsRDD2 [852]\nsave at NativeMethodAccessorImpl.java:0"]; 853 [label="MapPartitionsRDD [853]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2724 { label="Window"; 854 [label="MapPartitionsRDD [854]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2720 { label="WholeStageCodegen (17)"; 855 [label="MapPartitionsRDD [855]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2719 { label="Exchange"; 856 [label="MapPartitionsRDD [856]\nsave at NativeMethodAccessorImpl.java:0"]; } } 844->852; 851->852; 852->853; 853->854; 854->855; 855->856; }

820,844

830,851

digraph G { subgraph clusterstage_570 { label="Stage 570 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2629 { label="InMemoryTableScan"; 812 [label="MapPartitionsRDD [812]\nsave at NativeMethodAccessorImpl.java:0"]; 813 [label="MapPartitionsRDD [813]\nsave at NativeMethodAccessorImpl.java:0"]; 814 [label="MapPartitionsRDD [814]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2626 { label="WholeStageCodegen (3)"; 815 [label="MapPartitionsRDD [815]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2625 { label="Exchange"; 816 [label="MapPartitionsRDD [816]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->812; 812->813; 813->814; 814->815; 815->816; }

56,57

digraph G { subgraph clusterstage_571 { label="Stage 571 (skipped)"; subgraph cluster2615 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryconfigtable"; 803 [label="FileScanRDD [803]\nsave at NativeMethodAccessorImpl.java:0"]; 804 [label="MapPartitionsRDD [804]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2611 { label="WholeStageCodegen (1)"; 805 [label="MapPartitionsRDD [805]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2610 { label="Exchange"; 806 [label="MapPartitionsRDD [806]\nsave at NativeMethodAccessorImpl.java:0"]; } } 803->804; 804->805; 805->806; }

digraph G { subgraph clusterstage_572 { label="Stage 572 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2623 { label="InMemoryTableScan"; 807 [label="MapPartitionsRDD [807]\nsave at NativeMethodAccessorImpl.java:0"]; 808 [label="MapPartitionsRDD [808]\nsave at NativeMethodAccessorImpl.java:0"]; 809 [label="MapPartitionsRDD [809]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2618 { label="WholeStageCodegen (2)"; 810 [label="MapPartitionsRDD [810]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2617 { label="Exchange"; 811 [label="MapPartitionsRDD [811]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->807; 807->808; 808->809; 809->810; 810->811; }

56,57

digraph G { subgraph clusterstage_573 { label="Stage 573 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2644 { label="InMemoryTableScan"; 821 [label="MapPartitionsRDD [821]\nsave at NativeMethodAccessorImpl.java:0"]; 822 [label="MapPartitionsRDD [822]\nsave at NativeMethodAccessorImpl.java:0"]; 823 [label="MapPartitionsRDD [823]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2639 { label="WholeStageCodegen (5)"; 824 [label="MapPartitionsRDD [824]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2638 { label="Exchange"; 825 [label="MapPartitionsRDD [825]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->821; 821->822; 822->823; 823->824; 824->825; }

56,57

digraph G { subgraph clusterstage_574 { label="Stage 574 (skipped)"; subgraph cluster2714 { label="Exchange"; 844 [label="ShuffledRowRDD [844]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2717 { label="Exchange"; 845 [label="ShuffledRowRDD [845]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2705 { label="WholeStageCodegen (14)"; 846 [label="ZippedPartitionsRDD2 [846]\nsave at NativeMethodAccessorImpl.java:0"]; 847 [label="MapPartitionsRDD [847]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2704 { label="Window"; 848 [label="MapPartitionsRDD [848]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2700 { label="WholeStageCodegen (15)"; 849 [label="MapPartitionsRDD [849]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2699 { label="Exchange"; 850 [label="MapPartitionsRDD [850]\nsave at NativeMethodAccessorImpl.java:0"]; } } 844->846; 845->846; 846->847; 847->848; 848->849; 849->850; }

820,844

825,845

digraph G { subgraph clusterstage_575 { label="Stage 575 (skipped)"; subgraph cluster2682 { label="Exchange"; 831 [label="ShuffledRowRDD [831]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2678 { label="WholeStageCodegen (8)"; 832 [label="MapPartitionsRDD [832]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2688 { label="Exchange"; 833 [label="ShuffledRowRDD [833]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2684 { label="WholeStageCodegen (9)"; 834 [label="MapPartitionsRDD [834]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2671 { label="WholeStageCodegen (10)"; 835 [label="ZippedPartitionsRDD2 [835]\nsave at NativeMethodAccessorImpl.java:0"]; 836 [label="MapPartitionsRDD [836]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2670 { label="Window"; 837 [label="MapPartitionsRDD [837]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2665 { label="WholeStageCodegen (11)"; 838 [label="MapPartitionsRDD [838]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2694 { label="Exchange"; 839 [label="ShuffledRowRDD [839]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2690 { label="WholeStageCodegen (12)"; 840 [label="MapPartitionsRDD [840]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2661 { label="WholeStageCodegen (13)"; 841 [label="ZippedPartitionsRDD2 [841]\nsave at NativeMethodAccessorImpl.java:0"]; 842 [label="MapPartitionsRDD [842]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2660 { label="Exchange"; 843 [label="MapPartitionsRDD [843]\nsave at NativeMethodAccessorImpl.java:0"]; } } 831->832; 833->834; 832->835; 834->835; 835->836; 836->837; 837->838; 839->840; 838->841; 840->841; 841->842; 842->843; }

806,831

811,833

816,839

digraph G { subgraph clusterstage_576 { label="Stage 576 (skipped)"; subgraph cluster2749 { label="AQEShuffleRead"; 857 [label="ShuffledRowRDD [857]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2746 { label="WholeStageCodegen (18)"; 858 [label="MapPartitionsRDD [858]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2754 { label="AQEShuffleRead"; 859 [label="ShuffledRowRDD [859]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2751 { label="WholeStageCodegen (19)"; 860 [label="MapPartitionsRDD [860]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2742 { label="WholeStageCodegen (20)"; 861 [label="ZippedPartitionsRDD2 [861]\nsave at NativeMethodAccessorImpl.java:0"]; 862 [label="MapPartitionsRDD [862]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2741 { label="Exchange"; 863 [label="MapPartitionsRDD [863]\nsave at NativeMethodAccessorImpl.java:0"]; } } 857->858; 859->860; 858->861; 860->861; 861->862; 862->863; }

843,857

850,859

digraph G { subgraph clusterstage_577 { label="Stage 577"; subgraph cluster2765 { label="AQEShuffleRead"; 864 [label="ShuffledRowRDD [864]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2762 { label="WholeStageCodegen (21)"; 865 [label="MapPartitionsRDD [865]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2770 { label="AQEShuffleRead"; 866 [label="ShuffledRowRDD [866]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2767 { label="WholeStageCodegen (22)"; 867 [label="MapPartitionsRDD [867]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2758 { label="WholeStageCodegen (23)"; 868 [label="ZippedPartitionsRDD2 [868]\nsave at NativeMethodAccessorImpl.java:0"]; 869 [label="MapPartitionsRDD [869]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2757 { label="Coalesce"; 870 [label="CoalescedRDD [870]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2607 { label="Project"; 871 [label="MapPartitionsRDD [871]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2606 { label="Sort"; 872 [label="MapPartitionsRDD [872]\nsave at NativeMethodAccessorImpl.java:0"]; } } 864->865; 866->867; 865->868; 867->868; 868->869; 869->870; 870->871; 871->872; }

863,864

856,866

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
577	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:26:46	5 s	50/50		20.7 MB	311.7 MB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

577

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:26:46

5 s

50/50

20.7 MB

311.7 MB

Skipped Stages (11)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
576	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/333
575	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
574	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
573	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
572	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
571	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
570	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
569	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
568	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
567	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
566	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433