paxus_etl - Details for Job 183

Details for Job 183

Status: SUCCEEDED
Completed Stages: 1
Skipped Stages: 16

Enable zooming

digraph G { subgraph clusterstage_656 { label="Stage 656 (skipped)"; subgraph cluster120 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta"; 48 [label="FileScanRDD [48]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 49 [label="MapPartitionsRDD [49]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster116 { label="WholeStageCodegen (1)"; 50 [label="MapPartitionsRDD [50]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster115 { label="InMemoryTableScan"; 51 [label="MapPartitionsRDD [51]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 52 [label="*(1) Filter ((isnotnull(_ingest_timestamp#86) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) >= 20230418)) AND (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as int) <= 20230418))\n+- *(1) ColumnarToRow\n +- FileScan parquet dl_pac_ife_paxus_raw_normalized_prod_v1._meta[_ingest_dbid#83,_ingest_error_code#84,_ingest_input_s3uri#85,_ingest_timestamp#86,_intake_operator#87,_is_survey#88,_metafile_dttm#89,_table_count#90L,airplane_type#91,application_name#92,cmip#93,customer_name#94,flightarrival#95,flightdeparture#96,flightenddate#97,flightguid#98,flightnum#99,flightstartdate#100,icao_airline#101,metadata_info#102,multiple#103,offload_filename#104,offload_format_version#105,offload_type#106,... 7 more fields] Batched: true, DataFilters: [isnotnull(_ingest_timestamp#86), (cast(replace(substring(_ingest_timestamp#86, 1, 10), -, ) as i..., Format: Parquet, Location: InMemoryFileIndex(3788 paths)[s3://pac-dl-dst-ea5dd107-3e06-475b-9535-44a3016830f5/e9bb1a8a-c7d3-..., PartitionF... [52] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 53 [label="MapPartitionsRDD [53]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 54 [label="MapPartitionsRDD [54]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 55 [label="MapPartitionsRDD [55]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster114 { label="Exchange"; 56 [label="MapPartitionsRDD [56]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } } 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; 54->55; 55->56; }

digraph G { subgraph clusterstage_657 { label="Stage 657 (skipped)"; subgraph cluster2891 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryconfigtable"; 896 [label="FileScanRDD [896]\nsave at NativeMethodAccessorImpl.java:0"]; 897 [label="MapPartitionsRDD [897]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2887 { label="WholeStageCodegen (6)"; 898 [label="MapPartitionsRDD [898]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2886 { label="Exchange"; 899 [label="MapPartitionsRDD [899]\nsave at NativeMethodAccessorImpl.java:0"]; } } 896->897; 897->898; 898->899; }

digraph G { subgraph clusterstage_658 { label="Stage 658 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2899 { label="InMemoryTableScan"; 900 [label="MapPartitionsRDD [900]\nsave at NativeMethodAccessorImpl.java:0"]; 901 [label="MapPartitionsRDD [901]\nsave at NativeMethodAccessorImpl.java:0"]; 902 [label="MapPartitionsRDD [902]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2894 { label="WholeStageCodegen (7)"; 903 [label="MapPartitionsRDD [903]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2893 { label="Exchange"; 904 [label="MapPartitionsRDD [904]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->900; 900->901; 901->902; 902->903; 903->904; }

56,57

digraph G { subgraph clusterstage_659 { label="Stage 659 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2884 { label="InMemoryTableScan"; 891 [label="MapPartitionsRDD [891]\nsave at NativeMethodAccessorImpl.java:0"]; 892 [label="MapPartitionsRDD [892]\nsave at NativeMethodAccessorImpl.java:0"]; 893 [label="MapPartitionsRDD [893]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2879 { label="WholeStageCodegen (5)"; 894 [label="MapPartitionsRDD [894]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2878 { label="Exchange"; 895 [label="MapPartitionsRDD [895]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->891; 891->892; 892->893; 893->894; 894->895; }

56,57

digraph G { subgraph clusterstage_660 { label="Stage 660 (skipped)"; subgraph cluster2876 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 887 [label="FileScanRDD [887]\nsave at NativeMethodAccessorImpl.java:0"]; 888 [label="MapPartitionsRDD [888]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2872 { label="WholeStageCodegen (4)"; 889 [label="MapPartitionsRDD [889]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2871 { label="Exchange"; 890 [label="MapPartitionsRDD [890]\nsave at NativeMethodAccessorImpl.java:0"]; } } 887->888; 888->889; 889->890; }

digraph G { subgraph clusterstage_661 { label="Stage 661 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2863 { label="InMemoryTableScan"; 877 [label="MapPartitionsRDD [877]\nsave at NativeMethodAccessorImpl.java:0"]; 878 [label="MapPartitionsRDD [878]\nsave at NativeMethodAccessorImpl.java:0"]; 879 [label="MapPartitionsRDD [879]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2858 { label="WholeStageCodegen (2)"; 880 [label="MapPartitionsRDD [880]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2857 { label="Exchange"; 881 [label="MapPartitionsRDD [881]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->877; 877->878; 878->879; 879->880; 880->881; }

56,57

digraph G { subgraph clusterstage_662 { label="Stage 662 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2869 { label="InMemoryTableScan"; 882 [label="MapPartitionsRDD [882]\nsave at NativeMethodAccessorImpl.java:0"]; 883 [label="MapPartitionsRDD [883]\nsave at NativeMethodAccessorImpl.java:0"]; 884 [label="MapPartitionsRDD [884]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2866 { label="WholeStageCodegen (3)"; 885 [label="MapPartitionsRDD [885]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2865 { label="Exchange"; 886 [label="MapPartitionsRDD [886]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->882; 882->883; 883->884; 884->885; 885->886; }

56,57

digraph G { subgraph clusterstage_663 { label="Stage 663 (skipped)"; subgraph cluster2855 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.usagetable"; 873 [label="FileScanRDD [873]\nsave at NativeMethodAccessorImpl.java:0"]; 874 [label="MapPartitionsRDD [874]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2851 { label="WholeStageCodegen (1)"; 875 [label="MapPartitionsRDD [875]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2850 { label="Exchange"; 876 [label="MapPartitionsRDD [876]\nsave at NativeMethodAccessorImpl.java:0"]; } } 873->874; 874->875; 875->876; }

digraph G { subgraph clusterstage_664 { label="Stage 664 (skipped)"; subgraph cluster3035 { label="Exchange"; 968 [label="ShuffledRowRDD [968]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3038 { label="Exchange"; 969 [label="ShuffledRowRDD [969]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3026 { label="WholeStageCodegen (30)"; 970 [label="ZippedPartitionsRDD2 [970]\nsave at NativeMethodAccessorImpl.java:0"]; 971 [label="MapPartitionsRDD [971]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3044 { label="Exchange"; 972 [label="ShuffledRowRDD [972]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3040 { label="WholeStageCodegen (31)"; 973 [label="MapPartitionsRDD [973]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3023 { label="WholeStageCodegen (32)"; 974 [label="ZippedPartitionsRDD2 [974]\nsave at NativeMethodAccessorImpl.java:0"]; 975 [label="MapPartitionsRDD [975]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3022 { label="Exchange"; 976 [label="MapPartitionsRDD [976]\nsave at NativeMethodAccessorImpl.java:0"]; } } 968->970; 969->970; 970->971; 972->973; 971->974; 973->974; 974->975; 975->976; }

876,968

881,969

886,972

digraph G { subgraph clusterstage_665 { label="Stage 665 (skipped)"; subgraph cluster3235 { label="Exchange"; 1027 [label="ShuffledRowRDD [1027]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3238 { label="Exchange"; 1028 [label="ShuffledRowRDD [1028]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3226 { label="WholeStageCodegen (57)"; 1029 [label="ZippedPartitionsRDD2 [1029]\nsave at NativeMethodAccessorImpl.java:0"]; 1030 [label="MapPartitionsRDD [1030]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3225 { label="Window"; 1031 [label="MapPartitionsRDD [1031]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3221 { label="WholeStageCodegen (58)"; 1032 [label="MapPartitionsRDD [1032]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3220 { label="Exchange"; 1033 [label="MapPartitionsRDD [1033]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1027->1029; 1028->1029; 1029->1030; 1030->1031; 1031->1032; 1032->1033; }

890,1027

895,1028

digraph G { subgraph clusterstage_666 { label="Stage 666 (skipped)"; subgraph cluster3380 { label="AQEShuffleRead"; 1071 [label="ShuffledRowRDD [1071]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3377 { label="WholeStageCodegen (71)"; 1072 [label="MapPartitionsRDD [1072]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3385 { label="AQEShuffleRead"; 1073 [label="ShuffledRowRDD [1073]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3382 { label="WholeStageCodegen (72)"; 1074 [label="MapPartitionsRDD [1074]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3373 { label="WholeStageCodegen (73)"; 1075 [label="ZippedPartitionsRDD2 [1075]\nsave at NativeMethodAccessorImpl.java:0"]; 1076 [label="MapPartitionsRDD [1076]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3372 { label="Exchange"; 1077 [label="MapPartitionsRDD [1077]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1071->1072; 1073->1074; 1072->1075; 1074->1075; 1075->1076; 1076->1077; }

976,1071

1033,1073

digraph G { subgraph clusterstage_667 { label="Stage 667 (skipped)"; subgraph cluster3072 { label="Exchange"; 977 [label="ShuffledRowRDD [977]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3068 { label="WholeStageCodegen (33)"; 978 [label="MapPartitionsRDD [978]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3078 { label="Exchange"; 979 [label="ShuffledRowRDD [979]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3074 { label="WholeStageCodegen (34)"; 980 [label="MapPartitionsRDD [980]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3061 { label="WholeStageCodegen (35)"; 981 [label="ZippedPartitionsRDD2 [981]\nsave at NativeMethodAccessorImpl.java:0"]; 982 [label="MapPartitionsRDD [982]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3060 { label="Window"; 983 [label="MapPartitionsRDD [983]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3056 { label="WholeStageCodegen (36)"; 984 [label="MapPartitionsRDD [984]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3055 { label="Exchange"; 985 [label="MapPartitionsRDD [985]\nsave at NativeMethodAccessorImpl.java:0"]; } } 977->978; 979->980; 978->981; 980->981; 981->982; 982->983; 983->984; 984->985; }

899,977

904,979

digraph G { subgraph clusterstage_668 { label="Stage 668 (skipped)"; subgraph cluster2906 { label="Scan parquet dl_pac_ife_paxus_raw_normalized_prod_v1.categoryinfotable"; 905 [label="FileScanRDD [905]\nsave at NativeMethodAccessorImpl.java:0"]; 906 [label="MapPartitionsRDD [906]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2902 { label="WholeStageCodegen (8)"; 907 [label="MapPartitionsRDD [907]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2901 { label="Exchange"; 908 [label="MapPartitionsRDD [908]\nsave at NativeMethodAccessorImpl.java:0"]; } } 905->906; 906->907; 907->908; }

digraph G { subgraph clusterstage_669 { label="Stage 669 (skipped)"; subgraph cluster114 { label="Exchange"; 57 [label="ShuffledRowRDD [57]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="InMemoryTableScan"; 58 [label="MapPartitionsRDD [58]\njavaToPython at NativeMethodAccessorImpl.java:0"]; 59 [label="Exchange SinglePartition, REPARTITION_BY_NUM, [id=#243]\n+- InMemoryTableScan [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#106, ... 7 more fields]\n +- InMemoryRelation [_ingest_dbid#83, _ingest_error_code#84, _ingest_input_s3uri#85, _ingest_timestamp#86, _intake_operator#87, _is_survey#88, _metafile_dttm#89, _table_count#90L, airplane_type#91, application_name#92, cmip#93, customer_name#94, flightarrival#95, flightdeparture#96, flightenddate#97, flightguid#98, flightnum#99, flightstartdate#100, icao_airline#101, metadata_info#102, multiple#103, offload_filename#104, offload_format_version#105, offload_type#10... [59] [Cached]\njavaToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2914 { label="InMemoryTableScan"; 909 [label="MapPartitionsRDD [909]\nsave at NativeMethodAccessorImpl.java:0"]; 910 [label="MapPartitionsRDD [910]\nsave at NativeMethodAccessorImpl.java:0"]; 911 [label="MapPartitionsRDD [911]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2909 { label="WholeStageCodegen (9)"; 912 [label="MapPartitionsRDD [912]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2908 { label="Exchange"; 913 [label="MapPartitionsRDD [913]\nsave at NativeMethodAccessorImpl.java:0"]; } } 57->58; 58->59; 59->909; 909->910; 910->911; 911->912; 912->913; }

56,57

digraph G { subgraph clusterstage_670 { label="Stage 670 (skipped)"; subgraph cluster3261 { label="Exchange"; 1034 [label="ShuffledRowRDD [1034]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3264 { label="Exchange"; 1035 [label="ShuffledRowRDD [1035]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3252 { label="WholeStageCodegen (59)"; 1036 [label="ZippedPartitionsRDD2 [1036]\nsave at NativeMethodAccessorImpl.java:0"]; 1037 [label="MapPartitionsRDD [1037]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3251 { label="Window"; 1038 [label="MapPartitionsRDD [1038]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3247 { label="WholeStageCodegen (60)"; 1039 [label="MapPartitionsRDD [1039]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3246 { label="Exchange"; 1040 [label="MapPartitionsRDD [1040]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1034->1036; 1035->1036; 1036->1037; 1037->1038; 1038->1039; 1039->1040; }

908,1034

913,1035

digraph G { subgraph clusterstage_671 { label="Stage 671 (skipped)"; subgraph cluster3402 { label="AQEShuffleRead"; 1078 [label="ShuffledRowRDD [1078]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3399 { label="WholeStageCodegen (74)"; 1079 [label="MapPartitionsRDD [1079]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3407 { label="AQEShuffleRead"; 1080 [label="ShuffledRowRDD [1080]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3404 { label="WholeStageCodegen (75)"; 1081 [label="MapPartitionsRDD [1081]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3395 { label="WholeStageCodegen (76)"; 1082 [label="ZippedPartitionsRDD2 [1082]\nsave at NativeMethodAccessorImpl.java:0"]; 1083 [label="MapPartitionsRDD [1083]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3394 { label="Exchange"; 1084 [label="MapPartitionsRDD [1084]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1078->1079; 1080->1081; 1079->1082; 1081->1082; 1082->1083; 1083->1084; }

1077,1078

985,1080

digraph G { subgraph clusterstage_672 { label="Stage 672"; subgraph cluster3418 { label="AQEShuffleRead"; 1085 [label="ShuffledRowRDD [1085]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3415 { label="WholeStageCodegen (77)"; 1086 [label="MapPartitionsRDD [1086]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3423 { label="AQEShuffleRead"; 1087 [label="ShuffledRowRDD [1087]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3420 { label="WholeStageCodegen (78)"; 1088 [label="MapPartitionsRDD [1088]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3411 { label="WholeStageCodegen (79)"; 1089 [label="ZippedPartitionsRDD2 [1089]\nsave at NativeMethodAccessorImpl.java:0"]; 1090 [label="MapPartitionsRDD [1090]\nsave at NativeMethodAccessorImpl.java:0"]; } subgraph cluster3410 { label="Exchange"; 1091 [label="MapPartitionsRDD [1091]\nsave at NativeMethodAccessorImpl.java:0"]; } } 1085->1086; 1087->1088; 1086->1089; 1088->1089; 1089->1090; 1090->1091; }

1084,1085

1040,1087

Completed Stages (1)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
672	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	2023/05/03 17:28:48	5 s	87/87			189.2 MB	48.4 MB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

672

save at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.lang.Thread.run(Thread.java:750)

2023/05/03 17:28:48

5 s

87/87

189.2 MB

48.4 MB

Skipped Stages (16)

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
671	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/845
670	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
669	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
668	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
667	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
666	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/780
665	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
664	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1000
663	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/757
662	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
661	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
660	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/654
659	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
658	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1
657	save at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/704
656	javaToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:3747) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/1433