digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
1 [labelType="html" label="<br><b>TakeOrderedAndProject</b><br><br>"];
subgraph cluster2 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n1.4 s (715 ms, 729 ms, 729 ms (stage 81.0: task 81))";
3 [labelType="html" label="<br><b>Project</b><br><br>"];
}
4 [labelType="html" label="<b>Window</b><br><br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 81.0: task 80))"];
subgraph cluster5 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n3.5 s (1.8 s, 1.8 s, 1.8 s (stage 81.0: task 81))";
6 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>1.4 s (706 ms, 719 ms, 719 ms (stage 81.0: task 81))<br>peak memory total (min, med, max (stageId: taskId))<br>288.0 MiB (144.0 MiB, 144.0 MiB, 144.0 MiB (stage 81.0: task 80))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 81.0: task 80))"];
}
7 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 2<br>partition data size total (min, med, max (stageId: taskId))<br>35.3 MiB (17.6 MiB, 17.7 MiB, 17.7 MiB (driver))<br>number of coalesced partitions: 2"];
8 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1,190,091<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>147 ms (0 ms, 72 ms, 75 ms (stage 79.0: task 78))<br>remote merged bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 81.0: task 80))<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 81.0: task 80))<br>remote merged blocks fetched: 0<br>records read: 1,190,091<br>local bytes read total (min, med, max (stageId: taskId))<br>33.6 MiB (16.8 MiB, 16.9 MiB, 16.9 MiB (stage 81.0: task 81))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 81.0: task 80))<br>remote bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 81.0: task 80))<br>merged fetch fallback count: 0<br>local blocks read: 4<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>190.7 MiB (0.0 B, 91.9 MiB, 98.8 MiB (stage 79.0: task 78))<br>local merged bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 81.0: task 80))<br>number of partitions: 200<br>remote reqs duration total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 81.0: task 80))<br>remote bytes read to disk total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 81.0: task 80))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>33.6 MiB (0.0 B, 16.2 MiB, 17.4 MiB (stage 79.0: task 78))"];
subgraph cluster9 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n2.0 s (0 ms, 982 ms, 1.1 s (stage 79.0: task 78))";
10 [labelType="html" label="<br><b>Project</b><br><br>"];
11 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1,190,091"];
}
12 [labelType="html" label="<b>InMemoryTableScan</b><br><br>number of output rows: 1,219,444"];
13 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster14 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 79.0: task 79))";
15 [labelType="html" label="<br><b>Project</b><br><br>"];
16 [labelType="html" label="<br><b>Project</b><br><br>"];
}
17 [labelType="html" label="<b>InMemoryTableScan</b><br><br>number of output rows: 0"];
subgraph cluster18 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 79.0: task 79))";
19 [labelType="html" label="<br><b>Project</b><br><br>"];
}
20 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 0"];
1->0;
3->1;
4->3;
6->4;
7->6;
8->7;
10->8;
11->10;
12->11;
13->12;
15->13;
16->15;
17->16;
19->17;
20->19;
}
21
AdaptiveSparkPlan isFinalPlan=true
TakeOrderedAndProject(limit=11, orderBy=[start_station_id#3721 ASC NULLS FIRST,_start_station_ride_num#8383 ASC NULLS FIRST], output=[toprettystring(start_station_id)#8441,toprettystring(started_at)#8442,toprettystring(_start_station_ride_num)#8443,toprettystring(_source_file)#8444,toprettystring(_processed_dttm)#8445,toprettystring(year)#8446,toprettystring(month)#8447])
Project [start_station_id#3721, started_at#3718, _start_station_ride_num#8383, _source_file#3294, year#8384, month#8385]
WholeStageCodegen (3)
Window [row_number() windowspecdefinition(start_station_id#3721, _w0#8386, _w1#8387, started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#8383], [start_station_id#3721, _w0#8386, _w1#8387], [started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST]
Sort [start_station_id#3721 ASC NULLS FIRST, _w0#8386 ASC NULLS FIRST, _w1#8387 ASC NULLS FIRST, started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
AQEShuffleRead coalesced
Exchange hashpartitioning(start_station_id#3721, _w0#8386, _w1#8387, 200), ENSURE_REQUIREMENTS, [plan_id=2018]
Project [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, date_format(started_at#3718, yyyy, Some(UTC)) AS year#8384, date_format(started_at#3718, MM, Some(UTC)) AS month#8385, date_format(started_at#3718, yyyy, Some(UTC)) AS _w0#8386, date_format(started_at#3718, MM, Some(UTC)) AS _w1#8387]
Filter (isnotnull(_is_valid#3747) AND _is_valid#3747)
WholeStageCodegen (1)
InMemoryTableScan [_is_valid#3747, _source_file#3294, ride_id#3716, start_station_id#3721, started_at#3718], [isnotnull(_is_valid#3747), _is_valid#3747]
AdaptiveSparkPlan isFinalPlan=true
Project [ride_id#128, rideable_type#129, started_at#669, ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#675, start_lng#676, end_lat#677, end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156, (isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AS _is_ride_id_not_empty#695, coalesce((ended_at#670 > started_at#669), false) AS _is_ended_after_started#696, coalesce(NOT (end_station_id#135 = start_station_id#133), false) AS _is_different_stations#697, ((((isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AND coalesce((ended_at#670 > started_at#669), false)) AND coalesce(NOT (end_station_id#135 = start_station_id#133), false)) AND isnull(_corrupt_record#141)) AS _is_valid#698]
Project [ride_id#128, rideable_type#129, cast(started_at#130 as timestamp) AS started_at#669, cast(ended_at#131 as timestamp) AS ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, cast(start_lat#136 as double) AS start_lat#675, cast(start_lng#137 as double) AS start_lng#676, cast(end_lat#138 as double) AS end_lat#677, cast(end_lng#139 as double) AS end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156]
WholeStageCodegen (1)
InMemoryTableScan [_corrupt_record#141, _source_file#156, end_lat#138, end_lng#139, end_station_id#135, end_station_name#134, ended_at#131, member_casual#140, ride_id#128, rideable_type#129, start_lat#136, start_lng#137, start_station_id#133, start_station_name#132, started_at#130]
Project [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
WholeStageCodegen (1)
FileScan csv [ride_id#128,rideable_type#129,started_at#130,ended_at#131,start_station_name#132,start_station_id#133,end_station_name#134,end_station_id#135,start_lat#136,start_lng#137,end_lat#138,end_lng#139,member_casual#140,_corrupt_record#141] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
== Physical Plan ==
AdaptiveSparkPlan (27)
+- == Final Plan ==
TakeOrderedAndProject (19)
+- * Project (18)
+- Window (17)
+- * Sort (16)
+- AQEShuffleRead (15)
+- ShuffleQueryStage (14), Statistics(sizeInBytes=190.7 MiB, rowCount=1.19E+6)
+- Exchange (13)
+- * Project (12)
+- * Filter (11)
+- TableCacheQueryStage (10), Statistics(sizeInBytes=245.7 MiB, rowCount=1.22E+6)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (9)
+- Project (8)
+- Project (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
+- == Initial Plan ==
TakeOrderedAndProject (26)
+- Project (25)
+- Window (24)
+- Sort (23)
+- Exchange (22)
+- Project (21)
+- Filter (20)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (9)
+- Project (8)
+- Project (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [5]: [_is_valid#3747, _source_file#3294, ride_id#3716, start_station_id#3721, started_at#3718]
Arguments: [_is_valid#3747, _source_file#3294, ride_id#3716, start_station_id#3721, started_at#3718], [isnotnull(_is_valid#3747), _is_valid#3747]
(2) InMemoryRelation
Arguments: [ride_id#3716, rideable_type#3717, started_at#3718, ended_at#3719, start_station_name#3720, start_station_id#3721, end_station_name#3722, end_station_id#3723, start_lat#3724, start_lng#3725, end_lat#3726, end_lng#3727, member_casual#3728, _corrupt_record#3279, _source_file#3294, _is_ride_id_not_empty#3744, _is_ended_after_started#3745, _is_different_stations#3746, _is_valid#3747], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@28f1e8bc,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [ride_id#128, rideable_type#129, started_at#669, ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#675, start_lng#676, end_lat#677, end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156, (isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AS _is_ride_id_not_empty#695, coalesce((ended_at#670 > started_at#669), false) AS _is_ended_after_started#696, coalesce(NOT (end_station_id#135 = start_station_id#133), false) AS _is_different_stations#697, ((((isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AND coalesce((ended_at#670 > started_at#669), false)) AND coalesce(NOT (end_station_id#135 = start_station_id#133), false)) AND isnull(_corrupt_record#141)) AS _is_valid#698]
+- *(1) Project [ride_id#128, rideable_type#129, cast(started_at#130 as timestamp) AS started_at#669, cast(ended_at#131 as timestamp) AS ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, cast(start_lat#136 as double) AS start_lat#675, cast(start_lng#137 as double) AS start_lng#676, cast(end_lat#138 as double) AS end_lat#677, cast(end_lng#139 as double) AS end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156]
+- TableCacheQueryStage 0
+- InMemoryTableScan [_corrupt_record#141, _source_file#156, end_lat#138, end_lng#139, end_station_id#135, end_station_name#134, ended_at#131, member_casual#140, ride_id#128, rideable_type#129, start_lat#136, start_lng#137, start_station_id#133, start_station_name#132, started_at#130]
+- InMemoryRelation [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, _source_file#156], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
+- FileScan csv [ride_id#128,rideable_type#129,started_at#130,ended_at#131,start_station_name#132,start_station_id#133,end_station_name#134,end_station_id#135,start_lat#136,start_lng#137,end_lat#138,end_lng#139,member_casual#140,_corrupt_record#141] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
+- == Initial Plan ==
Project [ride_id#128, rideable_type#129, started_at#669, ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#675, start_lng#676, end_lat#677, end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156, (isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AS _is_ride_id_not_empty#695, coalesce((ended_at#670 > started_at#669), false) AS _is_ended_after_started#696, coalesce(NOT (end_station_id#135 = start_station_id#133), false) AS _is_different_stations#697, ((((isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AND coalesce((ended_at#670 > started_at#669), false)) AND coalesce(NOT (end_station_id#135 = start_station_id#133), false)) AND isnull(_corrupt_record#141)) AS _is_valid#698]
+- Project [ride_id#128, rideable_type#129, cast(started_at#130 as timestamp) AS started_at#669, cast(ended_at#131 as timestamp) AS ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, cast(start_lat#136 as double) AS start_lat#675, cast(start_lng#137 as double) AS start_lng#676, cast(end_lat#138 as double) AS end_lat#677, cast(end_lng#139 as double) AS end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156]
+- InMemoryTableScan [_corrupt_record#141, _source_file#156, end_lat#138, end_lng#139, end_station_id#135, end_station_name#134, ended_at#131, member_casual#140, ride_id#128, rideable_type#129, start_lat#136, start_lng#137, start_station_id#133, start_station_name#132, started_at#130]
+- InMemoryRelation [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, _source_file#156], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
+- FileScan csv [ride_id#128,rideable_type#129,started_at#130,ended_at#131,start_station_name#132,start_station_id#133,end_station_name#134,end_station_id#135,start_lat#136,start_lng#137,end_lat#138,end_lng#139,member_casual#140,_corrupt_record#141] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
,None)
(3) InMemoryTableScan
Output [15]: [_corrupt_record#141, _source_file#156, end_lat#138, end_lng#139, end_station_id#135, end_station_name#134, ended_at#131, member_casual#140, ride_id#128, rideable_type#129, start_lat#136, start_lng#137, start_station_id#133, start_station_name#132, started_at#130]
Arguments: [_corrupt_record#141, _source_file#156, end_lat#138, end_lng#139, end_station_id#135, end_station_name#134, ended_at#131, member_casual#140, ride_id#128, rideable_type#129, start_lat#136, start_lng#137, start_station_id#133, start_station_name#132, started_at#130]
(4) InMemoryRelation
Arguments: [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, _source_file#156], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@28f1e8bc,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
+- FileScan csv [ride_id#128,rideable_type#129,started_at#130,ended_at#131,start_station_name#132,start_station_id#133,end_station_name#134,end_station_id#135,start_lat#136,start_lng#137,end_lat#138,end_lng#139,member_casual#140,_corrupt_record#141] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
,None)
(5) Scan csv
Output [14]: [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-tripdata-part00.csv, ... 1 entries]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:string,start_lng:string,end_lat:string,end_lng:string,member_casual:string,_corrupt_record:string>
(6) Project [codegen id : 1]
Output [15]: [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
Input [14]: [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141]
(7) Project
Output [15]: [ride_id#128, rideable_type#129, cast(started_at#130 as timestamp) AS started_at#669, cast(ended_at#131 as timestamp) AS ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, cast(start_lat#136 as double) AS start_lat#675, cast(start_lng#137 as double) AS start_lng#676, cast(end_lat#138 as double) AS end_lat#677, cast(end_lng#139 as double) AS end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156]
Input [15]: [_corrupt_record#141, _source_file#156, end_lat#138, end_lng#139, end_station_id#135, end_station_name#134, ended_at#131, member_casual#140, ride_id#128, rideable_type#129, start_lat#136, start_lng#137, start_station_id#133, start_station_name#132, started_at#130]
(8) Project
Output [19]: [ride_id#128, rideable_type#129, started_at#669, ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#675, start_lng#676, end_lat#677, end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156, (isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AS _is_ride_id_not_empty#695, coalesce((ended_at#670 > started_at#669), false) AS _is_ended_after_started#696, coalesce(NOT (end_station_id#135 = start_station_id#133), false) AS _is_different_stations#697, ((((isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AND coalesce((ended_at#670 > started_at#669), false)) AND coalesce(NOT (end_station_id#135 = start_station_id#133), false)) AND isnull(_corrupt_record#141)) AS _is_valid#698]
Input [15]: [ride_id#128, rideable_type#129, started_at#669, ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#675, start_lng#676, end_lat#677, end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156]
(9) AdaptiveSparkPlan
Output [19]: [ride_id#128, rideable_type#129, started_at#669, ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#675, start_lng#676, end_lat#677, end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156, _is_ride_id_not_empty#695, _is_ended_after_started#696, _is_different_stations#697, _is_valid#698]
Arguments: isFinalPlan=false
(10) TableCacheQueryStage
Output [5]: [_is_valid#3747, _source_file#3294, ride_id#3716, start_station_id#3721, started_at#3718]
Arguments: 0
(11) Filter [codegen id : 1]
Input [5]: [_is_valid#3747, _source_file#3294, ride_id#3716, start_station_id#3721, started_at#3718]
Condition : (isnotnull(_is_valid#3747) AND _is_valid#3747)
(12) Project [codegen id : 1]
Output [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, date_format(started_at#3718, yyyy, Some(UTC)) AS year#8384, date_format(started_at#3718, MM, Some(UTC)) AS month#8385, date_format(started_at#3718, yyyy, Some(UTC)) AS _w0#8386, date_format(started_at#3718, MM, Some(UTC)) AS _w1#8387]
Input [5]: [_is_valid#3747, _source_file#3294, ride_id#3716, start_station_id#3721, started_at#3718]
(13) Exchange
Input [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387]
Arguments: hashpartitioning(start_station_id#3721, _w0#8386, _w1#8387, 200), ENSURE_REQUIREMENTS, [plan_id=2018]
(14) ShuffleQueryStage
Output [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387]
Arguments: 1
(15) AQEShuffleRead
Input [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387]
Arguments: coalesced
(16) Sort [codegen id : 2]
Input [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387]
Arguments: [start_station_id#3721 ASC NULLS FIRST, _w0#8386 ASC NULLS FIRST, _w1#8387 ASC NULLS FIRST, started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST], false, 0
(17) Window
Input [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387]
Arguments: [row_number() windowspecdefinition(start_station_id#3721, _w0#8386, _w1#8387, started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#8383], [start_station_id#3721, _w0#8386, _w1#8387], [started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST]
(18) Project [codegen id : 3]
Output [6]: [start_station_id#3721, started_at#3718, _start_station_ride_num#8383, _source_file#3294, year#8384, month#8385]
Input [9]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387, _start_station_ride_num#8383]
(19) TakeOrderedAndProject
Input [6]: [start_station_id#3721, started_at#3718, _start_station_ride_num#8383, _source_file#3294, year#8384, month#8385]
Arguments: 11, [start_station_id#3721 ASC NULLS FIRST, _start_station_ride_num#8383 ASC NULLS FIRST], [toprettystring(start_station_id#3721, Some(UTC)) AS toprettystring(start_station_id)#8441, toprettystring(started_at#3718, Some(UTC)) AS toprettystring(started_at)#8442, toprettystring(_start_station_ride_num#8383, Some(UTC)) AS toprettystring(_start_station_ride_num)#8443, toprettystring(_source_file#3294, Some(UTC)) AS toprettystring(_source_file)#8444, 2026-11-01 20:42:14.394711 AS toprettystring(_processed_dttm)#8445, toprettystring(year#8384, Some(UTC)) AS toprettystring(year)#8446, toprettystring(month#8385, Some(UTC)) AS toprettystring(month)#8447]
(20) Filter
Input [5]: [_is_valid#3747, _source_file#3294, ride_id#3716, start_station_id#3721, started_at#3718]
Condition : (isnotnull(_is_valid#3747) AND _is_valid#3747)
(21) Project
Output [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, date_format(started_at#3718, yyyy, Some(UTC)) AS year#8384, date_format(started_at#3718, MM, Some(UTC)) AS month#8385, date_format(started_at#3718, yyyy, Some(UTC)) AS _w0#8386, date_format(started_at#3718, MM, Some(UTC)) AS _w1#8387]
Input [5]: [_is_valid#3747, _source_file#3294, ride_id#3716, start_station_id#3721, started_at#3718]
(22) Exchange
Input [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387]
Arguments: hashpartitioning(start_station_id#3721, _w0#8386, _w1#8387, 200), ENSURE_REQUIREMENTS, [plan_id=1973]
(23) Sort
Input [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387]
Arguments: [start_station_id#3721 ASC NULLS FIRST, _w0#8386 ASC NULLS FIRST, _w1#8387 ASC NULLS FIRST, started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST], false, 0
(24) Window
Input [8]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387]
Arguments: [row_number() windowspecdefinition(start_station_id#3721, _w0#8386, _w1#8387, started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#8383], [start_station_id#3721, _w0#8386, _w1#8387], [started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST]
(25) Project
Output [6]: [start_station_id#3721, started_at#3718, _start_station_ride_num#8383, _source_file#3294, year#8384, month#8385]
Input [9]: [ride_id#3716, started_at#3718, start_station_id#3721, _source_file#3294, year#8384, month#8385, _w0#8386, _w1#8387, _start_station_ride_num#8383]
(26) TakeOrderedAndProject
Input [6]: [start_station_id#3721, started_at#3718, _start_station_ride_num#8383, _source_file#3294, year#8384, month#8385]
Arguments: 11, [start_station_id#3721 ASC NULLS FIRST, _start_station_ride_num#8383 ASC NULLS FIRST], [toprettystring(start_station_id#3721, Some(UTC)) AS toprettystring(start_station_id)#8441, toprettystring(started_at#3718, Some(UTC)) AS toprettystring(started_at)#8442, toprettystring(_start_station_ride_num#8383, Some(UTC)) AS toprettystring(_start_station_ride_num)#8443, toprettystring(_source_file#3294, Some(UTC)) AS toprettystring(_source_file)#8444, 2026-11-01 20:42:14.394711 AS toprettystring(_processed_dttm)#8445, toprettystring(year#8384, Some(UTC)) AS toprettystring(year)#8446, toprettystring(month#8385, Some(UTC)) AS toprettystring(month)#8447]
(27) AdaptiveSparkPlan
Output [7]: [toprettystring(start_station_id)#8441, toprettystring(started_at)#8442, toprettystring(_start_station_ride_num)#8443, toprettystring(_source_file)#8444, toprettystring(_processed_dttm)#8445, toprettystring(year)#8446, toprettystring(month)#8447]
Arguments: isFinalPlan=true