digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 3<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>1 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 3<br>local bytes read: 187.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 3<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>72.0 B (0.0 B, 24.0 B, 24.0 B (stage 14.0: task 15))<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>187.0 B (0.0 B, 64.0 B, 64.0 B (stage 14.0: task 15))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n959 ms (18 ms, 470 ms, 471 ms (stage 14.0: task 16))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>956 ms (17 ms, 469 ms, 470 ms (stage 14.0: task 16))<br>number of output rows: 3"];
}
6 [labelType="html" label="<b>InMemoryTableScan</b><br><br>number of output rows: 1,816,391"];
7 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster8 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))";
9 [labelType="html" label="<br><b>Project</b><br><br>"];
}
10 [labelType="html" label="<b>Window</b><br><br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))";
12 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))<br>peak memory total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))"];
}
13 [labelType="html" label="<br><b>AQEShuffleRead</b><br><br>"];
14 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 0<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))<br>remote merged bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))<br>remote merged blocks fetched: 0<br>records read: 0<br>local bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))<br>remote bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))<br>merged fetch fallback count: 0<br>local blocks read: 0<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))<br>local merged bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))<br>number of partitions: 0<br>remote reqs duration total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))<br>remote bytes read to disk total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))"];
subgraph cluster15 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))";
16 [labelType="html" label="<br><b>Project</b><br><br>"];
17 [labelType="html" label="<br><b>Project</b><br><br>"];
}
18 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 0<br>number of files read: 0<br>metadata time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 14.0: task 15))<br>size of files read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 14.0: task 15))"];
2->0;
3->2;
5->3;
6->5;
7->6;
9->7;
10->9;
12->10;
13->12;
14->13;
16->14;
17->16;
18->17;
}
19
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[sum(cast(_start_station_id_fixed#109 as int)), sum(cast(_end_station_id_fixed#139 as int))])
WholeStageCodegen (2)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=223]
HashAggregate(keys=[], functions=[partial_sum(cast(_start_station_id_fixed#109 as int)), partial_sum(cast(_end_station_id_fixed#139 as int))])
WholeStageCodegen (1)
InMemoryTableScan [_start_station_id_fixed#109, _end_station_id_fixed#139]
AdaptiveSparkPlan isFinalPlan=true
Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179]
WholeStageCodegen (3)
Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST]
Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
AQEShuffleRead coalesced
Exchange hashpartitioning(start_station_id#124, _w0#180, _w1#181, 200), ENSURE_REQUIREMENTS, [plan_id=124]
Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, (isnotnull(ride_id#56) AND NOT (trim(ride_id#56, None) = )) AS _ride_id_ok#171, coalesce((ended_at#94 > started_at#93), false) AS _duration_ok#172, coalesce(NOT (end_station_id#155 = start_station_id#124), false) AS _station_id_ok#173, input_file_name() AS _source_file#174, year(cast(started_at#93 as date)) AS year#176, month(cast(started_at#93 as date)) AS month#177, year(cast(started_at#93 as date)) AS _w0#180, month(cast(started_at#93 as date)) AS _w1#181]
Project [ride_id#56, rideable_type#57, cast(started_at#58 as timestamp) AS started_at#93, cast(ended_at#59 as timestamp) AS ended_at#94, start_station_name#60, CASE WHEN RLIKE(start_station_id#61, ^\d+\.\d$) THEN concat(start_station_id#61, 0) ELSE start_station_id#61 END AS start_station_id#124, end_station_name#62, CASE WHEN RLIKE(end_station_id#63, ^\d+\.\d$) THEN concat(end_station_id#63, 0) ELSE end_station_id#63 END AS end_station_id#155, cast(start_lat#64 as double) AS start_lat#88, cast(start_lng#65 as double) AS start_lng#89, cast(end_lat#66 as double) AS end_lat#90, cast(end_lng#67 as double) AS end_lng#91, member_casual#68, RLIKE(start_station_id#61, ^\d+\.\d$) AS _start_station_id_fixed#109, RLIKE(end_station_id#63, ^\d+\.\d$) AS _end_station_id_fixed#139]
WholeStageCodegen (1)
FileScan csv [ride_id#56,rideable_type#57,started_at#58,ended_at#59,start_station_name#60,start_station_id#61,end_station_name#62,end_station_id#63,start_lat#64,start_lng#65,end_lat#66,end_lng#67,member_casual#68] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-kiryanova-tatiana/raw/citibike_data/202601/202601-citi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
== Physical Plan ==
AdaptiveSparkPlan (19)
+- == Final Plan ==
* HashAggregate (15)
+- ShuffleQueryStage (14), Statistics(sizeInBytes=72.0 B, rowCount=3)
+- Exchange (13)
+- * HashAggregate (12)
+- TableCacheQueryStage (11), Statistics(sizeInBytes=497.6 MiB, rowCount=1.82E+6)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (10)
+- Project (9)
+- Window (8)
+- Sort (7)
+- Exchange (6)
+- Project (5)
+- Project (4)
+- Scan csv (3)
+- == Initial Plan ==
HashAggregate (18)
+- Exchange (17)
+- HashAggregate (16)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (10)
+- Project (9)
+- Window (8)
+- Sort (7)
+- Exchange (6)
+- Project (5)
+- Project (4)
+- Scan csv (3)
(1) InMemoryTableScan
Output [2]: [_start_station_id_fixed#109, _end_station_id_fixed#139]
Arguments: [_start_station_id_fixed#109, _end_station_id_fixed#139]
(2) InMemoryRelation
Arguments: [ride_id#82, rideable_type#83, started_at#93, ended_at#94, start_station_name#84, start_station_id#124, end_station_name#86, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#92, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, _processed_dttm#175, year#176, month#177, _start_station_ride_num#179], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7a9be29e,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(3) Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179]
+- Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST]
+- *(2) Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0
+- AQEShuffleRead coalesced
+- ShuffleQueryStage 0
+- Exchange hashpartitioning(start_station_id#124, _w0#180, _w1#181, 200), ENSURE_REQUIREMENTS, [plan_id=124]
+- *(1) Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, (isnotnull(ride_id#56) AND NOT (trim(ride_id#56, None) = )) AS _ride_id_ok#171, coalesce((ended_at#94 > started_at#93), false) AS _duration_ok#172, coalesce(NOT (end_station_id#155 = start_station_id#124), false) AS _station_id_ok#173, input_file_name() AS _source_file#174, year(cast(started_at#93 as date)) AS year#176, month(cast(started_at#93 as date)) AS month#177, year(cast(started_at#93 as date)) AS _w0#180, month(cast(started_at#93 as date)) AS _w1#181]
+- *(1) Project [ride_id#56, rideable_type#57, cast(started_at#58 as timestamp) AS started_at#93, cast(ended_at#59 as timestamp) AS ended_at#94, start_station_name#60, CASE WHEN RLIKE(start_station_id#61, ^\d+\.\d$) THEN concat(start_station_id#61, 0) ELSE start_station_id#61 END AS start_station_id#124, end_station_name#62, CASE WHEN RLIKE(end_station_id#63, ^\d+\.\d$) THEN concat(end_station_id#63, 0) ELSE end_station_id#63 END AS end_station_id#155, cast(start_lat#64 as double) AS start_lat#88, cast(start_lng#65 as double) AS start_lng#89, cast(end_lat#66 as double) AS end_lat#90, cast(end_lng#67 as double) AS end_lng#91, member_casual#68, RLIKE(start_station_id#61, ^\d+\.\d$) AS _start_station_id_fixed#109, RLIKE(end_station_id#63, ^\d+\.\d$) AS _end_station_id_fixed#139]
+- FileScan csv [ride_id#56,rideable_type#57,started_at#58,ended_at#59,start_station_name#60,start_station_id#61,end_station_name#62,end_station_id#63,start_lat#64,start_lng#65,end_lat#66,end_lng#67,member_casual#68] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-kiryanova-tatiana/raw/citibike_data/202601/202601-citi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
+- == Initial Plan ==
Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179]
+- Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST]
+- Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0
+- Exchange hashpartitioning(start_station_id#124, _w0#180, _w1#181, 200), ENSURE_REQUIREMENTS, [plan_id=83]
+- Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, (isnotnull(ride_id#56) AND NOT (trim(ride_id#56, None) = )) AS _ride_id_ok#171, coalesce((ended_at#94 > started_at#93), false) AS _duration_ok#172, coalesce(NOT (end_station_id#155 = start_station_id#124), false) AS _station_id_ok#173, input_file_name() AS _source_file#174, year(cast(started_at#93 as date)) AS year#176, month(cast(started_at#93 as date)) AS month#177, year(cast(started_at#93 as date)) AS _w0#180, month(cast(started_at#93 as date)) AS _w1#181]
+- Project [ride_id#56, rideable_type#57, cast(started_at#58 as timestamp) AS started_at#93, cast(ended_at#59 as timestamp) AS ended_at#94, start_station_name#60, CASE WHEN RLIKE(start_station_id#61, ^\d+\.\d$) THEN concat(start_station_id#61, 0) ELSE start_station_id#61 END AS start_station_id#124, end_station_name#62, CASE WHEN RLIKE(end_station_id#63, ^\d+\.\d$) THEN concat(end_station_id#63, 0) ELSE end_station_id#63 END AS end_station_id#155, cast(start_lat#64 as double) AS start_lat#88, cast(start_lng#65 as double) AS start_lng#89, cast(end_lat#66 as double) AS end_lat#90, cast(end_lng#67 as double) AS end_lng#91, member_casual#68, RLIKE(start_station_id#61, ^\d+\.\d$) AS _start_station_id_fixed#109, RLIKE(end_station_id#63, ^\d+\.\d$) AS _end_station_id_fixed#139]
+- FileScan csv [ride_id#56,rideable_type#57,started_at#58,ended_at#59,start_station_name#60,start_station_id#61,end_station_name#62,end_station_id#63,start_lat#64,start_lng#65,end_lat#66,end_lng#67,member_casual#68] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-kiryanova-tatiana/raw/citibike_data/202601/202601-citi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
,None)
(3) Scan csv
Output [13]: [ride_id#56, rideable_type#57, started_at#58, ended_at#59, start_station_name#60, start_station_id#61, end_station_name#62, end_station_id#63, start_lat#64, start_lng#65, end_lat#66, end_lng#67, member_casual#68]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g11-kiryanova-tatiana/raw/citibike_data/202601/202601-citibike-tripdata-part00.csv, ... 1 entries]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:string,start_lng:string,end_lat:string,end_lng:string,member_casual:string>
(4) Project
Output [15]: [ride_id#56, rideable_type#57, cast(started_at#58 as timestamp) AS started_at#93, cast(ended_at#59 as timestamp) AS ended_at#94, start_station_name#60, CASE WHEN RLIKE(start_station_id#61, ^\d+\.\d$) THEN concat(start_station_id#61, 0) ELSE start_station_id#61 END AS start_station_id#124, end_station_name#62, CASE WHEN RLIKE(end_station_id#63, ^\d+\.\d$) THEN concat(end_station_id#63, 0) ELSE end_station_id#63 END AS end_station_id#155, cast(start_lat#64 as double) AS start_lat#88, cast(start_lng#65 as double) AS start_lng#89, cast(end_lat#66 as double) AS end_lat#90, cast(end_lng#67 as double) AS end_lng#91, member_casual#68, RLIKE(start_station_id#61, ^\d+\.\d$) AS _start_station_id_fixed#109, RLIKE(end_station_id#63, ^\d+\.\d$) AS _end_station_id_fixed#139]
Input [13]: [ride_id#56, rideable_type#57, started_at#58, ended_at#59, start_station_name#60, start_station_id#61, end_station_name#62, end_station_id#63, start_lat#64, start_lng#65, end_lat#66, end_lng#67, member_casual#68]
(5) Project
Output [23]: [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, (isnotnull(ride_id#56) AND NOT (trim(ride_id#56, None) = )) AS _ride_id_ok#171, coalesce((ended_at#94 > started_at#93), false) AS _duration_ok#172, coalesce(NOT (end_station_id#155 = start_station_id#124), false) AS _station_id_ok#173, input_file_name() AS _source_file#174, year(cast(started_at#93 as date)) AS year#176, month(cast(started_at#93 as date)) AS month#177, year(cast(started_at#93 as date)) AS _w0#180, month(cast(started_at#93 as date)) AS _w1#181]
Input [15]: [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139]
(6) Exchange
Input [23]: [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, year#176, month#177, _w0#180, _w1#181]
Arguments: hashpartitioning(start_station_id#124, _w0#180, _w1#181, 200), ENSURE_REQUIREMENTS, [plan_id=197]
(7) Sort
Input [23]: [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, year#176, month#177, _w0#180, _w1#181]
Arguments: [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0
(8) Window
Input [23]: [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, year#176, month#177, _w0#180, _w1#181]
Arguments: [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST]
(9) Project
Output [23]: [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179]
Input [24]: [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, year#176, month#177, _w0#180, _w1#181, _start_station_ride_num#179]
(10) AdaptiveSparkPlan
Output [23]: [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, _processed_dttm#175, year#176, month#177, _start_station_ride_num#179]
Arguments: isFinalPlan=false
(11) TableCacheQueryStage
Output [2]: [_start_station_id_fixed#109, _end_station_id_fixed#139]
Arguments: 0
(12) HashAggregate [codegen id : 1]
Input [2]: [_start_station_id_fixed#109, _end_station_id_fixed#139]
Keys: []
Functions [2]: [partial_sum(cast(_start_station_id_fixed#109 as int)), partial_sum(cast(_end_station_id_fixed#139 as int))]
Aggregate Attributes [2]: [sum#1310L, sum#1311L]
Results [2]: [sum#1312L, sum#1313L]
(13) Exchange
Input [2]: [sum#1312L, sum#1313L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=223]
(14) ShuffleQueryStage
Output [2]: [sum#1312L, sum#1313L]
Arguments: 1
(15) HashAggregate [codegen id : 2]
Input [2]: [sum#1312L, sum#1313L]
Keys: []
Functions [2]: [sum(cast(_start_station_id_fixed#109 as int)), sum(cast(_end_station_id_fixed#139 as int))]
Aggregate Attributes [2]: [sum(cast(_start_station_id_fixed#109 as int))#936L, sum(cast(_end_station_id_fixed#139 as int))#938L]
Results [2]: [sum(cast(_start_station_id_fixed#109 as int))#936L AS start#937L, sum(cast(_end_station_id_fixed#139 as int))#938L AS end#939L]
(16) HashAggregate
Input [2]: [_start_station_id_fixed#109, _end_station_id_fixed#139]
Keys: []
Functions [2]: [partial_sum(cast(_start_station_id_fixed#109 as int)), partial_sum(cast(_end_station_id_fixed#139 as int))]
Aggregate Attributes [2]: [sum#1310L, sum#1311L]
Results [2]: [sum#1312L, sum#1313L]
(17) Exchange
Input [2]: [sum#1312L, sum#1313L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=194]
(18) HashAggregate
Input [2]: [sum#1312L, sum#1313L]
Keys: []
Functions [2]: [sum(cast(_start_station_id_fixed#109 as int)), sum(cast(_end_station_id_fixed#139 as int))]
Aggregate Attributes [2]: [sum(cast(_start_station_id_fixed#109 as int))#936L, sum(cast(_end_station_id_fixed#139 as int))#938L]
Results [2]: [sum(cast(_start_station_id_fixed#109 as int))#936L AS start#937L, sum(cast(_end_station_id_fixed#139 as int))#938L AS end#939L]
(19) AdaptiveSparkPlan
Output [2]: [start#937L, end#939L]
Arguments: isFinalPlan=true