== Physical Plan ==
AdaptiveSparkPlan (18)
+- == Final Plan ==
CollectLimit (10)
+- * Project (9)
+- Window (8)
+- * Sort (7)
+- AQEShuffleRead (6)
+- ShuffleQueryStage (5), Statistics(sizeInBytes=1612.1 MiB, rowCount=4.73E+6)
+- Exchange (4)
+- * Project (3)
+- * Project (2)
+- Scan csv (1)
+- == Initial Plan ==
CollectLimit (17)
+- Project (16)
+- Window (15)
+- Sort (14)
+- Exchange (13)
+- Project (12)
+- Project (11)
+- Scan csv (1)
(1) Scan csv
Output [13]: [ride_id#171, rideable_type#172, started_at#173, ended_at#174, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#179, start_lng#180, end_lat#181, end_lng#182, member_casual#183]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g11-terentyeva-kristina/raw/citibike_data/202510/202510-citibike-tripdata-part00.csv, ... 4 entries]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:string,start_lng:string,end_lat:string,end_lng:string,member_casual:string>
(2) Project [codegen id : 1]
Output [13]: [ride_id#171, rideable_type#172, cast(started_at#173 as timestamp) AS started_at#208, cast(ended_at#174 as timestamp) AS ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, cast(start_lat#179 as double) AS start_lat#203, cast(start_lng#180 as double) AS start_lng#204, cast(end_lat#181 as double) AS end_lat#205, cast(end_lng#182 as double) AS end_lng#206, member_casual#183]
Input [13]: [ride_id#171, rideable_type#172, started_at#173, ended_at#174, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#179, start_lng#180, end_lat#181, end_lng#182, member_casual#183]
(3) Project [codegen id : 1]
Output [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, (isnotnull(ride_id#171) AND NOT (trim(ride_id#171, None) = )) AS flag_ride_id_valid#308, ((isnotnull(ended_at#209) AND isnotnull(started_at#208)) AND (ended_at#209 > started_at#208)) AS flag_time_valid#309, ((isnotnull(end_station_id#178) AND isnotnull(start_station_id#176)) AND NOT (end_station_id#178 = start_station_id#176)) AS flag_station_valid#310, input_file_name() AS _source_file#311, year(cast(started_at#208 as date)) AS year#313, month(cast(started_at#208 as date)) AS month#314]
Input [13]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183]
(4) Exchange
Input [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314]
Arguments: hashpartitioning(start_station_id#176, 200), ENSURE_REQUIREMENTS, [plan_id=189]
(5) ShuffleQueryStage
Output [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314]
Arguments: 0
(6) AQEShuffleRead
Input [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314]
Arguments: coalesced
(7) Sort [codegen id : 2]
Input [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314]
Arguments: [start_station_id#176 ASC NULLS FIRST, started_at#208 ASC NULLS FIRST, ride_id#171 ASC NULLS FIRST], false, 0
(8) Window
Input [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314]
Arguments: [row_number() windowspecdefinition(start_station_id#176, started_at#208 ASC NULLS FIRST, ride_id#171 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#336], [start_station_id#176], [started_at#208 ASC NULLS FIRST, ride_id#171 ASC NULLS FIRST]
(9) Project [codegen id : 3]
Output [21]: [toprettystring(ride_id#171, Some(Etc/UTC)) AS toprettystring(ride_id)#405, toprettystring(rideable_type#172, Some(Etc/UTC)) AS toprettystring(rideable_type)#406, toprettystring(started_at#208, Some(Etc/UTC)) AS toprettystring(started_at)#407, toprettystring(ended_at#209, Some(Etc/UTC)) AS toprettystring(ended_at)#408, toprettystring(start_station_name#175, Some(Etc/UTC)) AS toprettystring(start_station_name)#409, toprettystring(start_station_id#176, Some(Etc/UTC)) AS toprettystring(start_station_id)#410, toprettystring(end_station_name#177, Some(Etc/UTC)) AS toprettystring(end_station_name)#411, toprettystring(end_station_id#178, Some(Etc/UTC)) AS toprettystring(end_station_id)#412, toprettystring(start_lat#203, Some(Etc/UTC)) AS toprettystring(start_lat)#413, toprettystring(start_lng#204, Some(Etc/UTC)) AS toprettystring(start_lng)#414, toprettystring(end_lat#205, Some(Etc/UTC)) AS toprettystring(end_lat)#415, toprettystring(end_lng#206, Some(Etc/UTC)) AS toprettystring(end_lng)#416, toprettystring(member_casual#183, Some(Etc/UTC)) AS toprettystring(member_casual)#417, toprettystring(flag_ride_id_valid#308, Some(Etc/UTC)) AS toprettystring(flag_ride_id_valid)#418, toprettystring(flag_time_valid#309, Some(Etc/UTC)) AS toprettystring(flag_time_valid)#419, toprettystring(flag_station_valid#310, Some(Etc/UTC)) AS toprettystring(flag_station_valid)#420, toprettystring(_source_file#311, Some(Etc/UTC)) AS toprettystring(_source_file)#421, 2026-11-04 18:25:15.543494 AS toprettystring(_processed_dttm)#422, toprettystring(year#313, Some(Etc/UTC)) AS toprettystring(year)#423, toprettystring(month#314, Some(Etc/UTC)) AS toprettystring(month)#424, toprettystring(_start_station_ride_num#336, Some(Etc/UTC)) AS toprettystring(_start_station_ride_num)#425]
Input [20]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314, _start_station_ride_num#336]
(10) CollectLimit
Input [21]: [toprettystring(ride_id)#405, toprettystring(rideable_type)#406, toprettystring(started_at)#407, toprettystring(ended_at)#408, toprettystring(start_station_name)#409, toprettystring(start_station_id)#410, toprettystring(end_station_name)#411, toprettystring(end_station_id)#412, toprettystring(start_lat)#413, toprettystring(start_lng)#414, toprettystring(end_lat)#415, toprettystring(end_lng)#416, toprettystring(member_casual)#417, toprettystring(flag_ride_id_valid)#418, toprettystring(flag_time_valid)#419, toprettystring(flag_station_valid)#420, toprettystring(_source_file)#421, toprettystring(_processed_dttm)#422, toprettystring(year)#423, toprettystring(month)#424, toprettystring(_start_station_ride_num)#425]
Arguments: 6
(11) Project
Output [13]: [ride_id#171, rideable_type#172, cast(started_at#173 as timestamp) AS started_at#208, cast(ended_at#174 as timestamp) AS ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, cast(start_lat#179 as double) AS start_lat#203, cast(start_lng#180 as double) AS start_lng#204, cast(end_lat#181 as double) AS end_lat#205, cast(end_lng#182 as double) AS end_lng#206, member_casual#183]
Input [13]: [ride_id#171, rideable_type#172, started_at#173, ended_at#174, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#179, start_lng#180, end_lat#181, end_lng#182, member_casual#183]
(12) Project
Output [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, (isnotnull(ride_id#171) AND NOT (trim(ride_id#171, None) = )) AS flag_ride_id_valid#308, ((isnotnull(ended_at#209) AND isnotnull(started_at#208)) AND (ended_at#209 > started_at#208)) AS flag_time_valid#309, ((isnotnull(end_station_id#178) AND isnotnull(start_station_id#176)) AND NOT (end_station_id#178 = start_station_id#176)) AS flag_station_valid#310, input_file_name() AS _source_file#311, year(cast(started_at#208 as date)) AS year#313, month(cast(started_at#208 as date)) AS month#314]
Input [13]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183]
(13) Exchange
Input [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314]
Arguments: hashpartitioning(start_station_id#176, 200), ENSURE_REQUIREMENTS, [plan_id=173]
(14) Sort
Input [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314]
Arguments: [start_station_id#176 ASC NULLS FIRST, started_at#208 ASC NULLS FIRST, ride_id#171 ASC NULLS FIRST], false, 0
(15) Window
Input [19]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314]
Arguments: [row_number() windowspecdefinition(start_station_id#176, started_at#208 ASC NULLS FIRST, ride_id#171 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#336], [start_station_id#176], [started_at#208 ASC NULLS FIRST, ride_id#171 ASC NULLS FIRST]
(16) Project
Output [21]: [toprettystring(ride_id#171, Some(Etc/UTC)) AS toprettystring(ride_id)#405, toprettystring(rideable_type#172, Some(Etc/UTC)) AS toprettystring(rideable_type)#406, toprettystring(started_at#208, Some(Etc/UTC)) AS toprettystring(started_at)#407, toprettystring(ended_at#209, Some(Etc/UTC)) AS toprettystring(ended_at)#408, toprettystring(start_station_name#175, Some(Etc/UTC)) AS toprettystring(start_station_name)#409, toprettystring(start_station_id#176, Some(Etc/UTC)) AS toprettystring(start_station_id)#410, toprettystring(end_station_name#177, Some(Etc/UTC)) AS toprettystring(end_station_name)#411, toprettystring(end_station_id#178, Some(Etc/UTC)) AS toprettystring(end_station_id)#412, toprettystring(start_lat#203, Some(Etc/UTC)) AS toprettystring(start_lat)#413, toprettystring(start_lng#204, Some(Etc/UTC)) AS toprettystring(start_lng)#414, toprettystring(end_lat#205, Some(Etc/UTC)) AS toprettystring(end_lat)#415, toprettystring(end_lng#206, Some(Etc/UTC)) AS toprettystring(end_lng)#416, toprettystring(member_casual#183, Some(Etc/UTC)) AS toprettystring(member_casual)#417, toprettystring(flag_ride_id_valid#308, Some(Etc/UTC)) AS toprettystring(flag_ride_id_valid)#418, toprettystring(flag_time_valid#309, Some(Etc/UTC)) AS toprettystring(flag_time_valid)#419, toprettystring(flag_station_valid#310, Some(Etc/UTC)) AS toprettystring(flag_station_valid)#420, toprettystring(_source_file#311, Some(Etc/UTC)) AS toprettystring(_source_file)#421, 2026-11-04 18:25:15.543494 AS toprettystring(_processed_dttm)#422, toprettystring(year#313, Some(Etc/UTC)) AS toprettystring(year)#423, toprettystring(month#314, Some(Etc/UTC)) AS toprettystring(month)#424, toprettystring(_start_station_ride_num#336, Some(Etc/UTC)) AS toprettystring(_start_station_ride_num)#425]
Input [20]: [ride_id#171, rideable_type#172, started_at#208, ended_at#209, start_station_name#175, start_station_id#176, end_station_name#177, end_station_id#178, start_lat#203, start_lng#204, end_lat#205, end_lng#206, member_casual#183, flag_ride_id_valid#308, flag_time_valid#309, flag_station_valid#310, _source_file#311, year#313, month#314, _start_station_ride_num#336]
(17) CollectLimit
Input [21]: [toprettystring(ride_id)#405, toprettystring(rideable_type)#406, toprettystring(started_at)#407, toprettystring(ended_at)#408, toprettystring(start_station_name)#409, toprettystring(start_station_id)#410, toprettystring(end_station_name)#411, toprettystring(end_station_id)#412, toprettystring(start_lat)#413, toprettystring(start_lng)#414, toprettystring(end_lat)#415, toprettystring(end_lng)#416, toprettystring(member_casual)#417, toprettystring(flag_ride_id_valid)#418, toprettystring(flag_time_valid)#419, toprettystring(flag_station_valid)#420, toprettystring(_source_file)#421, toprettystring(_processed_dttm)#422, toprettystring(year)#423, toprettystring(month)#424, toprettystring(_start_station_ride_num)#425]
Arguments: 6
(18) AdaptiveSparkPlan
Output [21]: [toprettystring(ride_id)#405, toprettystring(rideable_type)#406, toprettystring(started_at)#407, toprettystring(ended_at)#408, toprettystring(start_station_name)#409, toprettystring(start_station_id)#410, toprettystring(end_station_name)#411, toprettystring(end_station_id)#412, toprettystring(start_lat)#413, toprettystring(start_lng)#414, toprettystring(end_lat)#415, toprettystring(end_lng)#416, toprettystring(member_casual)#417, toprettystring(flag_ride_id_valid)#418, toprettystring(flag_time_valid)#419, toprettystring(flag_station_valid)#420, toprettystring(_source_file)#421, toprettystring(_processed_dttm)#422, toprettystring(year)#423, toprettystring(month)#424, toprettystring(_start_station_ride_num)#425]
Arguments: isFinalPlan=true