| 25 |
*(1) Project [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
+- FileScan csv [ride_id#128,rideable_type#129,started_at#130,ended_at#131,start_station_name#132,start_station_id#133,end_station_name#134,end_station_id#135,start_lat#136,start_lng#137,end_lat#138,end_lng#139,member_casual#140,_corrupt_record#141] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
|
Disk Memory Deserialized 1x Replicated
|
2 |
100.00% |
126.2 MiB |
0.0 B |
| 40 |
AdaptiveSparkPlan isFinalPlan=false
+- Project [ride_id#128, rideable_type#129, started_at#669, ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#675, start_lng#676, end_lat#677, end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156, (isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AS _is_ride_id_not_empty#695, coalesce((ended_at#670 > started_at#669), false) AS _is_ended_after_started#696, coalesce(NOT (end_station_id#135 = start_station_id#133), false) AS _is_different_stations#697, ((((isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AND coalesce((ended_at#670 > started_at#669), false)) AND coalesce(NOT (end_station_id#135 = start_station_id#133), false)) AND isnull(_corrupt_record#141)) AS _is_valid#698]
+- Project [ride_id#128, rideable_type#129, cast(started_at#130 as timestamp) AS started_at#669, cast(ended_at#131 as timestamp) AS ended_at#670, start_station_name#132, start_station_id#133, end_s...
|
Disk Memory Deserialized 1x Replicated
|
2 |
100.00% |
104.6 MiB |
0.0 B |
| 366 |
AdaptiveSparkPlan isFinalPlan=false
+- Project [ride_id#3716, rideable_type#3717, started_at#3718, ended_at#3719, start_station_name#3720, start_station_id#3721, end_station_name#3722, end_station_id#3723, start_lat#3724, start_lng#3725, end_lat#3726, end_lng#3727, member_casual#3728, _source_file#3294, 2026-11-01 20:50:07.599499 AS _processed_dttm#13637, _start_station_ride_num#13639, year#13640, month#13641]
+- Window [row_number() windowspecdefinition(start_station_id#3721, _w0#13642, _w1#13643, started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#13639], [start_station_id#3721, _w0#13642, _w1#13643], [started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST]
+- Sort [start_station_id#3721 ASC NULLS FIRST, _w0#13642 ASC NULLS FIRST, _w1#13643 ASC NULLS FIRST, started_at#3718 ASC NULLS FIRST, ride_id#3716 ASC NULLS FIRST], false, 0
+- Exchange hashpartitioning(start_station_id...
|
Disk Memory Deserialized 1x Replicated
|
2 |
100.00% |
93.2 MiB |
0.0 B |
| 380 |
AdaptiveSparkPlan isFinalPlan=false
+- Project [ride_id#3716, rideable_type#3717, started_at#3718, ended_at#3719, start_station_name#3720, start_station_id#3721, end_station_name#3722, end_station_id#3723, start_lat#3724, start_lng#3725, end_lat#3726, end_lng#3727, member_casual#3728, _corrupt_record#3279, _source_file#3294, _is_ride_id_not_empty#12608, _is_ended_after_started#12609, _is_different_stations#12610, _is_valid#12611, 2026-11-01 20:50:07.641085 AS _processed_dttm#14037]
+- Filter (isnotnull(_is_valid#12611) AND NOT _is_valid#12611)
+- InMemoryTableScan [_corrupt_record#3279, _is_different_stations#12610, _is_ended_after_started#12609, _is_ride_id_not_empty#12608, _is_valid#12611, _source_file#3294, end_lat#3726, end_lng#3727, end_station_id#3723, end_station_name#3722, ended_at#3719, member_casual#3728, ride_id#3716, rideable_type#3717, start_lat#3724, start_lng#3725, start_station_id#3721, start_station_name#3720, started_at#3718], [isnotnull(_is_valid#12611), NOT _is_valid#12611]
...
|
Disk Memory Deserialized 1x Replicated
|
2 |
100.00% |
2.7 MiB |
0.0 B |