digraph G {
0 [labelType="html" label="<br><b>Execute InsertIntoHadoopFsRelationCommand</b><br><br>"];
1 [labelType="html" label="<br><b>WriteFiles</b><br><br>"];
subgraph cluster2 {
isCluster="true";
label="WholeStageCodegen (1)";
3 [labelType="html" label="<br><b>Project</b><br><br>"];
4 [labelType="html" label="<br><b>Project</b><br><br>"];
5 [labelType="html" label="<br><b>Project</b><br><br>"];
6 [labelType="html" label="<br><b>Filter</b><br><br>"];
}
7 [labelType="html" label="<br><b>Scan csv </b><br><br>"];
1->0;
3->1;
4->3;
5->4;
6->5;
7->6;
}
8
Execute InsertIntoHadoopFsRelationCommand s3a://rzvde-g9-chernyshev-miron/raw_invalid/citibike_data/year=2026/month=Column<'add_months(to_date(2026-07-01), -1)'>, false, CSV, [header=true, path=s3a://rzvde-g9-chernyshev-miron/raw_invalid/citibike_data/year=2026/month=Column<'add_months(to_date(2026-07-01), -1)'>], Overwrite, [ride_id, rideable_type, started_at, ended_at, start_station_name, start_station_id, end_station_name, end_station_id, start_lat, start_lng, end_lat, end_lng, member_casual, year, month, year, month, year, month, year, month]
WriteFiles
Project [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year#2748, month#2749, year#2748, month#2749, year#2748, month#2749, year(cast(started_at#19 as date)) AS year#2748, month(cast(started_at#19 as date)) AS month#2749]
Project [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year#2748, month#2749, year#2748, month#2749, year(cast(started_at#19 as date)) AS year#2748, month(cast(started_at#19 as date)) AS month#2749]
Project [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, cast(start_station_id#22 as float) AS start_station_id#1654, end_station_name#23, cast(end_station_id#24 as float) AS end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year(cast(started_at#19 as date)) AS year#2748, month(cast(started_at#19 as date)) AS month#2749, year(cast(started_at#19 as date)) AS year#2748, month(cast(started_at#19 as date)) AS month#2749]
Filter ((isnotnull(started_at#19) AND ((isnull(ride_id#17) OR (ended_at#20 <= started_at#19)) OR (cast(end_station_id#24 as float) = cast(start_station_id#22 as float)))) AND (started_at#19 >= 2026-07-01 00:00:00))
WholeStageCodegen (1)
FileScan csv [ride_id#17,rideable_type#18,started_at#19,ended_at#20,start_station_name#21,start_station_id#22,end_station_name#23,end_station_id#24,start_lat#25,start_lng#26,end_lat#27,end_lng#28,member_casual#29] Batched: false, DataFilters: [isnotnull(started_at#19), ((isnull(ride_id#17) OR (ended_at#20 <= started_at#19)) OR (cast(end_s..., Format: CSV, Location: InMemoryFileIndex(5 paths)[s3a://rzvde-g9-chernyshev-miron/raw/citibike_data/202507/202507-citibi..., PartitionFilters: [], PushedFilters: [IsNotNull(started_at), GreaterThanOrEqual(started_at,2026-07-01 00:00:00.0)], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...
== Physical Plan ==
Execute InsertIntoHadoopFsRelationCommand (7)
+- WriteFiles (6)
+- * Project (5)
+- * Project (4)
+- * Project (3)
+- * Filter (2)
+- Scan csv (1)
(1) Scan csv
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g9-chernyshev-miron/raw/citibike_data/202507/202507-citibike-tripdata-part00.csv, ... 4 entries]
PushedFilters: [IsNotNull(started_at), GreaterThanOrEqual(started_at,2026-07-01 00:00:00.0)]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:double,start_lng:double,end_lat:double,end_lng:double,member_casual:string>
(2) Filter [codegen id : 1]
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Condition : ((isnotnull(started_at#19) AND ((isnull(ride_id#17) OR (ended_at#20 <= started_at#19)) OR (cast(end_station_id#24 as float) = cast(start_station_id#22 as float)))) AND (started_at#19 >= 2026-07-01 00:00:00))
(3) Project [codegen id : 1]
Output [17]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, cast(start_station_id#22 as float) AS start_station_id#1654, end_station_name#23, cast(end_station_id#24 as float) AS end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year(cast(started_at#19 as date)) AS year#2748, month(cast(started_at#19 as date)) AS month#2749, year(cast(started_at#19 as date)) AS year#2748, month(cast(started_at#19 as date)) AS month#2749]
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
(4) Project [codegen id : 1]
Output [19]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year#2748, month#2749, year#2748, month#2749, year(cast(started_at#19 as date)) AS year#2748, month(cast(started_at#19 as date)) AS month#2749]
Input [17]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year#2748, month#2749, year#2748, month#2749]
(5) Project [codegen id : 1]
Output [21]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year#2748, month#2749, year#2748, month#2749, year#2748, month#2749, year(cast(started_at#19 as date)) AS year#2748, month(cast(started_at#19 as date)) AS month#2749]
Input [19]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year#2748, month#2749, year#2748, month#2749, year#2748, month#2749]
(6) WriteFiles
Input [21]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, year#2748, month#2749, year#2748, month#2749, year#2748, month#2749, year#2748, month#2749]
(7) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: s3a://rzvde-g9-chernyshev-miron/raw_invalid/citibike_data/year=2026/month=Column<'add_months(to_date(2026-07-01), -1)'>, false, CSV, [header=true, path=s3a://rzvde-g9-chernyshev-miron/raw_invalid/citibike_data/year=2026/month=Column<'add_months(to_date(2026-07-01), -1)'>], Overwrite, [ride_id, rideable_type, started_at, ended_at, start_station_name, start_station_id, end_station_name, end_station_id, start_lat, start_lng, end_lat, end_lng, member_casual, year, month, year, month, year, month, year, month]