digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
1 [labelType="html" label="<b>Execute InsertIntoHadoopFsRelationCommand</b><br><br>task commit time total (min, med, max (stageId: taskId))<br>942 ms (411 ms, 531 ms, 531 ms (stage 134.0: task 134))<br>number of written files: 2<br>job commit time: 1.8 s<br>number of output rows: 30,774<br>number of dynamic part: 1<br>written output: 1364.5 KiB"];
2 [labelType="html" label="<br><b>WriteFiles</b><br><br>"];
subgraph cluster3 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n1.8 s (918 ms, 918 ms, 918 ms (stage 134.0: task 135))";
4 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>13 ms (6 ms, 7 ms, 7 ms (stage 134.0: task 135))<br>peak memory total (min, med, max (stageId: taskId))<br>65.0 MiB (32.5 MiB, 32.5 MiB, 32.5 MiB (stage 134.0: task 135))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 134.0: task 135))"];
5 [labelType="html" label="<br><b>Project</b><br><br>"];
6 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 30,774"];
}
7 [labelType="html" label="<b>Window</b><br><br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 134.0: task 135))"];
subgraph cluster8 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n240 ms (116 ms, 124 ms, 124 ms (stage 134.0: task 134))";
9 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>19 ms (9 ms, 10 ms, 10 ms (stage 134.0: task 135))<br>peak memory total (min, med, max (stageId: taskId))<br>65.0 MiB (32.5 MiB, 32.5 MiB, 32.5 MiB (stage 134.0: task 135))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 134.0: task 135))"];
}
10 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 2<br>partition data size total (min, med, max (stageId: taskId))<br>2.4 MiB (1223.8 KiB, 1234.2 KiB, 1234.2 KiB (driver))<br>number of coalesced partitions: 2"];
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 31,257<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>58 ms (0 ms, 26 ms, 32 ms (stage 132.0: task 133))<br>remote merged bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 134.0: task 135))<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 134.0: task 135))<br>remote merged blocks fetched: 0<br>records read: 31,257<br>local bytes read total (min, med, max (stageId: taskId))<br>2.3 MiB (1165.2 KiB, 1174.1 KiB, 1174.1 KiB (stage 134.0: task 135))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 134.0: task 135))<br>remote bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 134.0: task 135))<br>merged fetch fallback count: 0<br>local blocks read: 4<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>10.9 MiB (0.0 B, 1726.4 KiB, 9.2 MiB (stage 132.0: task 132))<br>local merged bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 134.0: task 135))<br>number of partitions: 200<br>remote reqs duration total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 134.0: task 135))<br>remote bytes read to disk total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 134.0: task 135))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.3 MiB (0.0 B, 413.5 KiB, 1925.7 KiB (stage 132.0: task 132))"];
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n640 ms (0 ms, 274 ms, 366 ms (stage 132.0: task 133))";
13 [labelType="html" label="<br><b>Project</b><br><br>"];
}
14 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 31,257<br>number of files read: 1<br>metadata time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 134.0: task 135))<br>size of files read total (min, med, max (stageId: taskId))<br>5.8 MiB (0.0 B, 0.0 B, 5.8 MiB (driver))"];
1->0;
2->1;
4->2;
5->4;
6->5;
7->6;
9->7;
10->9;
11->10;
13->11;
14->13;
}
15
AdaptiveSparkPlan isFinalPlan=true
Execute InsertIntoHadoopFsRelationCommand s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data, false, [year#9582, month#9583], Parquet, [partitionOverwriteMode=dynamic, __partition_columns=["year","month"], path=s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data/], Overwrite, [ride_id, rideable_type, started_at, ended_at, start_station_name, start_station_id, end_station_name, end_station_id, start_lat, start_lng, end_lat, end_lng, member_casual, _source_file, _processed_dttm, _start_station_ride_num, year, month]
WriteFiles
Sort [year#9582 ASC NULLS FIRST, month#9583 ASC NULLS FIRST], false, 0
Project [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, _source_file#9578, 2026-04-08 09:23:31.844159 AS _processed_dttm#9579, _start_station_ride_num#9581, year#9582, month#9583]
Filter (isnotnull(valid_time#9576) AND ((valid_ride_id#9575 AND valid_time#9576) AND valid_station#9577))
WholeStageCodegen (3)
Window [row_number() windowspecdefinition(start_station_id#9487, started_at#9484 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#9581], [start_station_id#9487], [started_at#9484 ASC NULLS FIRST]
Sort [start_station_id#9487 ASC NULLS FIRST, started_at#9484 ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
AQEShuffleRead coalesced
Exchange hashpartitioning(start_station_id#9487, 200), ENSURE_REQUIREMENTS, [plan_id=1959]
Project [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, cast(start_station_id#9487 as double) AS start_station_id#9589, end_station_name#9488, cast(end_station_id#9489 as double) AS end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, isnotnull(ride_id#9482) AS valid_ride_id#9575, (ended_at#9485 > started_at#9484) AS valid_time#9576, (((isnotnull(end_station_id#9489) AND isnotnull(start_station_id#9487)) AND NOT (end_station_id#9489 = start_station_id#9487)) <=> true) AS valid_station#9577, input_file_name() AS _source_file#9578, year(cast(started_at#9484 as date)) AS year#9582, month(cast(started_at#9484 as date)) AS month#9583, start_station_id#9487, started_at#9484]
WholeStageCodegen (1)
FileScan csv [ride_id#9482,rideable_type#9483,started_at#9484,ended_at#9485,start_station_name#9486,start_station_id#9487,end_station_name#9488,end_station_id#9489,start_lat#9490,start_lng#9491,end_lat#9492,end_lng#9493,member_casual#9494] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[s3a://rzvde-g8-kirsanov-dmitry/raw/citibike_data/202502/202502-citibik..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...
== Physical Plan ==
AdaptiveSparkPlan (22)
+- == Final Plan ==
Execute InsertIntoHadoopFsRelationCommand (12)
+- WriteFiles (11)
+- * Sort (10)
+- * Project (9)
+- * Filter (8)
+- Window (7)
+- * Sort (6)
+- AQEShuffleRead (5)
+- ShuffleQueryStage (4), Statistics(sizeInBytes=10.9 MiB, rowCount=3.13E+4)
+- Exchange (3)
+- * Project (2)
+- Scan csv (1)
+- == Initial Plan ==
Execute InsertIntoHadoopFsRelationCommand (21)
+- WriteFiles (20)
+- Sort (19)
+- Project (18)
+- Filter (17)
+- Window (16)
+- Sort (15)
+- Exchange (14)
+- Project (13)
+- Scan csv (1)
(1) Scan csv
Output [13]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9487, end_station_name#9488, end_station_id#9489, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g8-kirsanov-dmitry/raw/citibike_data/202502/202502-citibike-tripdata-part00.csv]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:double,start_lng:double,end_lat:double,end_lng:double,member_casual:string>
(2) Project [codegen id : 1]
Output [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, cast(start_station_id#9487 as double) AS start_station_id#9589, end_station_name#9488, cast(end_station_id#9489 as double) AS end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, isnotnull(ride_id#9482) AS valid_ride_id#9575, (ended_at#9485 > started_at#9484) AS valid_time#9576, (((isnotnull(end_station_id#9489) AND isnotnull(start_station_id#9487)) AND NOT (end_station_id#9489 = start_station_id#9487)) <=> true) AS valid_station#9577, input_file_name() AS _source_file#9578, year(cast(started_at#9484 as date)) AS year#9582, month(cast(started_at#9484 as date)) AS month#9583, start_station_id#9487, started_at#9484]
Input [13]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9487, end_station_name#9488, end_station_id#9489, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494]
(3) Exchange
Input [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484]
Arguments: hashpartitioning(start_station_id#9487, 200), ENSURE_REQUIREMENTS, [plan_id=1959]
(4) ShuffleQueryStage
Output [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484]
Arguments: 0
(5) AQEShuffleRead
Input [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484]
Arguments: coalesced
(6) Sort [codegen id : 2]
Input [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484]
Arguments: [start_station_id#9487 ASC NULLS FIRST, started_at#9484 ASC NULLS FIRST], false, 0
(7) Window
Input [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484]
Arguments: [row_number() windowspecdefinition(start_station_id#9487, started_at#9484 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#9581], [start_station_id#9487], [started_at#9484 ASC NULLS FIRST]
(8) Filter [codegen id : 3]
Input [22]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484, _start_station_ride_num#9581]
Condition : (isnotnull(valid_time#9576) AND ((valid_ride_id#9575 AND valid_time#9576) AND valid_station#9577))
(9) Project [codegen id : 3]
Output [18]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, _source_file#9578, 2026-04-08 09:23:31.844159 AS _processed_dttm#9579, _start_station_ride_num#9581, year#9582, month#9583]
Input [22]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484, _start_station_ride_num#9581]
(10) Sort [codegen id : 3]
Input [18]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, _source_file#9578, _processed_dttm#9579, _start_station_ride_num#9581, year#9582, month#9583]
Arguments: [year#9582 ASC NULLS FIRST, month#9583 ASC NULLS FIRST], false, 0
(11) WriteFiles
Input [18]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, _source_file#9578, _processed_dttm#9579, _start_station_ride_num#9581, year#9582, month#9583]
(12) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data, false, [year#9582, month#9583], Parquet, [partitionOverwriteMode=dynamic, __partition_columns=["year","month"], path=s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data/], Overwrite, [ride_id, rideable_type, started_at, ended_at, start_station_name, start_station_id, end_station_name, end_station_id, start_lat, start_lng, end_lat, end_lng, member_casual, _source_file, _processed_dttm, _start_station_ride_num, year, month]
(13) Project
Output [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, cast(start_station_id#9487 as double) AS start_station_id#9589, end_station_name#9488, cast(end_station_id#9489 as double) AS end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, isnotnull(ride_id#9482) AS valid_ride_id#9575, (ended_at#9485 > started_at#9484) AS valid_time#9576, (((isnotnull(end_station_id#9489) AND isnotnull(start_station_id#9487)) AND NOT (end_station_id#9489 = start_station_id#9487)) <=> true) AS valid_station#9577, input_file_name() AS _source_file#9578, year(cast(started_at#9484 as date)) AS year#9582, month(cast(started_at#9484 as date)) AS month#9583, start_station_id#9487, started_at#9484]
Input [13]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9487, end_station_name#9488, end_station_id#9489, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494]
(14) Exchange
Input [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484]
Arguments: hashpartitioning(start_station_id#9487, 200), ENSURE_REQUIREMENTS, [plan_id=1943]
(15) Sort
Input [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484]
Arguments: [start_station_id#9487 ASC NULLS FIRST, started_at#9484 ASC NULLS FIRST], false, 0
(16) Window
Input [21]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484]
Arguments: [row_number() windowspecdefinition(start_station_id#9487, started_at#9484 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#9581], [start_station_id#9487], [started_at#9484 ASC NULLS FIRST]
(17) Filter
Input [22]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484, _start_station_ride_num#9581]
Condition : (isnotnull(valid_time#9576) AND ((valid_ride_id#9575 AND valid_time#9576) AND valid_station#9577))
(18) Project
Output [18]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, _source_file#9578, 2026-04-08 09:23:31.844159 AS _processed_dttm#9579, _start_station_ride_num#9581, year#9582, month#9583]
Input [22]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, valid_ride_id#9575, valid_time#9576, valid_station#9577, _source_file#9578, year#9582, month#9583, start_station_id#9487, started_at#9484, _start_station_ride_num#9581]
(19) Sort
Input [18]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, _source_file#9578, _processed_dttm#9579, _start_station_ride_num#9581, year#9582, month#9583]
Arguments: [year#9582 ASC NULLS FIRST, month#9583 ASC NULLS FIRST], false, 0
(20) WriteFiles
Input [18]: [ride_id#9482, rideable_type#9483, started_at#9484, ended_at#9485, start_station_name#9486, start_station_id#9589, end_station_name#9488, end_station_id#9591, start_lat#9490, start_lng#9491, end_lat#9492, end_lng#9493, member_casual#9494, _source_file#9578, _processed_dttm#9579, _start_station_ride_num#9581, year#9582, month#9583]
(21) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data, false, [year#9582, month#9583], Parquet, [partitionOverwriteMode=dynamic, __partition_columns=["year","month"], path=s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data/], Overwrite, [ride_id, rideable_type, started_at, ended_at, start_station_name, start_station_id, end_station_name, end_station_id, start_lat, start_lng, end_lat, end_lng, member_casual, _source_file, _processed_dttm, _start_station_ride_num, year, month]
(22) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true