digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
1 [labelType="html" label="<br><b>TakeOrderedAndProject</b><br><br>"];
2 [labelType="html" label="<b>Window</b><br><br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 130.0: task 369))"];
subgraph cluster3 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n19.3 s (3.5 s, 3.6 s, 4.3 s (stage 130.0: task 370))";
4 [labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>9.3 s (1.8 s, 1.8 s, 2.0 s (stage 130.0: task 370))<br>peak memory total (min, med, max (stageId: taskId))<br>1216.0 MiB (224.0 MiB, 256.0 MiB, 256.0 MiB (stage 130.0: task 370))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 130.0: task 369))"];
}
5 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 5<br>partition data size total (min, med, max (stageId: taskId))<br>312.4 MiB (60.4 MiB, 63.1 MiB, 63.6 MiB (driver))<br>number of coalesced partitions: 5"];
6 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 4,988,053<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.8 s (0 ms, 173 ms, 283 ms (stage 128.0: task 363))<br>remote merged bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 130.0: task 369))<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 130.0: task 369))<br>remote merged blocks fetched: 0<br>records read: 4,988,053<br>local bytes read total (min, med, max (stageId: taskId))<br>297.7 MiB (57.5 MiB, 60.2 MiB, 60.8 MiB (stage 130.0: task 370))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 130.0: task 369))<br>remote bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 130.0: task 369))<br>merged fetch fallback count: 0<br>local blocks read: 40<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>939.4 MiB (0.0 B, 117.7 MiB, 129.6 MiB (stage 128.0: task 362))<br>local merged bytes read total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 130.0: task 369))<br>number of partitions: 200<br>remote reqs duration total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 130.0: task 369))<br>remote bytes read to disk total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 130.0: task 369))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>297.7 MiB (0.0 B, 37.1 MiB, 43.7 MiB (stage 128.0: task 361))"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n43.9 s (0 ms, 4.8 s, 6.5 s (stage 128.0: task 363))";
8 [labelType="html" label="<br><b>Project</b><br><br>"];
}
9 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 4,988,053<br>number of files read: 5<br>metadata time total (min, med, max (stageId: taskId))<br>6 ms (0 ms, 0 ms, 6 ms (driver))<br>size of files read total (min, med, max (stageId: taskId))<br>928.8 MiB (0.0 B, 0.0 B, 928.8 MiB (driver))"];
1->0;
2->1;
4->2;
5->4;
6->5;
8->6;
9->8;
}
10
AdaptiveSparkPlan isFinalPlan=true
TakeOrderedAndProject(limit=21, orderBy=[started_at#19 ASC NULLS FIRST], output=[toprettystring(ride_id)#2230,toprettystring(rideable_type)#2231,toprettystring(started_at)#2232,toprettystring(ended_at)#2233,toprettystring(start_station_name)#2234,toprettystring(start_station_id)#2235,toprettystring(end_station_name)#2236,toprettystring(end_station_id)#2237,toprettystring(start_lat)#2238,toprettystring(start_lng)#2239,toprettystring(end_lat)#2240,toprettystring(end_lng)#2241,toprettystring(member_casual)#2242,toprettystring(_start_station_ride_num)#2243])
Window [dense_rank(started_at#19) windowspecdefinition(start_station_name#21, started_at#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#2195], [start_station_name#21], [started_at#19 ASC NULLS FIRST]
Sort [start_station_name#21 ASC NULLS FIRST, started_at#19 ASC NULLS FIRST], false, 0
WholeStageCodegen (2)
AQEShuffleRead coalesced
Exchange hashpartitioning(start_station_name#21, 200), ENSURE_REQUIREMENTS, [plan_id=1913]
Project [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, cast(start_station_id#22 as float) AS start_station_id#1654, end_station_name#23, cast(end_station_id#24 as float) AS end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
WholeStageCodegen (1)
FileScan csv [ride_id#17,rideable_type#18,started_at#19,ended_at#20,start_station_name#21,start_station_id#22,end_station_name#23,end_station_id#24,start_lat#25,start_lng#26,end_lat#27,end_lng#28,member_casual#29] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(5 paths)[s3a://rzvde-g9-chernyshev-miron/raw/citibike_data/202507/202507-citibi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...
== Physical Plan ==
AdaptiveSparkPlan (14)
+- == Final Plan ==
TakeOrderedAndProject (8)
+- Window (7)
+- * Sort (6)
+- AQEShuffleRead (5)
+- ShuffleQueryStage (4), Statistics(sizeInBytes=939.4 MiB, rowCount=4.99E+6)
+- Exchange (3)
+- * Project (2)
+- Scan csv (1)
+- == Initial Plan ==
TakeOrderedAndProject (13)
+- Window (12)
+- Sort (11)
+- Exchange (10)
+- Project (9)
+- Scan csv (1)
(1) Scan csv
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g9-chernyshev-miron/raw/citibike_data/202507/202507-citibike-tripdata-part00.csv, ... 4 entries]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:double,start_lng:double,end_lat:double,end_lng:double,member_casual:string>
(2) Project [codegen id : 1]
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, cast(start_station_id#22 as float) AS start_station_id#1654, end_station_name#23, cast(end_station_id#24 as float) AS end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
(3) Exchange
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: hashpartitioning(start_station_name#21, 200), ENSURE_REQUIREMENTS, [plan_id=1913]
(4) ShuffleQueryStage
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: 0
(5) AQEShuffleRead
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: coalesced
(6) Sort [codegen id : 2]
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: [start_station_name#21 ASC NULLS FIRST, started_at#19 ASC NULLS FIRST], false, 0
(7) Window
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: [dense_rank(started_at#19) windowspecdefinition(start_station_name#21, started_at#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#2195], [start_station_name#21], [started_at#19 ASC NULLS FIRST]
(8) TakeOrderedAndProject
Input [14]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, _start_station_ride_num#2195]
Arguments: 21, [started_at#19 ASC NULLS FIRST], [toprettystring(ride_id#17, Some(Etc/UTC)) AS toprettystring(ride_id)#2230, toprettystring(rideable_type#18, Some(Etc/UTC)) AS toprettystring(rideable_type)#2231, toprettystring(started_at#19, Some(Etc/UTC)) AS toprettystring(started_at)#2232, toprettystring(ended_at#20, Some(Etc/UTC)) AS toprettystring(ended_at)#2233, toprettystring(start_station_name#21, Some(Etc/UTC)) AS toprettystring(start_station_name)#2234, toprettystring(start_station_id#1654, Some(Etc/UTC)) AS toprettystring(start_station_id)#2235, toprettystring(end_station_name#23, Some(Etc/UTC)) AS toprettystring(end_station_name)#2236, toprettystring(end_station_id#1656, Some(Etc/UTC)) AS toprettystring(end_station_id)#2237, toprettystring(start_lat#25, Some(Etc/UTC)) AS toprettystring(start_lat)#2238, toprettystring(start_lng#26, Some(Etc/UTC)) AS toprettystring(start_lng)#2239, toprettystring(end_lat#27, Some(Etc/UTC)) AS toprettystring(end_lat)#2240, toprettystring(end_lng#28, Some(Etc/UTC)) AS toprettystring(end_lng)#2241, toprettystring(member_casual#29, Some(Etc/UTC)) AS toprettystring(member_casual)#2242, toprettystring(_start_station_ride_num#2195, Some(Etc/UTC)) AS toprettystring(_start_station_ride_num)#2243]
(9) Project
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, cast(start_station_id#22 as float) AS start_station_id#1654, end_station_name#23, cast(end_station_id#24 as float) AS end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
(10) Exchange
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: hashpartitioning(start_station_name#21, 200), ENSURE_REQUIREMENTS, [plan_id=1901]
(11) Sort
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: [start_station_name#21 ASC NULLS FIRST, started_at#19 ASC NULLS FIRST], false, 0
(12) Window
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: [dense_rank(started_at#19) windowspecdefinition(start_station_name#21, started_at#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#2195], [start_station_name#21], [started_at#19 ASC NULLS FIRST]
(13) TakeOrderedAndProject
Input [14]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, _start_station_ride_num#2195]
Arguments: 21, [started_at#19 ASC NULLS FIRST], [toprettystring(ride_id#17, Some(Etc/UTC)) AS toprettystring(ride_id)#2230, toprettystring(rideable_type#18, Some(Etc/UTC)) AS toprettystring(rideable_type)#2231, toprettystring(started_at#19, Some(Etc/UTC)) AS toprettystring(started_at)#2232, toprettystring(ended_at#20, Some(Etc/UTC)) AS toprettystring(ended_at)#2233, toprettystring(start_station_name#21, Some(Etc/UTC)) AS toprettystring(start_station_name)#2234, toprettystring(start_station_id#1654, Some(Etc/UTC)) AS toprettystring(start_station_id)#2235, toprettystring(end_station_name#23, Some(Etc/UTC)) AS toprettystring(end_station_name)#2236, toprettystring(end_station_id#1656, Some(Etc/UTC)) AS toprettystring(end_station_id)#2237, toprettystring(start_lat#25, Some(Etc/UTC)) AS toprettystring(start_lat)#2238, toprettystring(start_lng#26, Some(Etc/UTC)) AS toprettystring(start_lng)#2239, toprettystring(end_lat#27, Some(Etc/UTC)) AS toprettystring(end_lat)#2240, toprettystring(end_lng#28, Some(Etc/UTC)) AS toprettystring(end_lng)#2241, toprettystring(member_casual#29, Some(Etc/UTC)) AS toprettystring(member_casual)#2242, toprettystring(_start_station_ride_num#2195, Some(Etc/UTC)) AS toprettystring(_start_station_ride_num)#2243]
(14) AdaptiveSparkPlan
Output [14]: [toprettystring(ride_id)#2230, toprettystring(rideable_type)#2231, toprettystring(started_at)#2232, toprettystring(ended_at)#2233, toprettystring(start_station_name)#2234, toprettystring(start_station_id)#2235, toprettystring(end_station_name)#2236, toprettystring(end_station_id)#2237, toprettystring(start_lat)#2238, toprettystring(start_lng)#2239, toprettystring(end_lat)#2240, toprettystring(end_lng)#2241, toprettystring(member_casual)#2242, toprettystring(_start_station_ride_num)#2243]
Arguments: isFinalPlan=true