g9_chernyshev_miron_s2 - Details for Query 43

Submitted Time: 2026/08/20 09:27:09
Duration: 26 s
Succeeded Jobs: 80 81

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label=" CollectLimit "]; subgraph cluster2 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: 0 ms"; 3 [labelType="html" label=" Project "]; } 4 [labelType="html" label="Window spill size: 0.0 B"]; subgraph cluster5 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: 0 ms"; 6 [labelType="html" label="Sort sort time: 1.5 s peak memory: 224.0 MiB spill size: 0.0 B"]; } 7 [labelType="html" label="AQEShuffleRead number of partitions: 5 partition data size total (min, med, max (stageId: taskId)) 312.4 MiB (60.4 MiB, 63.1 MiB, 63.6 MiB (driver)) number of coalesced partitions: 5"]; 8 [labelType="html" label="Exchange shuffle records written: 4,988,053 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 1.5 s (0 ms, 203 ms, 244 ms (stage 116.0: task 327)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 980,488 local bytes read: 58.7 MiB fetch wait time: 0 ms remote bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 8 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 939.4 MiB (0.0 B, 129.1 MiB, 129.6 MiB (stage 116.0: task 326)) local merged bytes read: 0.0 B number of partitions: 200 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 297.7 MiB (0.0 B, 39.5 MiB, 43.7 MiB (stage 116.0: task 325))"]; subgraph cluster9 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n43.4 s (0 ms, 5.8 s, 6.6 s (stage 116.0: task 326))"; 10 [labelType="html" label=" Project "]; } 11 [labelType="html" label="Scan csv number of output rows: 4,988,053 number of files read: 5 metadata time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 118.0: task 333)) size of files read total (min, med, max (stageId: taskId)) 928.8 MiB (0.0 B, 928.8 MiB, 928.8 MiB (driver))"]; 1->0; 3->1; 4->3; 6->4; 7->6; 8->7; 10->8; 11->10; }

AdaptiveSparkPlan isFinalPlan=true

CollectLimit 21

Project [toprettystring(ride_id#17, Some(Etc/UTC)) AS toprettystring(ride_id)#1844, toprettystring(rideable_type#18, Some(Etc/UTC)) AS toprettystring(rideable_type)#1845, toprettystring(started_at#19, Some(Etc/UTC)) AS toprettystring(started_at)#1846, toprettystring(ended_at#20, Some(Etc/UTC)) AS toprettystring(ended_at)#1847, toprettystring(start_station_name#21, Some(Etc/UTC)) AS toprettystring(start_station_name)#1848, toprettystring(start_station_id#1654, Some(Etc/UTC)) AS toprettystring(start_station_id)#1849, toprettystring(end_station_name#23, Some(Etc/UTC)) AS toprettystring(end_station_name)#1850, toprettystring(end_station_id#1656, Some(Etc/UTC)) AS toprettystring(end_station_id)#1851, toprettystring(start_lat#25, Some(Etc/UTC)) AS toprettystring(start_lat)#1852, toprettystring(start_lng#26, Some(Etc/UTC)) AS toprettystring(start_lng)#1853, toprettystring(end_lat#27, Some(Etc/UTC)) AS toprettystring(end_lat)#1854, toprettystring(end_lng#28, Some(Etc/UTC)) AS toprettystring(end_lng)#1855, toprettystring(member_casual#29, Some(Etc/UTC)) AS toprettystring(member_casual)#1856, toprettystring(_start_station_ride_num#1809, Some(Etc/UTC)) AS toprettystring(_start_station_ride_num)#1857]

WholeStageCodegen (3)

Window [dense_rank(started_at#19) windowspecdefinition(start_station_name#21, started_at#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#1809], [start_station_name#21], [started_at#19 ASC NULLS FIRST]

Sort [start_station_name#21 ASC NULLS FIRST, started_at#19 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

AQEShuffleRead coalesced

Exchange hashpartitioning(start_station_name#21, 200), ENSURE_REQUIREMENTS, [plan_id=1646]

Project [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, cast(start_station_id#22 as float) AS start_station_id#1654, end_station_name#23, cast(end_station_id#24 as float) AS end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]

WholeStageCodegen (1)

FileScan csv [ride_id#17,rideable_type#18,started_at#19,ended_at#20,start_station_name#21,start_station_id#22,end_station_name#23,end_station_id#24,start_lat#25,start_lng#26,end_lat#27,end_lng#28,member_casual#29] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(5 paths)[s3a://rzvde-g9-chernyshev-miron/raw/citibike_data/202507/202507-citibi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...

Details

== Physical Plan ==
AdaptiveSparkPlan (16)
+- == Final Plan ==
   CollectLimit (9)
   +- * Project (8)
      +- Window (7)
         +- * Sort (6)
            +- AQEShuffleRead (5)
               +- ShuffleQueryStage (4), Statistics(sizeInBytes=939.4 MiB, rowCount=4.99E+6)
                  +- Exchange (3)
                     +- * Project (2)
                        +- Scan csv  (1)
+- == Initial Plan ==
   CollectLimit (15)
   +- Project (14)
      +- Window (13)
         +- Sort (12)
            +- Exchange (11)
               +- Project (10)
                  +- Scan csv  (1)


(1) Scan csv 
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g9-chernyshev-miron/raw/citibike_data/202507/202507-citibike-tripdata-part00.csv, ... 4 entries]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:double,start_lng:double,end_lat:double,end_lng:double,member_casual:string>

(2) Project [codegen id : 1]
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, cast(start_station_id#22 as float) AS start_station_id#1654, end_station_name#23, cast(end_station_id#24 as float) AS end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]

(3) Exchange
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: hashpartitioning(start_station_name#21, 200), ENSURE_REQUIREMENTS, [plan_id=1646]

(4) ShuffleQueryStage
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: 0

(5) AQEShuffleRead
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: coalesced

(6) Sort [codegen id : 2]
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: [start_station_name#21 ASC NULLS FIRST, started_at#19 ASC NULLS FIRST], false, 0

(7) Window
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: [dense_rank(started_at#19) windowspecdefinition(start_station_name#21, started_at#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#1809], [start_station_name#21], [started_at#19 ASC NULLS FIRST]

(8) Project [codegen id : 3]
Output [14]: [toprettystring(ride_id#17, Some(Etc/UTC)) AS toprettystring(ride_id)#1844, toprettystring(rideable_type#18, Some(Etc/UTC)) AS toprettystring(rideable_type)#1845, toprettystring(started_at#19, Some(Etc/UTC)) AS toprettystring(started_at)#1846, toprettystring(ended_at#20, Some(Etc/UTC)) AS toprettystring(ended_at)#1847, toprettystring(start_station_name#21, Some(Etc/UTC)) AS toprettystring(start_station_name)#1848, toprettystring(start_station_id#1654, Some(Etc/UTC)) AS toprettystring(start_station_id)#1849, toprettystring(end_station_name#23, Some(Etc/UTC)) AS toprettystring(end_station_name)#1850, toprettystring(end_station_id#1656, Some(Etc/UTC)) AS toprettystring(end_station_id)#1851, toprettystring(start_lat#25, Some(Etc/UTC)) AS toprettystring(start_lat)#1852, toprettystring(start_lng#26, Some(Etc/UTC)) AS toprettystring(start_lng)#1853, toprettystring(end_lat#27, Some(Etc/UTC)) AS toprettystring(end_lat)#1854, toprettystring(end_lng#28, Some(Etc/UTC)) AS toprettystring(end_lng)#1855, toprettystring(member_casual#29, Some(Etc/UTC)) AS toprettystring(member_casual)#1856, toprettystring(_start_station_ride_num#1809, Some(Etc/UTC)) AS toprettystring(_start_station_ride_num)#1857]
Input [14]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, _start_station_ride_num#1809]

(9) CollectLimit
Input [14]: [toprettystring(ride_id)#1844, toprettystring(rideable_type)#1845, toprettystring(started_at)#1846, toprettystring(ended_at)#1847, toprettystring(start_station_name)#1848, toprettystring(start_station_id)#1849, toprettystring(end_station_name)#1850, toprettystring(end_station_id)#1851, toprettystring(start_lat)#1852, toprettystring(start_lng)#1853, toprettystring(end_lat)#1854, toprettystring(end_lng)#1855, toprettystring(member_casual)#1856, toprettystring(_start_station_ride_num)#1857]
Arguments: 21

(10) Project
Output [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, cast(start_station_id#22 as float) AS start_station_id#1654, end_station_name#23, cast(end_station_id#24 as float) AS end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#22, end_station_name#23, end_station_id#24, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]

(11) Exchange
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: hashpartitioning(start_station_name#21, 200), ENSURE_REQUIREMENTS, [plan_id=1633]

(12) Sort
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: [start_station_name#21 ASC NULLS FIRST, started_at#19 ASC NULLS FIRST], false, 0

(13) Window
Input [13]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29]
Arguments: [dense_rank(started_at#19) windowspecdefinition(start_station_name#21, started_at#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#1809], [start_station_name#21], [started_at#19 ASC NULLS FIRST]

(14) Project
Output [14]: [toprettystring(ride_id#17, Some(Etc/UTC)) AS toprettystring(ride_id)#1844, toprettystring(rideable_type#18, Some(Etc/UTC)) AS toprettystring(rideable_type)#1845, toprettystring(started_at#19, Some(Etc/UTC)) AS toprettystring(started_at)#1846, toprettystring(ended_at#20, Some(Etc/UTC)) AS toprettystring(ended_at)#1847, toprettystring(start_station_name#21, Some(Etc/UTC)) AS toprettystring(start_station_name)#1848, toprettystring(start_station_id#1654, Some(Etc/UTC)) AS toprettystring(start_station_id)#1849, toprettystring(end_station_name#23, Some(Etc/UTC)) AS toprettystring(end_station_name)#1850, toprettystring(end_station_id#1656, Some(Etc/UTC)) AS toprettystring(end_station_id)#1851, toprettystring(start_lat#25, Some(Etc/UTC)) AS toprettystring(start_lat)#1852, toprettystring(start_lng#26, Some(Etc/UTC)) AS toprettystring(start_lng)#1853, toprettystring(end_lat#27, Some(Etc/UTC)) AS toprettystring(end_lat)#1854, toprettystring(end_lng#28, Some(Etc/UTC)) AS toprettystring(end_lng)#1855, toprettystring(member_casual#29, Some(Etc/UTC)) AS toprettystring(member_casual)#1856, toprettystring(_start_station_ride_num#1809, Some(Etc/UTC)) AS toprettystring(_start_station_ride_num)#1857]
Input [14]: [ride_id#17, rideable_type#18, started_at#19, ended_at#20, start_station_name#21, start_station_id#1654, end_station_name#23, end_station_id#1656, start_lat#25, start_lng#26, end_lat#27, end_lng#28, member_casual#29, _start_station_ride_num#1809]

(15) CollectLimit
Input [14]: [toprettystring(ride_id)#1844, toprettystring(rideable_type)#1845, toprettystring(started_at)#1846, toprettystring(ended_at)#1847, toprettystring(start_station_name)#1848, toprettystring(start_station_id)#1849, toprettystring(end_station_name)#1850, toprettystring(end_station_id)#1851, toprettystring(start_lat)#1852, toprettystring(start_lng)#1853, toprettystring(end_lat)#1854, toprettystring(end_lng)#1855, toprettystring(member_casual)#1856, toprettystring(_start_station_ride_num)#1857]
Arguments: 21

(16) AdaptiveSparkPlan
Output [14]: [toprettystring(ride_id)#1844, toprettystring(rideable_type)#1845, toprettystring(started_at)#1846, toprettystring(ended_at)#1847, toprettystring(start_station_name)#1848, toprettystring(start_station_id)#1849, toprettystring(end_station_name)#1850, toprettystring(end_station_id)#1851, toprettystring(start_lat)#1852, toprettystring(start_lng)#1853, toprettystring(end_lat)#1854, toprettystring(end_lng)#1855, toprettystring(member_casual)#1856, toprettystring(_start_station_ride_num)#1857]
Arguments: isFinalPlan=true