digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 2 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 1 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 8<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>3 ms (0 ms, 0 ms, 0 ms (stage 55.0: task 164))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 8<br>local bytes read: 576.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 8<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>256.0 B (0.0 B, 32.0 B, 32.0 B (stage 55.0: task 158))<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>576.0 B (0.0 B, 72.0 B, 72.0 B (stage 55.0: task 158))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n21.7 s (1.4 s, 3.0 s, 3.2 s (stage 55.0: task 159))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>21.7 s (1.4 s, 3.0 s, 3.2 s (stage 55.0: task 159))<br>number of output rows: 8"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
}
7 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 4,988,053<br>number of files read: 5<br>metadata time: 7 ms<br>size of files read: 928.8 MiB"];
2->0;
3->2;
5->3;
6->5;
7->6;
}
8
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[stddev(cast(duration_sec#891L as double))])
WholeStageCodegen (2)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=712]
HashAggregate(keys=[], functions=[partial_stddev(cast(duration_sec#891L as double))])
Project [(unix_timestamp(ended_at#20, yyyy-MM-dd HH:mm:ss, Some(Etc/UTC), false) - unix_timestamp(started_at#19, yyyy-MM-dd HH:mm:ss, Some(Etc/UTC), false)) AS duration_sec#891L]
WholeStageCodegen (1)
FileScan csv [started_at#19,ended_at#20] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(5 paths)[s3a://rzvde-g9-chernyshev-miron/raw/citibike_data/202507/202507-citibi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<started_at:timestamp,ended_at:timestamp>
== Physical Plan ==
AdaptiveSparkPlan (11)
+- == Final Plan ==
* HashAggregate (6)
+- ShuffleQueryStage (5), Statistics(sizeInBytes=256.0 B, rowCount=8)
+- Exchange (4)
+- * HashAggregate (3)
+- * Project (2)
+- Scan csv (1)
+- == Initial Plan ==
HashAggregate (10)
+- Exchange (9)
+- HashAggregate (8)
+- Project (7)
+- Scan csv (1)
(1) Scan csv
Output [2]: [started_at#19, ended_at#20]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g9-chernyshev-miron/raw/citibike_data/202507/202507-citibike-tripdata-part00.csv, ... 4 entries]
ReadSchema: struct<started_at:timestamp,ended_at:timestamp>
(2) Project [codegen id : 1]
Output [1]: [(unix_timestamp(ended_at#20, yyyy-MM-dd HH:mm:ss, Some(Etc/UTC), false) - unix_timestamp(started_at#19, yyyy-MM-dd HH:mm:ss, Some(Etc/UTC), false)) AS duration_sec#891L]
Input [2]: [started_at#19, ended_at#20]
(3) HashAggregate [codegen id : 1]
Input [1]: [duration_sec#891L]
Keys: []
Functions [1]: [partial_stddev(cast(duration_sec#891L as double))]
Aggregate Attributes [3]: [n#1007, avg#1008, m2#1009]
Results [3]: [n#1012, avg#1013, m2#1014]
(4) Exchange
Input [3]: [n#1012, avg#1013, m2#1014]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=712]
(5) ShuffleQueryStage
Output [3]: [n#1012, avg#1013, m2#1014]
Arguments: 0
(6) HashAggregate [codegen id : 2]
Input [3]: [n#1012, avg#1013, m2#1014]
Keys: []
Functions [1]: [stddev(cast(duration_sec#891L as double))]
Aggregate Attributes [1]: [stddev(cast(duration_sec#891L as double))#989]
Results [1]: [toprettystring(stddev(cast(duration_sec#891L as double))#989, Some(Etc/UTC)) AS toprettystring(stddev(duration_sec))#1017]
(7) Project
Output [1]: [(unix_timestamp(ended_at#20, yyyy-MM-dd HH:mm:ss, Some(Etc/UTC), false) - unix_timestamp(started_at#19, yyyy-MM-dd HH:mm:ss, Some(Etc/UTC), false)) AS duration_sec#891L]
Input [2]: [started_at#19, ended_at#20]
(8) HashAggregate
Input [1]: [duration_sec#891L]
Keys: []
Functions [1]: [partial_stddev(cast(duration_sec#891L as double))]
Aggregate Attributes [3]: [n#1007, avg#1008, m2#1009]
Results [3]: [n#1012, avg#1013, m2#1014]
(9) Exchange
Input [3]: [n#1012, avg#1013, m2#1014]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=700]
(10) HashAggregate
Input [3]: [n#1012, avg#1013, m2#1014]
Keys: []
Functions [1]: [stddev(cast(duration_sec#891L as double))]
Aggregate Attributes [1]: [stddev(cast(duration_sec#891L as double))#989]
Results [1]: [toprettystring(stddev(cast(duration_sec#891L as double))#989, Some(Etc/UTC)) AS toprettystring(stddev(duration_sec))#1017]
(11) AdaptiveSparkPlan
Output [1]: [toprettystring(stddev(duration_sec))#1017]
Arguments: isFinalPlan=true