digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 2<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>1 ms (0 ms, 0 ms, 0 ms (stage 107.0: task 107))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 2<br>local bytes read: 118.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 2<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>32.0 B (0.0 B, 16.0 B, 16.0 B (stage 107.0: task 107))<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>118.0 B (0.0 B, 59.0 B, 59.0 B (stage 107.0: task 107))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n8 ms (4 ms, 4 ms, 4 ms (stage 107.0: task 107))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>6 ms (3 ms, 3 ms, 3 ms (stage 107.0: task 107))<br>number of output rows: 2"];
}
6 [labelType="html" label="<b>InMemoryTableScan</b><br><br>number of output rows: 1,219,444"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 107.0: task 107))";
8 [labelType="html" label="<br><b>Project</b><br><br>"];
}
9 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 0"];
2->0;
3->2;
5->3;
6->5;
8->6;
9->8;
}
10
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (2)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2651]
HashAggregate(keys=[], functions=[partial_count(1)])
WholeStageCodegen (1)
InMemoryTableScan
Project [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
WholeStageCodegen (1)
FileScan csv [ride_id#128,rideable_type#129,started_at#130,ended_at#131,start_station_name#132,start_station_id#133,end_station_name#134,end_station_id#135,start_lat#136,start_lng#137,end_lat#138,end_lng#139,member_casual#140,_corrupt_record#141] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
== Physical Plan ==
AdaptiveSparkPlan (13)
+- == Final Plan ==
* HashAggregate (9)
+- ShuffleQueryStage (8), Statistics(sizeInBytes=32.0 B, rowCount=2)
+- Exchange (7)
+- * HashAggregate (6)
+- TableCacheQueryStage (5), Statistics(sizeInBytes=322.0 MiB, rowCount=1.22E+6)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Project (4)
+- Scan csv (3)
+- == Initial Plan ==
HashAggregate (12)
+- Exchange (11)
+- HashAggregate (10)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Project (4)
+- Scan csv (3)
(1) InMemoryTableScan
Output: []
(2) InMemoryRelation
Arguments: [ride_id#11472, rideable_type#11473, started_at#11474, ended_at#11475, start_station_name#11476, start_station_id#11477, end_station_name#11478, end_station_id#11479, start_lat#11480, start_lng#11481, end_lat#11482, end_lng#11483, member_casual#11484, _corrupt_record#11485, _source_file#11500], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@28f1e8bc,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
+- FileScan csv [ride_id#128,rideable_type#129,started_at#130,ended_at#131,start_station_name#132,start_station_id#133,end_station_name#134,end_station_id#135,start_lat#136,start_lng#137,end_lat#138,end_lng#139,member_casual#140,_corrupt_record#141] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...
,None)
(3) Scan csv
Output [14]: [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-tripdata-part00.csv, ... 1 entries]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:string,start_lng:string,end_lat:string,end_lng:string,member_casual:string,_corrupt_record:string>
(4) Project [codegen id : 1]
Output [15]: [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]
Input [14]: [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141]
(5) TableCacheQueryStage
Output: []
Arguments: 0
(6) HashAggregate [codegen id : 1]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#12060L]
Results [1]: [count#12061L]
(7) Exchange
Input [1]: [count#12061L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2651]
(8) ShuffleQueryStage
Output [1]: [count#12061L]
Arguments: 1
(9) HashAggregate [codegen id : 2]
Input [1]: [count#12061L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#11832L]
Results [1]: [count(1)#11832L AS count#11833L]
(10) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#12060L]
Results [1]: [count#12061L]
(11) Exchange
Input [1]: [count#12061L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2623]
(12) HashAggregate
Input [1]: [count#12061L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#11832L]
Results [1]: [count(1)#11832L AS count#11833L]
(13) AdaptiveSparkPlan
Output [1]: [count#11833L]
Arguments: isFinalPlan=true