g10-chepusov-vladislav-stage2 - Details for Job 8

Status: SUCCEEDED
Submitted: 2026/08/18 07:27:05
Duration: 35 ms
Associated SQL Query: 2
Completed Stages: 1
Skipped Stages: 2

Enable zooming

digraph G { subgraph clusterstage_12 { label="Stage 12 (skipped)"; subgraph cluster0 { label="Scan csv "; 0 [labelType="html" label="FileScanRDD [0] count at NativeMethodAccessorImpl.java:0"]; 1 [labelType="html" label="MapPartitionsRDD [1] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1 { label="mapPartitionsInternal"; 2 [labelType="html" label="MapPartitionsRDD [2] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2 { label="mapPartitionsInternal"; 3 [labelType="html" label="FileScan csv [ride_id#0,rideable_type#1,started_at#2,ended_at#3,start_station_name#4,start_station_id#5,end_station_name#6,end_station_id#7,start_lat#8,start_lng#9,end_lat#10,end_lng#11,member_casual#12] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(6 paths)[s3a://rzvde-g10-chepusov-vladislav/raw/citibike_data/202508/202508-cit..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...\n [3] [Cached] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster57 { label="InMemoryTableScan"; 20 [labelType="html" label="MapPartitionsRDD [20] count at NativeMethodAccessorImpl.java:0"]; 21 [labelType="html" label="MapPartitionsRDD [21] count at NativeMethodAccessorImpl.java:0"]; 22 [labelType="html" label="MapPartitionsRDD [22] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster52 { label="WholeStageCodegen (1)"; 23 [labelType="html" label="MapPartitionsRDD [23] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster51 { label="Exchange"; 24 [labelType="html" label="MapPartitionsRDD [24] count at NativeMethodAccessorImpl.java:0"]; } } 0->1; 1->2; 2->3; 3->20; 20->21; 21->22; 22->23; 23->24; }

digraph G { subgraph clusterstage_13 { label="Stage 13 (skipped)"; subgraph cluster65 { label="AQEShuffleRead"; 25 [labelType="html" label="ShuffledRowRDD [25] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster62 { label="WholeStageCodegen (2)"; 26 [labelType="html" label="MapPartitionsRDD [26] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster61 { label="Window"; 27 [labelType="html" label="MapPartitionsRDD [27] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster58 { label="WholeStageCodegen (3)"; 28 [labelType="html" label="MapPartitionsRDD [28] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster66 { label="mapPartitionsInternal"; 29 [labelType="html" label="MapPartitionsRDD [29] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster67 { label="mapPartitionsInternal"; 30 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Filter (isnotnull(is_ride_id_valid#474) AND ((is_ride_id_valid#474 AND is_duration_valid#475) AND is_station_valid#476))\n +- Window [row_number() windowspecdefinition(start_station_id#31, started_at#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#536], [start_station_id#31], [started_at#28 ASC NULLS FIRST]\n +- Sort [start_station_id#31 ASC NULLS FIRST, started_at#28 ASC NULLS FIRST], false, 0\n +- Exchange hashpartitioning(start_station_id#31, 200), ENSURE_REQUIREMENTS, [plan_id=68]\n +- Project [ride_id#26, rideable_type#27, started_at#28, ended_at#29, start_station_name#30, start_station_id#31, end_station_name#32, end_station_id#33, start_lat#34, start_lng#35, end_lat#36, end_lng#37, member_casual#38, is_ride_id_valid#474, is_duration_valid#475, is_station_valid#476, _source_file#493, 2026-08-18 07:26:21.132535 AS _processed_dttm#494, year(cast(started_at#28 as date)) AS ye... [30] [Cached] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster73 { label="InMemoryTableScan"; 31 [labelType="html" label="MapPartitionsRDD [31] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 32 [labelType="html" label="MapPartitionsRDD [32] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 33 [labelType="html" label="MapPartitionsRDD [33] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster69 { label="WholeStageCodegen (1)"; 34 [labelType="html" label="MapPartitionsRDD [34] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster68 { label="Exchange"; 35 [labelType="html" label="MapPartitionsRDD [35] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 25->26; 26->27; 27->28; 28->29; 29->30; 30->31; 31->32; 32->33; 33->34; 34->35; }

24,25

digraph G { subgraph clusterstage_14 { label="Stage 14"; subgraph cluster78 { label="Exchange"; 36 [labelType="html" label="ShuffledRowRDD [36] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster74 { label="WholeStageCodegen (2)"; 37 [labelType="html" label="MapPartitionsRDD [37] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster79 { label="mapPartitionsInternal"; 38 [labelType="html" label="MapPartitionsRDD [38] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 36->37; 37->38; }

35,36

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
14	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/08/18 07:27:05	26 ms	1/1			354.0 B

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

count at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

2026/08/18 07:27:05

26 ms

1/1

354.0 B

Page:

Skipped Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
13	count at NativeMethodAccessorImpl.java:0 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Filter (isnotnull(is_ride_id_valid#474) AND ((is_ride_id_valid#474 AND is_duration_valid#475) AND is_station_valid#476)) +- Window [row_number() windowspecdefinition(start_station_id#31, started_at#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#536], [start_station_id#31], [started_at#28 ASC NULLS FIRST] +- Sort [start_station_id#31 ASC NULLS FIRST, started_at#28 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#31, 200), ENSURE_REQUIREMENTS, [plan_id=68] +- Project [ride_id#26, rideable_type#27, started_at#28, ended_at#29, start_station_name#30, start_station_id#31, end_station_name#32, end_station_id#33, start_lat#34, start_lng#35, end_lat#36, end_lng#37, member_casual#38, is_ride_id_valid#474, is_duration_valid#475, is_station_valid#476, _source_file#493, 2026-08-18 07:26:21.132535 AS _processed_dttm#494, year(cast(started_at#28 as date)) AS ye... org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/6
12	count at NativeMethodAccessorImpl.java:0 +details RDD: FileScan csv [ride_id#0,rideable_type#1,started_at#2,ended_at#3,start_station_name#4,start_station_id#5,end_station_name#6,end_station_id#7,start_lat#8,start_lng#9,end_lat#10,end_lng#11,member_casual#12] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(6 paths)[s3a://rzvde-g10-chepusov-vladislav/raw/citibike_data/202508/202508-cit..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_... org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/8

Page: