CitiBike-kiryanova-st2 - Details for Job 7

Status: SUCCEEDED
Submitted: 2026/11/06 11:04:42
Duration: 76 ms
Associated SQL Query: 2
Completed Stages: 1
Skipped Stages: 2

Enable zooming

digraph G { subgraph clusterstage_10 { label="Stage 10 (skipped)"; subgraph cluster43 { label="Scan csv "; 19 [labelType="html" label="FileScanRDD [19] count at NativeMethodAccessorImpl.java:0"]; 20 [labelType="html" label="MapPartitionsRDD [20] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster39 { label="WholeStageCodegen (1)"; 21 [labelType="html" label="MapPartitionsRDD [21] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster38 { label="Exchange"; 22 [labelType="html" label="MapPartitionsRDD [22] count at NativeMethodAccessorImpl.java:0"]; } } 19->20; 20->21; 21->22; }

digraph G { subgraph clusterstage_11 { label="Stage 11 (skipped)"; subgraph cluster51 { label="AQEShuffleRead"; 23 [labelType="html" label="ShuffledRowRDD [23] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster48 { label="WholeStageCodegen (2)"; 24 [labelType="html" label="MapPartitionsRDD [24] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster47 { label="Window"; 25 [labelType="html" label="MapPartitionsRDD [25] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster44 { label="WholeStageCodegen (3)"; 26 [labelType="html" label="MapPartitionsRDD [26] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster52 { label="mapPartitionsInternal"; 27 [labelType="html" label="MapPartitionsRDD [27] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster53 { label="mapPartitionsInternal"; 28 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179]\n +- Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST]\n +- Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0\n ... [28] [Cached] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster59 { label="InMemoryTableScan"; 29 [labelType="html" label="MapPartitionsRDD [29] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 30 [labelType="html" label="MapPartitionsRDD [30] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 31 [labelType="html" label="MapPartitionsRDD [31] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster55 { label="WholeStageCodegen (1)"; 32 [labelType="html" label="MapPartitionsRDD [32] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster54 { label="Exchange"; 33 [labelType="html" label="MapPartitionsRDD [33] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 23->24; 24->25; 25->26; 26->27; 27->28; 28->29; 29->30; 30->31; 31->32; 32->33; }

22,23

digraph G { subgraph clusterstage_12 { label="Stage 12"; subgraph cluster64 { label="Exchange"; 34 [labelType="html" label="ShuffledRowRDD [34] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster60 { label="WholeStageCodegen (2)"; 35 [labelType="html" label="MapPartitionsRDD [35] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster65 { label="mapPartitionsInternal"; 36 [labelType="html" label="MapPartitionsRDD [36] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 34->35; 35->36; }

33,34

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
12	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/11/06 11:04:42	58 ms	1/1			177.0 B

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

count at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

2026/11/06 11:04:42

58 ms

1/1

177.0 B

Page:

Skipped Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
11	count at NativeMethodAccessorImpl.java:0 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179] +- Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST] +- Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0 ... org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/3
10	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/3

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

count at NativeMethodAccessorImpl.java:0 +details

RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179] +- Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST] +- Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0 ...

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/3

count at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/3

Page: