CitiBike-kiryanova-st2 - Details for Job 9

Status: SUCCEEDED
Submitted: 2026/11/06 11:04:42
Duration: 54 ms
Associated SQL Query: 3
Completed Stages: 1
Skipped Stages: 2

Enable zooming

digraph G { subgraph clusterstage_15 { label="Stage 15 (skipped)"; subgraph cluster43 { label="Scan csv "; 19 [labelType="html" label="FileScanRDD [19] count at NativeMethodAccessorImpl.java:0"]; 20 [labelType="html" label="MapPartitionsRDD [20] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster39 { label="WholeStageCodegen (1)"; 21 [labelType="html" label="MapPartitionsRDD [21] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster38 { label="Exchange"; 22 [labelType="html" label="MapPartitionsRDD [22] count at NativeMethodAccessorImpl.java:0"]; } } 19->20; 20->21; 21->22; }

digraph G { subgraph clusterstage_16 { label="Stage 16 (skipped)"; subgraph cluster51 { label="AQEShuffleRead"; 23 [labelType="html" label="ShuffledRowRDD [23] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster48 { label="WholeStageCodegen (2)"; 24 [labelType="html" label="MapPartitionsRDD [24] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster47 { label="Window"; 25 [labelType="html" label="MapPartitionsRDD [25] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster44 { label="WholeStageCodegen (3)"; 26 [labelType="html" label="MapPartitionsRDD [26] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster52 { label="mapPartitionsInternal"; 27 [labelType="html" label="MapPartitionsRDD [27] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster53 { label="mapPartitionsInternal"; 28 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179]\n +- Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST]\n +- Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0\n ... [28] [Cached] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster90 { label="InMemoryTableScan"; 37 [labelType="html" label="MapPartitionsRDD [37] [Unordered] first at /tmp/ipykernel_480/4147188698.py:131"]; 38 [labelType="html" label="MapPartitionsRDD [38] [Unordered] first at /tmp/ipykernel_480/4147188698.py:131"]; 39 [labelType="html" label="MapPartitionsRDD [39] [Unordered] first at /tmp/ipykernel_480/4147188698.py:131"]; } subgraph cluster86 { label="WholeStageCodegen (1)"; 40 [labelType="html" label="MapPartitionsRDD [40] [Unordered] first at /tmp/ipykernel_480/4147188698.py:131"]; } subgraph cluster85 { label="Exchange"; 41 [labelType="html" label="MapPartitionsRDD [41] [Unordered] first at /tmp/ipykernel_480/4147188698.py:131"]; } } 23->24; 24->25; 25->26; 26->27; 27->28; 28->37; 37->38; 38->39; 39->40; 40->41; }

22,23

digraph G { subgraph clusterstage_17 { label="Stage 17"; subgraph cluster95 { label="Exchange"; 42 [labelType="html" label="ShuffledRowRDD [42] [Unordered] first at /tmp/ipykernel_480/4147188698.py:131"]; } subgraph cluster91 { label="WholeStageCodegen (2)"; 43 [labelType="html" label="MapPartitionsRDD [43] [Unordered] first at /tmp/ipykernel_480/4147188698.py:131"]; } subgraph cluster96 { label="mapPartitionsInternal"; 44 [labelType="html" label="MapPartitionsRDD [44] [Unordered] first at /tmp/ipykernel_480/4147188698.py:131"]; } } 42->43; 43->44; }

41,42

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
17	first at /tmp/ipykernel_480/4147188698.py:131 +details org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4157) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/11/06 11:04:42	43 ms	1/1			187.0 B

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

first at /tmp/ipykernel_480/4147188698.py:131 +details

org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4157)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

2026/11/06 11:04:42

43 ms

1/1

187.0 B

Page:

Skipped Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
16	first at /tmp/ipykernel_480/4147188698.py:131 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179] +- Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST] +- Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0 ... org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4157) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/3
15	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/3

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

first at /tmp/ipykernel_480/4147188698.py:131 +details

RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#56, rideable_type#57, started_at#93, ended_at#94, start_station_name#60, start_station_id#124, end_station_name#62, end_station_id#155, start_lat#88, start_lng#89, end_lat#90, end_lng#91, member_casual#68, _start_station_id_fixed#109, _end_station_id_fixed#139, _ride_id_ok#171, _duration_ok#172, _station_id_ok#173, _source_file#174, 2026-11-06 11:04:13.790835 AS _processed_dttm#175, year#176, month#177, _start_station_ride_num#179] +- Window [row_number() windowspecdefinition(start_station_id#124, _w0#180, _w1#181, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#179], [start_station_id#124, _w0#180, _w1#181], [started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST] +- Sort [start_station_id#124 ASC NULLS FIRST, _w0#180 ASC NULLS FIRST, _w1#181 ASC NULLS FIRST, started_at#93 ASC NULLS FIRST, ride_id#56 ASC NULLS FIRST], false, 0 ...

org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4157)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/3

count at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/3

Page: