rzvde-g8-kirsanov-dmitry - Details for Job 97

Status: SUCCEEDED
Submitted: 2026/04/08 10:12:44
Duration: 17 ms
Associated SQL Query: 53
Completed Stages: 1
Skipped Stages: 2

Enable zooming

digraph G { subgraph clusterstage_165 { label="Stage 165 (skipped)"; subgraph cluster908 { label="Scan csv "; 360 [labelType="html" label="FileScanRDD [360] count at <unknown>:0"]; 361 [labelType="html" label="MapPartitionsRDD [361] count at <unknown>:0"]; } subgraph cluster905 { label="WholeStageCodegen (1)"; 362 [labelType="html" label="MapPartitionsRDD [362] count at <unknown>:0"]; } subgraph cluster904 { label="Exchange"; 363 [labelType="html" label="MapPartitionsRDD [363] count at <unknown>:0"]; } } 360->361; 361->362; 362->363; }

digraph G { subgraph clusterstage_166 { label="Stage 166 (skipped)"; subgraph cluster916 { label="AQEShuffleRead"; 364 [labelType="html" label="ShuffledRowRDD [364] [Unordered] count at <unknown>:0"]; } subgraph cluster913 { label="WholeStageCodegen (2)"; 365 [labelType="html" label="MapPartitionsRDD [365] [Unordered] count at <unknown>:0"]; } subgraph cluster912 { label="Window"; 366 [labelType="html" label="MapPartitionsRDD [366] [Unordered] count at <unknown>:0"]; } subgraph cluster909 { label="WholeStageCodegen (3)"; 367 [labelType="html" label="MapPartitionsRDD [367] [Unordered] count at <unknown>:0"]; } subgraph cluster917 { label="mapPartitionsInternal"; 368 [labelType="html" label="MapPartitionsRDD [368] [Unordered] count at <unknown>:0"]; } subgraph cluster918 { label="mapPartitionsInternal"; 369 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]\n +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST]\n +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0\n +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095]\n +- Project [ride_id#10530, rideable_type#10531... [369] [Cached] [Unordered] count at <unknown>:0"]; } subgraph cluster1015 { label="InMemoryTableScan"; 399 [labelType="html" label="MapPartitionsRDD [399] [Unordered] count at <unknown>:0"]; 400 [labelType="html" label="MapPartitionsRDD [400] [Unordered] count at <unknown>:0"]; 401 [labelType="html" label="MapPartitionsRDD [401] [Unordered] count at <unknown>:0"]; } subgraph cluster1009 { label="WholeStageCodegen (1)"; 402 [labelType="html" label="MapPartitionsRDD [402] [Unordered] count at <unknown>:0"]; } subgraph cluster1008 { label="Exchange"; 403 [labelType="html" label="MapPartitionsRDD [403] [Unordered] count at <unknown>:0"]; } } 364->365; 365->366; 366->367; 367->368; 368->369; 369->399; 399->400; 400->401; 401->402; 402->403; }

363,364

369

digraph G { subgraph clusterstage_167 { label="Stage 167"; subgraph cluster1020 { label="Exchange"; 404 [labelType="html" label="ShuffledRowRDD [404] [Unordered] count at <unknown>:0"]; } subgraph cluster1016 { label="WholeStageCodegen (2)"; 405 [labelType="html" label="MapPartitionsRDD [405] [Unordered] count at <unknown>:0"]; } subgraph cluster1021 { label="mapPartitionsInternal"; 406 [labelType="html" label="MapPartitionsRDD [406] [Unordered] count at <unknown>:0"]; } } 404->405; 405->406; }

403,404

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
167	count at <unknown>:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/04/08 10:12:44	14 ms	1/1			4.1 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

167

count at <unknown>:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

2026/04/08 10:12:44

14 ms

1/1

4.1 KiB

Page:

Skipped Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
166	count at <unknown>:0 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649] +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST] +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095] +- Project [ride_id#10530, rideable_type#10531... org.apache.spark.sql.Dataset.count(Dataset.scala:3625) jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/72
165	count at <unknown>:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/74

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

166

count at <unknown>:0 +details

RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649] +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST] +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095] +- Project [ride_id#10530, rideable_type#10531...

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/72

165

count at <unknown>:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/74

Page: