rzvde-g8-kirsanov-dmitry - Details for Job 40

Status: SUCCEEDED
Submitted: 2026/04/08 08:49:19
Duration: 26 ms
Associated SQL Query: 24
Completed Stages: 1
Skipped Stages: 2

Enable zooming

digraph G { subgraph clusterstage_73 { label="Stage 73 (skipped)"; subgraph cluster97 { label="Scan csv "; 45 [labelType="html" label="FileScanRDD [45] count at NativeMethodAccessorImpl.java:0"]; 46 [labelType="html" label="MapPartitionsRDD [46] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster94 { label="WholeStageCodegen (1)"; 47 [labelType="html" label="MapPartitionsRDD [47] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster93 { label="Exchange"; 48 [labelType="html" label="MapPartitionsRDD [48] count at NativeMethodAccessorImpl.java:0"]; } } 45->46; 46->47; 47->48; }

digraph G { subgraph clusterstage_74 { label="Stage 74 (skipped)"; subgraph cluster105 { label="AQEShuffleRead"; 49 [labelType="html" label="ShuffledRowRDD [49] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster102 { label="WholeStageCodegen (2)"; 50 [labelType="html" label="MapPartitionsRDD [50] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster101 { label="Window"; 51 [labelType="html" label="MapPartitionsRDD [51] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster98 { label="WholeStageCodegen (3)"; 52 [labelType="html" label="MapPartitionsRDD [52] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster106 { label="mapPartitionsInternal"; 53 [labelType="html" label="MapPartitionsRDD [53] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster107 { label="mapPartitionsInternal"; 54 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, start_station_id#359, end_station_name#258, end_station_id#361, start_lat#260, start_lng#261, end_lat#262, end_lng#263, member_casual#264, valid_ride_id#345, valid_time#346, valid_station#347, _source_file#348, 2026-04-08 08:20:23.917168 AS _processed_dttm#349, _start_station_ride_num#351, year#352, month#353]\n +- Window [row_number() windowspecdefinition(start_station_id#257, started_at#254 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#351], [start_station_id#257], [started_at#254 ASC NULLS FIRST]\n +- Sort [start_station_id#257 ASC NULLS FIRST, started_at#254 ASC NULLS FIRST], false, 0\n +- Exchange hashpartitioning(start_station_id#257, 200), ENSURE_REQUIREMENTS, [plan_id=113]\n +- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, cast(st... [54] [Cached] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster417 { label="InMemoryTableScan"; 169 [labelType="html" label="MapPartitionsRDD [169] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 170 [labelType="html" label="MapPartitionsRDD [170] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 171 [labelType="html" label="MapPartitionsRDD [171] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster411 { label="WholeStageCodegen (1)"; 172 [labelType="html" label="MapPartitionsRDD [172] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster410 { label="Exchange"; 173 [labelType="html" label="MapPartitionsRDD [173] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 49->50; 50->51; 51->52; 52->53; 53->54; 54->169; 169->170; 170->171; 171->172; 172->173; }

48,49

digraph G { subgraph clusterstage_75 { label="Stage 75"; subgraph cluster422 { label="Exchange"; 174 [labelType="html" label="ShuffledRowRDD [174] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster418 { label="WholeStageCodegen (2)"; 175 [labelType="html" label="MapPartitionsRDD [175] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster423 { label="mapPartitionsInternal"; 176 [labelType="html" label="MapPartitionsRDD [176] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 174->175; 175->176; }

173,174

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
75	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/04/08 08:49:19	18 ms	1/1			118.0 B

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

count at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

2026/04/08 08:49:19

18 ms

1/1

118.0 B

Page:

Skipped Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
74	count at NativeMethodAccessorImpl.java:0 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, start_station_id#359, end_station_name#258, end_station_id#361, start_lat#260, start_lng#261, end_lat#262, end_lng#263, member_casual#264, valid_ride_id#345, valid_time#346, valid_station#347, _source_file#348, 2026-04-08 08:20:23.917168 AS _processed_dttm#349, _start_station_ride_num#351, year#352, month#353] +- Window [row_number() windowspecdefinition(start_station_id#257, started_at#254 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#351], [start_station_id#257], [started_at#254 ASC NULLS FIRST] +- Sort [start_station_id#257 ASC NULLS FIRST, started_at#254 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#257, 200), ENSURE_REQUIREMENTS, [plan_id=113] +- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, cast(st... org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/2
73	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/2

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

count at NativeMethodAccessorImpl.java:0 +details

RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, start_station_id#359, end_station_name#258, end_station_id#361, start_lat#260, start_lng#261, end_lat#262, end_lng#263, member_casual#264, valid_ride_id#345, valid_time#346, valid_station#347, _source_file#348, 2026-04-08 08:20:23.917168 AS _processed_dttm#349, _start_station_ride_num#351, year#352, month#353] +- Window [row_number() windowspecdefinition(start_station_id#257, started_at#254 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#351], [start_station_id#257], [started_at#254 ASC NULLS FIRST] +- Sort [start_station_id#257 ASC NULLS FIRST, started_at#254 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#257, 200), ENSURE_REQUIREMENTS, [plan_id=113] +- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, cast(st...

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/2

count at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/2

Page: