rzvde-g8-kirsanov-dmitry - Details for Job 37

Status: SUCCEEDED
Submitted: 2026/04/08 08:49:19
Duration: 31 ms
Associated SQL Query: 23
Completed Stages: 1
Skipped Stages: 1

Enable zooming

digraph G { subgraph clusterstage_66 { label="Stage 66 (skipped)"; subgraph cluster97 { label="Scan csv "; 45 [labelType="html" label="FileScanRDD [45] count at NativeMethodAccessorImpl.java:0"]; 46 [labelType="html" label="MapPartitionsRDD [46] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster94 { label="WholeStageCodegen (1)"; 47 [labelType="html" label="MapPartitionsRDD [47] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster93 { label="Exchange"; 48 [labelType="html" label="MapPartitionsRDD [48] count at NativeMethodAccessorImpl.java:0"]; } } 45->46; 46->47; 47->48; }

digraph G { subgraph clusterstage_67 { label="Stage 67"; subgraph cluster105 { label="AQEShuffleRead"; 49 [labelType="html" label="ShuffledRowRDD [49] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster102 { label="WholeStageCodegen (2)"; 50 [labelType="html" label="MapPartitionsRDD [50] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster101 { label="Window"; 51 [labelType="html" label="MapPartitionsRDD [51] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster98 { label="WholeStageCodegen (3)"; 52 [labelType="html" label="MapPartitionsRDD [52] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster106 { label="mapPartitionsInternal"; 53 [labelType="html" label="MapPartitionsRDD [53] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster107 { label="mapPartitionsInternal"; 54 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, start_station_id#359, end_station_name#258, end_station_id#361, start_lat#260, start_lng#261, end_lat#262, end_lng#263, member_casual#264, valid_ride_id#345, valid_time#346, valid_station#347, _source_file#348, 2026-04-08 08:20:23.917168 AS _processed_dttm#349, _start_station_ride_num#351, year#352, month#353]\n +- Window [row_number() windowspecdefinition(start_station_id#257, started_at#254 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#351], [start_station_id#257], [started_at#254 ASC NULLS FIRST]\n +- Sort [start_station_id#257 ASC NULLS FIRST, started_at#254 ASC NULLS FIRST], false, 0\n +- Exchange hashpartitioning(start_station_id#257, 200), ENSURE_REQUIREMENTS, [plan_id=113]\n +- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, cast(st... [54] [Cached] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster393 { label="InMemoryTableScan"; 161 [labelType="html" label="MapPartitionsRDD [161] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 162 [labelType="html" label="MapPartitionsRDD [162] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 163 [labelType="html" label="MapPartitionsRDD [163] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster389 { label="WholeStageCodegen (1)"; 164 [labelType="html" label="MapPartitionsRDD [164] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster388 { label="Exchange"; 165 [labelType="html" label="MapPartitionsRDD [165] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 49->50; 50->51; 51->52; 52->53; 53->54; 54->161; 161->162; 162->163; 163->164; 164->165; }

48,49

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
67	count at NativeMethodAccessorImpl.java:0 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, start_station_id#359, end_station_name#258, end_station_id#361, start_lat#260, start_lng#261, end_lat#262, end_lng#263, member_casual#264, valid_ride_id#345, valid_time#346, valid_station#347, _source_file#348, 2026-04-08 08:20:23.917168 AS _processed_dttm#349, _start_station_ride_num#351, year#352, month#353] +- Window [row_number() windowspecdefinition(start_station_id#257, started_at#254 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#351], [start_station_id#257], [started_at#254 ASC NULLS FIRST] +- Sort [start_station_id#257 ASC NULLS FIRST, started_at#254 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#257, 200), ENSURE_REQUIREMENTS, [plan_id=113] +- Project [ride_id#252, rideable_type#253, started_at#254, ended_at#255, start_station_name#256, cast(st... org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/04/08 08:49:19	22 ms	2/2	2.8 MiB			118.0 B

Page:

Skipped Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
66	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/2

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

count at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/2

Page: