rzvde-g8-kirsanov-dmitry - Details for Job 104

Status: SUCCEEDED
Submitted: 2026/04/08 12:14:36
Duration: 4.0 min
Associated SQL Query: 59
Completed Stages: 1
Skipped Stages: 1

Enable zooming

digraph G { subgraph clusterstage_181 { label="Stage 181 (skipped)"; subgraph cluster908 { label="Scan csv "; 360 [labelType="html" label="FileScanRDD [360] count at <unknown>:0"]; 361 [labelType="html" label="MapPartitionsRDD [361] count at <unknown>:0"]; } subgraph cluster905 { label="WholeStageCodegen (1)"; 362 [labelType="html" label="MapPartitionsRDD [362] count at <unknown>:0"]; } subgraph cluster904 { label="Exchange"; 363 [labelType="html" label="MapPartitionsRDD [363] count at <unknown>:0"]; } } 360->361; 361->362; 362->363; }

digraph G { subgraph clusterstage_182 { label="Stage 182"; subgraph cluster916 { label="AQEShuffleRead"; 364 [labelType="html" label="ShuffledRowRDD [364] [Unordered] count at <unknown>:0"]; } subgraph cluster913 { label="WholeStageCodegen (2)"; 365 [labelType="html" label="MapPartitionsRDD [365] [Unordered] count at <unknown>:0"]; } subgraph cluster912 { label="Window"; 366 [labelType="html" label="MapPartitionsRDD [366] [Unordered] count at <unknown>:0"]; } subgraph cluster909 { label="WholeStageCodegen (3)"; 367 [labelType="html" label="MapPartitionsRDD [367] [Unordered] count at <unknown>:0"]; } subgraph cluster917 { label="mapPartitionsInternal"; 368 [labelType="html" label="MapPartitionsRDD [368] [Unordered] count at <unknown>:0"]; } subgraph cluster918 { label="mapPartitionsInternal"; 369 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]\n +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST]\n +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0\n +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095]\n +- Project [ride_id#10530, rideable_type#10531... [369] [Cached] [Unordered] count at <unknown>:0"]; } subgraph cluster1118 { label="InMemoryTableScan"; 436 [labelType="html" label="MapPartitionsRDD [436] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; 437 [labelType="html" label="MapPartitionsRDD [437] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; 438 [labelType="html" label="MapPartitionsRDD [438] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1112 { label="WholeStageCodegen (1)"; 439 [labelType="html" label="MapPartitionsRDD [439] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1111 { label="WriteFiles"; 440 [labelType="html" label="MapPartitionsRDD [440] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } } 364->365; 365->366; 366->367; 367->368; 368->369; 369->436; 436->437; 437->438; 438->439; 439->440; }

363,364

369

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
182	parquet at NativeMethodAccessorImpl.java:0 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649] +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST] +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095] +- Project [ride_id#10530, rideable_type#10531... org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/04/08 12:14:36	4.0 min	72/72	3.2 GiB	2.2 GiB

Page:

Skipped Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
181	count at <unknown>:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/74

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

181

count at <unknown>:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/74

Page: