rzvde-g8-kirsanov-dmitry - Details for Job 102

Status: SUCCEEDED
Submitted: 2026/04/08 10:30:21
Duration: 3.4 min
Associated SQL Query: 57
Completed Stages: 1
Skipped Stages: 2

Enable zooming

digraph G { subgraph clusterstage_176 { label="Stage 176 (skipped)"; subgraph cluster908 { label="Scan csv "; 360 [labelType="html" label="FileScanRDD [360] count at <unknown>:0"]; 361 [labelType="html" label="MapPartitionsRDD [361] count at <unknown>:0"]; } subgraph cluster905 { label="WholeStageCodegen (1)"; 362 [labelType="html" label="MapPartitionsRDD [362] count at <unknown>:0"]; } subgraph cluster904 { label="Exchange"; 363 [labelType="html" label="MapPartitionsRDD [363] count at <unknown>:0"]; } } 360->361; 361->362; 362->363; }

digraph G { subgraph clusterstage_177 { label="Stage 177 (skipped)"; subgraph cluster916 { label="AQEShuffleRead"; 364 [labelType="html" label="ShuffledRowRDD [364] [Unordered] count at <unknown>:0"]; } subgraph cluster913 { label="WholeStageCodegen (2)"; 365 [labelType="html" label="MapPartitionsRDD [365] [Unordered] count at <unknown>:0"]; } subgraph cluster912 { label="Window"; 366 [labelType="html" label="MapPartitionsRDD [366] [Unordered] count at <unknown>:0"]; } subgraph cluster909 { label="WholeStageCodegen (3)"; 367 [labelType="html" label="MapPartitionsRDD [367] [Unordered] count at <unknown>:0"]; } subgraph cluster917 { label="mapPartitionsInternal"; 368 [labelType="html" label="MapPartitionsRDD [368] [Unordered] count at <unknown>:0"]; } subgraph cluster918 { label="mapPartitionsInternal"; 369 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]\n +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST]\n +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0\n +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095]\n +- Project [ride_id#10530, rideable_type#10531... [369] [Cached] [Unordered] count at <unknown>:0"]; } subgraph cluster1080 { label="InMemoryTableScan"; 422 [labelType="html" label="MapPartitionsRDD [422] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; 423 [labelType="html" label="MapPartitionsRDD [423] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; 424 [labelType="html" label="MapPartitionsRDD [424] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1075 { label="WholeStageCodegen (1)"; 425 [labelType="html" label="MapPartitionsRDD [425] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1074 { label="Exchange"; 426 [labelType="html" label="MapPartitionsRDD [426] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; 427 [labelType="html" label="MapPartitionsRDD [427] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } } 364->365; 365->366; 366->367; 367->368; 368->369; 369->422; 422->423; 423->424; 424->425; 425->426; 426->427; }

363,364

369

digraph G { subgraph clusterstage_178 { label="Stage 178"; subgraph cluster1086 { label="Exchange"; 428 [labelType="html" label="ShuffledRowRDD [428] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1082 { label="WholeStageCodegen (2)"; 429 [labelType="html" label="MapPartitionsRDD [429] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1081 { label="WriteFiles"; 430 [labelType="html" label="MapPartitionsRDD [430] [Unordered] parquet at NativeMethodAccessorImpl.java:0"]; } } 428->429; 429->430; }

427,428

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
178	parquet at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/04/08 10:30:21	3.4 min	4/4		2.2 GiB	5.4 GiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

178

parquet at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

2026/04/08 10:30:21

3.4 min

4/4

2.2 GiB

5.4 GiB

Page:

Skipped Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
177	parquet at NativeMethodAccessorImpl.java:0 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649] +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST] +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095] +- Project [ride_id#10530, rideable_type#10531... org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/72
176	count at <unknown>:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/74

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

177

parquet at NativeMethodAccessorImpl.java:0 +details

RDD: AdaptiveSparkPlan isFinalPlan=false +- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649] +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST] +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095] +- Project [ride_id#10530, rideable_type#10531...

org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/72

176

count at <unknown>:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

Unknown

0/74

Page: