g10-chepusov-vladislav-stage2 - Details for Job 12

Status: SUCCEEDED
Submitted: 2026/08/18 07:27:24
Duration: 33 ms
Associated SQL Query: 3
Completed Stages: 1
Skipped Stages: 2

Enable zooming

digraph G { subgraph clusterstage_20 { label="Stage 20 (skipped)"; subgraph cluster0 { label="Scan csv "; 0 [labelType="html" label="FileScanRDD [0] count at NativeMethodAccessorImpl.java:0"]; 1 [labelType="html" label="MapPartitionsRDD [1] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster1 { label="mapPartitionsInternal"; 2 [labelType="html" label="MapPartitionsRDD [2] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster2 { label="mapPartitionsInternal"; 3 [labelType="html" label="FileScan csv [ride_id#0,rideable_type#1,started_at#2,ended_at#3,start_station_name#4,start_station_id#5,end_station_name#6,end_station_id#7,start_lat#8,start_lng#9,end_lat#10,end_lng#11,member_casual#12] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(6 paths)[s3a://rzvde-g10-chepusov-vladislav/raw/citibike_data/202508/202508-cit..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...\n [3] [Cached] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster108 { label="InMemoryTableScan"; 39 [labelType="html" label="MapPartitionsRDD [39] count at NativeMethodAccessorImpl.java:0"]; 40 [labelType="html" label="MapPartitionsRDD [40] count at NativeMethodAccessorImpl.java:0"]; 41 [labelType="html" label="MapPartitionsRDD [41] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster103 { label="WholeStageCodegen (1)"; 42 [labelType="html" label="MapPartitionsRDD [42] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster102 { label="Exchange"; 43 [labelType="html" label="MapPartitionsRDD [43] count at NativeMethodAccessorImpl.java:0"]; } } 0->1; 1->2; 2->3; 3->39; 39->40; 40->41; 41->42; 42->43; }

digraph G { subgraph clusterstage_21 { label="Stage 21 (skipped)"; subgraph cluster116 { label="AQEShuffleRead"; 44 [labelType="html" label="ShuffledRowRDD [44] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster113 { label="WholeStageCodegen (2)"; 45 [labelType="html" label="MapPartitionsRDD [45] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster112 { label="Window"; 46 [labelType="html" label="MapPartitionsRDD [46] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster109 { label="WholeStageCodegen (3)"; 47 [labelType="html" label="MapPartitionsRDD [47] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster117 { label="mapPartitionsInternal"; 48 [labelType="html" label="MapPartitionsRDD [48] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster118 { label="mapPartitionsInternal"; 49 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Filter ((NOT is_ride_id_valid#474 OR NOT is_duration_valid#475) OR NOT is_station_valid#476)\n +- Window [row_number() windowspecdefinition(start_station_id#31, started_at#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#536], [start_station_id#31], [started_at#28 ASC NULLS FIRST]\n +- Sort [start_station_id#31 ASC NULLS FIRST, started_at#28 ASC NULLS FIRST], false, 0\n +- Exchange hashpartitioning(start_station_id#31, 200), ENSURE_REQUIREMENTS, [plan_id=91]\n +- Project [ride_id#26, rideable_type#27, started_at#28, ended_at#29, start_station_name#30, start_station_id#31, end_station_name#32, end_station_id#33, start_lat#34, start_lng#35, end_lat#36, end_lng#37, member_casual#38, is_ride_id_valid#474, is_duration_valid#475, is_station_valid#476, _source_file#493, 2026-08-18 07:26:21.350521 AS _processed_dttm#494, year(cast(started_at#28 as date)) AS year#513, month(cast(started_a... [49] [Cached] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster124 { label="InMemoryTableScan"; 50 [labelType="html" label="MapPartitionsRDD [50] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 51 [labelType="html" label="MapPartitionsRDD [51] [Unordered] count at NativeMethodAccessorImpl.java:0"]; 52 [labelType="html" label="MapPartitionsRDD [52] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster120 { label="WholeStageCodegen (1)"; 53 [labelType="html" label="MapPartitionsRDD [53] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster119 { label="Exchange"; 54 [labelType="html" label="MapPartitionsRDD [54] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 44->45; 45->46; 46->47; 47->48; 48->49; 49->50; 50->51; 51->52; 52->53; 53->54; }

43,44

digraph G { subgraph clusterstage_22 { label="Stage 22"; subgraph cluster129 { label="Exchange"; 55 [labelType="html" label="ShuffledRowRDD [55] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster125 { label="WholeStageCodegen (2)"; 56 [labelType="html" label="MapPartitionsRDD [56] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } subgraph cluster130 { label="mapPartitionsInternal"; 57 [labelType="html" label="MapPartitionsRDD [57] [Unordered] count at NativeMethodAccessorImpl.java:0"]; } } 55->56; 56->57; }

54,55

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
22	count at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	2026/08/18 07:27:24	25 ms	1/1			354.0 B

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

count at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3625)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base/java.lang.reflect.Method.invoke(Method.java:568)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Thread.java:840)

2026/08/18 07:27:24

25 ms

1/1

354.0 B

Page:

Skipped Stages (2)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
21	count at NativeMethodAccessorImpl.java:0 +details RDD: AdaptiveSparkPlan isFinalPlan=false +- Filter ((NOT is_ride_id_valid#474 OR NOT is_duration_valid#475) OR NOT is_station_valid#476) +- Window [row_number() windowspecdefinition(start_station_id#31, started_at#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#536], [start_station_id#31], [started_at#28 ASC NULLS FIRST] +- Sort [start_station_id#31 ASC NULLS FIRST, started_at#28 ASC NULLS FIRST], false, 0 +- Exchange hashpartitioning(start_station_id#31, 200), ENSURE_REQUIREMENTS, [plan_id=91] +- Project [ride_id#26, rideable_type#27, started_at#28, ended_at#29, start_station_name#30, start_station_id#31, end_station_name#32, end_station_id#33, start_lat#34, start_lng#35, end_lat#36, end_lng#37, member_casual#38, is_ride_id_valid#474, is_duration_valid#475, is_station_valid#476, _source_file#493, 2026-08-18 07:26:21.350521 AS _processed_dttm#494, year(cast(started_at#28 as date)) AS year#513, month(cast(started_a... org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/6
20	count at NativeMethodAccessorImpl.java:0 +details RDD: FileScan csv [ride_id#0,rideable_type#1,started_at#2,ended_at#3,start_station_name#4,start_station_id#5,end_station_name#6,end_station_id#7,start_lat#8,start_lng#9,end_lat#10,end_lng#11,member_casual#12] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(6 paths)[s3a://rzvde-g10-chepusov-vladislav/raw/citibike_data/202508/202508-cit..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_... org.apache.spark.sql.Dataset.count(Dataset.scala:3625) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Thread.java:840)	Unknown	Unknown	0/8

Page: