digraph G {
subgraph clusterstage_144 {
label="Stage 144";
subgraph cluster44 {
label="Scan csv ";
21 [labelType="html" label="FileScanRDD [21]<br>showString at NativeMethodAccessorImpl.java:0"];
22 [labelType="html" label="MapPartitionsRDD [22]<br>showString at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster41 {
label="WholeStageCodegen (1)";
23 [labelType="html" label="MapPartitionsRDD [23]<br>showString at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster45 {
label="mapPartitionsInternal";
24 [labelType="html" label="MapPartitionsRDD [24]<br>showString at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster46 {
label="mapPartitionsInternal";
25 [labelType="html" label="*(1) Project [ride_id#128, rideable_type#129, started_at#130, ended_at#131, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#136, start_lng#137, end_lat#138, end_lng#139, member_casual#140, _corrupt_record#141, regexp_extract(input_file_name(), [^/]+$, 0) AS _source_file#156]\n+- FileScan csv [ride_id#128,rideable_type#129,started_at#130,ended_at#131,start_station_name#132,start_station_id#133,end_station_name#134,end_station_id#135,start_lat#136,start_lng#137,end_lat#138,end_lng#139,member_casual#140,_corrupt_record#141] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(2 paths)[s3a://rzvde-g11-durnev-pavel/raw/citibike_data/202602/202602-citibike-..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:string,ended_at:string,start_station_name:s...\n [25] [Cached]<br>showString at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster79 {
label="InMemoryTableScan";
35 [labelType="html" label="MapPartitionsRDD [35]<br>parquet at NativeMethodAccessorImpl.java:0"];
36 [labelType="html" label="MapPartitionsRDD [36]<br>parquet at NativeMethodAccessorImpl.java:0"];
37 [labelType="html" label="MapPartitionsRDD [37]<br>parquet at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster74 {
label="WholeStageCodegen (1)";
38 [labelType="html" label="MapPartitionsRDD [38]<br>parquet at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster80 {
label="mapPartitionsInternal";
39 [labelType="html" label="MapPartitionsRDD [39]<br>parquet at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster81 {
label="mapPartitionsInternal";
40 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#128, rideable_type#129, started_at#669, ended_at#670, start_station_name#132, start_station_id#133, end_station_name#134, end_station_id#135, start_lat#675, start_lng#676, end_lat#677, end_lng#678, member_casual#140, _corrupt_record#141, _source_file#156, (isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AS _is_ride_id_not_empty#695, coalesce((ended_at#670 > started_at#669), false) AS _is_ended_after_started#696, coalesce(NOT (end_station_id#135 = start_station_id#133), false) AS _is_different_stations#697, ((((isnotnull(ride_id#128) AND NOT (trim(ride_id#128, None) = )) AND coalesce((ended_at#670 > started_at#669), false)) AND coalesce(NOT (end_station_id#135 = start_station_id#133), false)) AND isnull(_corrupt_record#141)) AS _is_valid#698]\n +- Project [ride_id#128, rideable_type#129, cast(started_at#130 as timestamp) AS started_at#669, cast(ended_at#131 as timestamp) AS ended_at#670, start_station_name#132, start_station_id#133, end_s... [40] [Cached]<br>parquet at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster959 {
label="InMemoryTableScan";
375 [labelType="html" label="MapPartitionsRDD [375]<br>count at <unknown>:0"];
376 [labelType="html" label="MapPartitionsRDD [376]<br>count at <unknown>:0"];
377 [labelType="html" label="MapPartitionsRDD [377]<br>count at <unknown>:0"];
}
subgraph cluster954 {
label="WholeStageCodegen (1)";
378 [labelType="html" label="MapPartitionsRDD [378]<br>count at <unknown>:0"];
}
subgraph cluster960 {
label="mapPartitionsInternal";
379 [labelType="html" label="MapPartitionsRDD [379]<br>count at <unknown>:0"];
}
subgraph cluster961 {
label="mapPartitionsInternal";
380 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Project [ride_id#3716, rideable_type#3717, started_at#3718, ended_at#3719, start_station_name#3720, start_station_id#3721, end_station_name#3722, end_station_id#3723, start_lat#3724, start_lng#3725, end_lat#3726, end_lng#3727, member_casual#3728, _corrupt_record#3279, _source_file#3294, _is_ride_id_not_empty#12608, _is_ended_after_started#12609, _is_different_stations#12610, _is_valid#12611, 2026-11-01 20:50:07.641085 AS _processed_dttm#14037]\n +- Filter (isnotnull(_is_valid#12611) AND NOT _is_valid#12611)\n +- InMemoryTableScan [_corrupt_record#3279, _is_different_stations#12610, _is_ended_after_started#12609, _is_ride_id_not_empty#12608, _is_valid#12611, _source_file#3294, end_lat#3726, end_lng#3727, end_station_id#3723, end_station_name#3722, ended_at#3719, member_casual#3728, ride_id#3716, rideable_type#3717, start_lat#3724, start_lng#3725, start_station_id#3721, start_station_name#3720, started_at#3718], [isnotnull(_is_valid#12611), NOT _is_valid#12611]\n ... [380] [Cached]<br>count at <unknown>:0"];
}
subgraph cluster1024 {
label="InMemoryTableScan";
404 [labelType="html" label="MapPartitionsRDD [404]<br>csv at NativeMethodAccessorImpl.java:0"];
405 [labelType="html" label="MapPartitionsRDD [405]<br>csv at NativeMethodAccessorImpl.java:0"];
406 [labelType="html" label="MapPartitionsRDD [406]<br>csv at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster1022 {
label="Coalesce";
407 [labelType="html" label="CoalescedRDD [407]<br>csv at NativeMethodAccessorImpl.java:0"];
}
subgraph cluster1021 {
label="WriteFiles";
408 [labelType="html" label="MapPartitionsRDD [408]<br>csv at NativeMethodAccessorImpl.java:0"];
}
}
21->22;
22->23;
23->24;
24->25;
25->35;
35->36;
36->37;
37->38;
38->39;
39->40;
40->375;
375->376;
376->377;
377->378;
378->379;
379->380;
380->404;
404->405;
405->406;
406->407;
407->408;
}
25
40
380