rzvde-g8-kirsanov-dmitry - Details for Query 55

Submitted Time: 2026/04/08 10:14:44
Duration: 12 min
Succeeded Jobs: 99

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label="Execute InsertIntoHadoopFsRelationCommand task commit time total (min, med, max (stageId: taskId)) 50.9 s (567 ms, 684 ms, 1.1 s (stage 171.0: task 927)) number of written files: 1,073 job commit time: 8.0 m number of output rows: 47,295,356 number of dynamic part: 15 written output: 2.2 GiB"]; 2 [labelType="html" label=" WriteFiles "]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n7.5 m (4.1 s, 6.2 s, 8.1 s (stage 171.0: task 961))"; 4 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 3.4 s (26 ms, 47 ms, 72 ms (stage 171.0: task 922)) peak memory total (min, med, max (stageId: taskId)) 17.4 GiB (144.0 MiB, 240.0 MiB, 320.0 MiB (stage 171.0: task 922)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908))"]; 5 [labelType="html" label=" Project "]; 6 [labelType="html" label="Filter number of output rows: 47,295,356"]; } 7 [labelType="html" label="InMemoryTableScan number of output rows: 48,408,771"]; 8 [labelType="html" label=" AdaptiveSparkPlan "]; subgraph cluster9 { isCluster="true"; label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908))"; 10 [labelType="html" label=" Project "]; } 11 [labelType="html" label="Window spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908))"]; subgraph cluster12 { isCluster="true"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908))"; 13 [labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908)) peak memory total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908))"]; } 14 [labelType="html" label=" AQEShuffleRead "]; 15 [labelType="html" label="Exchange shuffle records written: 0 local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908)) remote merged bytes read total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908)) local merged blocks fetched: 0 corrupt merged block chunks: 0 remote merged reqs duration total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908)) remote merged blocks fetched: 0 records read: 0 local bytes read total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908)) fetch wait time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908)) remote bytes read total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908)) merged fetch fallback count: 0 local blocks read: 0 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908)) local merged bytes read total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908)) number of partitions: 0 remote reqs duration total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908)) remote bytes read to disk total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908)) shuffle bytes written total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908))"]; subgraph cluster16 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908))"; 17 [labelType="html" label=" Project "]; } 18 [labelType="html" label="Scan csv number of output rows: 0 number of files read: 0 metadata time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 171.0: task 908)) size of files read total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 171.0: task 908))"]; 1->0; 2->1; 4->2; 5->4; 6->5; 7->6; 8->7; 10->8; 11->10; 13->11; 14->13; 15->14; 17->15; 18->17; }

AdaptiveSparkPlan isFinalPlan=true

Execute InsertIntoHadoopFsRelationCommand s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data, false, [year#10648, month#10649], Parquet, [partitionOverwriteMode=dynamic, __partition_columns=["year","month"], path=s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data/], Overwrite, [ride_id, rideable_type, started_at, ended_at, start_station_name, start_station_id, end_station_name, end_station_id, start_lat, start_lng, end_lat, end_lng, member_casual, _source_file, _processed_dttm, _start_station_ride_num, year, month]

WriteFiles

Sort [year#10648 ASC NULLS FIRST, month#10649 ASC NULLS FIRST], false, 0

Project [ride_id#10650, rideable_type#10651, started_at#10652, ended_at#10653, start_station_name#10654, start_station_id#10655, end_station_name#10656, end_station_id#10657, start_lat#10658, start_lng#10659, end_lat#10660, end_lng#10661, member_casual#10662, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]

Filter (((((((isnotnull(valid_time#10642) AND valid_ride_id#10641) AND valid_time#10642) AND valid_station#10643) AND isnotnull(ride_id#10650)) AND NOT (ride_id#10650 = )) AND isnotnull(rideable_type#10651)) AND NOT (rideable_type#10651 = ))

WholeStageCodegen (1)

InMemoryTableScan [_processed_dttm#10645, _source_file#10644, _start_station_ride_num#10647, end_lat#10660, end_lng#10661, end_station_id#10657, end_station_name#10656, ended_at#10653, member_casual#10662, month#10649, ride_id#10650, rideable_type#10651, start_lat#10658, start_lng#10659, start_station_id#10655, start_station_name#10654, started_at#10652, valid_ride_id#10641, valid_station#10643, valid_time#10642, year#10648], [isnotnull(valid_time#10642), valid_ride_id#10641, valid_time#10642, valid_station#10643, isnotnull(ride_id#10650), NOT (ride_id#10650 = ), isnotnull(rideable_type#10651), NOT (rideable_type#10651 = )]

AdaptiveSparkPlan isFinalPlan=true

Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]

WholeStageCodegen (3)

Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST]

Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0

WholeStageCodegen (2)

AQEShuffleRead coalesced

Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2133]

Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, cast(start_station_id#10535 as double) AS start_station_id#10655, end_station_name#10536, cast(end_station_id#10537 as double) AS end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, isnotnull(ride_id#10530) AS valid_ride_id#10641, (ended_at#10533 > started_at#10532) AS valid_time#10642, (((isnotnull(end_station_id#10537) AND isnotnull(start_station_id#10535)) AND NOT (end_station_id#10537 = start_station_id#10535)) <=> true) AS valid_station#10643, input_file_name() AS _source_file#10644, year(cast(started_at#10532 as date)) AS year#10648, month(cast(started_at#10532 as date)) AS month#10649, start_station_id#10535, started_at#10532]

WholeStageCodegen (1)

FileScan csv [ride_id#10530,rideable_type#10531,started_at#10532,ended_at#10533,start_station_name#10534,start_station_id#10535,end_station_name#10536,end_station_id#10537,start_lat#10538,start_lng#10539,end_lat#10540,end_lng#10541,member_casual#10542] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(55 paths)[s3a://rzvde-g8-kirsanov-dmitry/raw/citibike_data/202502/202502-citibi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...

Details

== Physical Plan ==
AdaptiveSparkPlan (21)
+- == Final Plan ==
   Execute InsertIntoHadoopFsRelationCommand (15)
   +- WriteFiles (14)
      +- * Sort (13)
         +- * Project (12)
            +- * Filter (11)
               +- TableCacheQueryStage (10), Statistics(sizeInBytes=12.5 GiB, rowCount=4.84E+7)
                  +- InMemoryTableScan (1)
                        +- InMemoryRelation (2)
                              +- AdaptiveSparkPlan (9)
                                 +- Project (8)
                                    +- Window (7)
                                       +- Sort (6)
                                          +- Exchange (5)
                                             +- Project (4)
                                                +- Scan csv  (3)
+- == Initial Plan ==
   Execute InsertIntoHadoopFsRelationCommand (20)
   +- WriteFiles (19)
      +- Sort (18)
         +- Project (17)
            +- Filter (16)
               +- InMemoryTableScan (1)
                     +- InMemoryRelation (2)
                           +- AdaptiveSparkPlan (9)
                              +- Project (8)
                                 +- Window (7)
                                    +- Sort (6)
                                       +- Exchange (5)
                                          +- Project (4)
                                             +- Scan csv  (3)


(1) InMemoryTableScan
Output [21]: [_processed_dttm#10645, _source_file#10644, _start_station_ride_num#10647, end_lat#10660, end_lng#10661, end_station_id#10657, end_station_name#10656, ended_at#10653, member_casual#10662, month#10649, ride_id#10650, rideable_type#10651, start_lat#10658, start_lng#10659, start_station_id#10655, start_station_name#10654, started_at#10652, valid_ride_id#10641, valid_station#10643, valid_time#10642, year#10648]
Arguments: [_processed_dttm#10645, _source_file#10644, _start_station_ride_num#10647, end_lat#10660, end_lng#10661, end_station_id#10657, end_station_name#10656, ended_at#10653, member_casual#10662, month#10649, ride_id#10650, rideable_type#10651, start_lat#10658, start_lng#10659, start_station_id#10655, start_station_name#10654, started_at#10652, valid_ride_id#10641, valid_station#10643, valid_time#10642, year#10648], [isnotnull(valid_time#10642), valid_ride_id#10641, valid_time#10642, valid_station#10643, isnotnull(ride_id#10650), NOT (ride_id#10650 = ), isnotnull(rideable_type#10651), NOT (rideable_type#10651 = )]

(2) InMemoryRelation
Arguments: [ride_id#10650, rideable_type#10651, started_at#10652, ended_at#10653, start_station_name#10654, start_station_id#10655, end_station_name#10656, end_station_id#10657, start_lat#10658, start_lng#10659, end_lat#10660, end_lng#10661, member_casual#10662, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@43d79ed6,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
   *(3) Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]
   +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST]
      +- *(2) Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0
         +- AQEShuffleRead coalesced
            +- ShuffleQueryStage 0
               +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2133]
                  +- *(1) Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, cast(start_station_id#10535 as double) AS start_station_id#10655, end_station_name#10536, cast(end_station_id#10537 as double) AS end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, isnotnull(ride_id#10530) AS valid_ride_id#10641, (ended_at#10533 > started_at#10532) AS valid_time#10642, (((isnotnull(end_station_id#10537) AND isnotnull(start_station_id#10535)) AND NOT (end_station_id#10537 = start_station_id#10535)) <=> true) AS valid_station#10643, input_file_name() AS _source_file#10644, year(cast(started_at#10532 as date)) AS year#10648, month(cast(started_at#10532 as date)) AS month#10649, start_station_id#10535, started_at#10532]
                     +- FileScan csv [ride_id#10530,rideable_type#10531,started_at#10532,ended_at#10533,start_station_name#10534,start_station_id#10535,end_station_name#10536,end_station_id#10537,start_lat#10538,start_lng#10539,end_lat#10540,end_lng#10541,member_casual#10542] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(55 paths)[s3a://rzvde-g8-kirsanov-dmitry/raw/citibike_data/202502/202502-citibi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...
+- == Initial Plan ==
   Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]
   +- Window [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST]
      +- Sort [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0
         +- Exchange hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2095]
            +- Project [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, cast(start_station_id#10535 as double) AS start_station_id#10655, end_station_name#10536, cast(end_station_id#10537 as double) AS end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, isnotnull(ride_id#10530) AS valid_ride_id#10641, (ended_at#10533 > started_at#10532) AS valid_time#10642, (((isnotnull(end_station_id#10537) AND isnotnull(start_station_id#10535)) AND NOT (end_station_id#10537 = start_station_id#10535)) <=> true) AS valid_station#10643, input_file_name() AS _source_file#10644, year(cast(started_at#10532 as date)) AS year#10648, month(cast(started_at#10532 as date)) AS month#10649, start_station_id#10535, started_at#10532]
               +- FileScan csv [ride_id#10530,rideable_type#10531,started_at#10532,ended_at#10533,start_station_name#10534,start_station_id#10535,end_station_name#10536,end_station_id#10537,start_lat#10538,start_lng#10539,end_lat#10540,end_lng#10541,member_casual#10542] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(55 paths)[s3a://rzvde-g8-kirsanov-dmitry/raw/citibike_data/202502/202502-citibi..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_...
,None)

(3) Scan csv 
Output [13]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10535, end_station_name#10536, end_station_id#10537, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542]
Batched: false
Location: InMemoryFileIndex [s3a://rzvde-g8-kirsanov-dmitry/raw/citibike_data/202502/202502-citibike-tripdata-part00.csv, ... 54 entries]
ReadSchema: struct<ride_id:string,rideable_type:string,started_at:timestamp,ended_at:timestamp,start_station_name:string,start_station_id:string,end_station_name:string,end_station_id:string,start_lat:double,start_lng:double,end_lat:double,end_lng:double,member_casual:string>

(4) Project
Output [21]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, cast(start_station_id#10535 as double) AS start_station_id#10655, end_station_name#10536, cast(end_station_id#10537 as double) AS end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, isnotnull(ride_id#10530) AS valid_ride_id#10641, (ended_at#10533 > started_at#10532) AS valid_time#10642, (((isnotnull(end_station_id#10537) AND isnotnull(start_station_id#10535)) AND NOT (end_station_id#10537 = start_station_id#10535)) <=> true) AS valid_station#10643, input_file_name() AS _source_file#10644, year(cast(started_at#10532 as date)) AS year#10648, month(cast(started_at#10532 as date)) AS month#10649, start_station_id#10535, started_at#10532]
Input [13]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10535, end_station_name#10536, end_station_id#10537, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542]

(5) Exchange
Input [21]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, year#10648, month#10649, start_station_id#10535, started_at#10532]
Arguments: hashpartitioning(start_station_id#10535, 200), ENSURE_REQUIREMENTS, [plan_id=2471]

(6) Sort
Input [21]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, year#10648, month#10649, start_station_id#10535, started_at#10532]
Arguments: [start_station_id#10535 ASC NULLS FIRST, started_at#10532 ASC NULLS FIRST], false, 0

(7) Window
Input [21]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, year#10648, month#10649, start_station_id#10535, started_at#10532]
Arguments: [row_number() windowspecdefinition(start_station_id#10535, started_at#10532 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS _start_station_ride_num#10647], [start_station_id#10535], [started_at#10532 ASC NULLS FIRST]

(8) Project
Output [21]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, 2026-04-08 09:56:30.312891 AS _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]
Input [22]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, year#10648, month#10649, start_station_id#10535, started_at#10532, _start_station_ride_num#10647]

(9) AdaptiveSparkPlan
Output [21]: [ride_id#10530, rideable_type#10531, started_at#10532, ended_at#10533, start_station_name#10534, start_station_id#10655, end_station_name#10536, end_station_id#10657, start_lat#10538, start_lng#10539, end_lat#10540, end_lng#10541, member_casual#10542, valid_ride_id#10641, valid_time#10642, valid_station#10643, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]
Arguments: isFinalPlan=false

(10) TableCacheQueryStage
Output [21]: [_processed_dttm#10645, _source_file#10644, _start_station_ride_num#10647, end_lat#10660, end_lng#10661, end_station_id#10657, end_station_name#10656, ended_at#10653, member_casual#10662, month#10649, ride_id#10650, rideable_type#10651, start_lat#10658, start_lng#10659, start_station_id#10655, start_station_name#10654, started_at#10652, valid_ride_id#10641, valid_station#10643, valid_time#10642, year#10648]
Arguments: 0

(11) Filter [codegen id : 1]
Input [21]: [_processed_dttm#10645, _source_file#10644, _start_station_ride_num#10647, end_lat#10660, end_lng#10661, end_station_id#10657, end_station_name#10656, ended_at#10653, member_casual#10662, month#10649, ride_id#10650, rideable_type#10651, start_lat#10658, start_lng#10659, start_station_id#10655, start_station_name#10654, started_at#10652, valid_ride_id#10641, valid_station#10643, valid_time#10642, year#10648]
Condition : (((((((isnotnull(valid_time#10642) AND valid_ride_id#10641) AND valid_time#10642) AND valid_station#10643) AND isnotnull(ride_id#10650)) AND NOT (ride_id#10650 = )) AND isnotnull(rideable_type#10651)) AND NOT (rideable_type#10651 = ))

(12) Project [codegen id : 1]
Output [18]: [ride_id#10650, rideable_type#10651, started_at#10652, ended_at#10653, start_station_name#10654, start_station_id#10655, end_station_name#10656, end_station_id#10657, start_lat#10658, start_lng#10659, end_lat#10660, end_lng#10661, member_casual#10662, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]
Input [21]: [_processed_dttm#10645, _source_file#10644, _start_station_ride_num#10647, end_lat#10660, end_lng#10661, end_station_id#10657, end_station_name#10656, ended_at#10653, member_casual#10662, month#10649, ride_id#10650, rideable_type#10651, start_lat#10658, start_lng#10659, start_station_id#10655, start_station_name#10654, started_at#10652, valid_ride_id#10641, valid_station#10643, valid_time#10642, year#10648]

(13) Sort [codegen id : 1]
Input [18]: [ride_id#10650, rideable_type#10651, started_at#10652, ended_at#10653, start_station_name#10654, start_station_id#10655, end_station_name#10656, end_station_id#10657, start_lat#10658, start_lng#10659, end_lat#10660, end_lng#10661, member_casual#10662, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]
Arguments: [year#10648 ASC NULLS FIRST, month#10649 ASC NULLS FIRST], false, 0

(14) WriteFiles
Input [18]: [ride_id#10650, rideable_type#10651, started_at#10652, ended_at#10653, start_station_name#10654, start_station_id#10655, end_station_name#10656, end_station_id#10657, start_lat#10658, start_lng#10659, end_lat#10660, end_lng#10661, member_casual#10662, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]

(15) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data, false, [year#10648, month#10649], Parquet, [partitionOverwriteMode=dynamic, __partition_columns=["year","month"], path=s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data/], Overwrite, [ride_id, rideable_type, started_at, ended_at, start_station_name, start_station_id, end_station_name, end_station_id, start_lat, start_lng, end_lat, end_lng, member_casual, _source_file, _processed_dttm, _start_station_ride_num, year, month]

(16) Filter
Input [21]: [_processed_dttm#10645, _source_file#10644, _start_station_ride_num#10647, end_lat#10660, end_lng#10661, end_station_id#10657, end_station_name#10656, ended_at#10653, member_casual#10662, month#10649, ride_id#10650, rideable_type#10651, start_lat#10658, start_lng#10659, start_station_id#10655, start_station_name#10654, started_at#10652, valid_ride_id#10641, valid_station#10643, valid_time#10642, year#10648]
Condition : (((((((isnotnull(valid_time#10642) AND valid_ride_id#10641) AND valid_time#10642) AND valid_station#10643) AND isnotnull(ride_id#10650)) AND NOT (ride_id#10650 = )) AND isnotnull(rideable_type#10651)) AND NOT (rideable_type#10651 = ))

(17) Project
Output [18]: [ride_id#10650, rideable_type#10651, started_at#10652, ended_at#10653, start_station_name#10654, start_station_id#10655, end_station_name#10656, end_station_id#10657, start_lat#10658, start_lng#10659, end_lat#10660, end_lng#10661, member_casual#10662, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]
Input [21]: [_processed_dttm#10645, _source_file#10644, _start_station_ride_num#10647, end_lat#10660, end_lng#10661, end_station_id#10657, end_station_name#10656, ended_at#10653, member_casual#10662, month#10649, ride_id#10650, rideable_type#10651, start_lat#10658, start_lng#10659, start_station_id#10655, start_station_name#10654, started_at#10652, valid_ride_id#10641, valid_station#10643, valid_time#10642, year#10648]

(18) Sort
Input [18]: [ride_id#10650, rideable_type#10651, started_at#10652, ended_at#10653, start_station_name#10654, start_station_id#10655, end_station_name#10656, end_station_id#10657, start_lat#10658, start_lng#10659, end_lat#10660, end_lng#10661, member_casual#10662, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]
Arguments: [year#10648 ASC NULLS FIRST, month#10649 ASC NULLS FIRST], false, 0

(19) WriteFiles
Input [18]: [ride_id#10650, rideable_type#10651, started_at#10652, ended_at#10653, start_station_name#10654, start_station_id#10655, end_station_name#10656, end_station_id#10657, start_lat#10658, start_lng#10659, end_lat#10660, end_lng#10661, member_casual#10662, _source_file#10644, _processed_dttm#10645, _start_station_ride_num#10647, year#10648, month#10649]

(20) Execute InsertIntoHadoopFsRelationCommand
Input: []
Arguments: s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data, false, [year#10648, month#10649], Parquet, [partitionOverwriteMode=dynamic, __partition_columns=["year","month"], path=s3a://rzvde-g8-kirsanov-dmitry/raw_valid/citibike_data/], Overwrite, [ride_id, rideable_type, started_at, ended_at, start_station_name, start_station_id, end_station_name, end_station_id, start_lat, start_lng, end_lat, end_lng, member_casual, _source_file, _processed_dttm, _start_station_ride_num, year, month]

(21) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true