Added new data preparation phases
authorAnton Gozhiy <anton5813@gmail.com>
Sat, 12 Mar 2022 09:20:48 +0000 (11:20 +0200)
committerAnton Gozhiy <anton5813@gmail.com>
Fri, 8 Apr 2022 13:34:36 +0000 (16:34 +0300)
commit5adf73d0cb6d2a8abd77f385f2449e2d97f00b25
tree39602b733f757411430c9bf057918609702ca1ac
parentb4e0895bbd9c8204d3c1e19bf2531f0b4d490346
Added new data preparation phases

New preparation phases:
- "del" for deletion data that needs to be fresh
- "mkdir" for directory creation
- "ddl" for ddl files execution
- "post_cp" for copying previously generated data
Other changes:
- Added a new exclude option "all", so starting tests with "-x all" argument will not run test suites with any dependency
- Refactored test suites to use the newly added preparation phases instead of bash scripts
- Temporary excluded several suites where "gen" scripts were too complicated
115 files changed:
.gitignore
bin/run_tests
framework/pom.xml
framework/resources/Datasources/ctas/create_tables_complex_parquet.ddl
framework/resources/Datasources/ctas/create_tables_parquet.ddl
framework/resources/Datasources/ctas/decimal/fragments/decimal_fragments.sh
framework/resources/Datasources/ctas_auto_partition/create_tables.ddl
framework/resources/Datasources/ctas_auto_partition/ctas_existing_partition_pruning.ddl
framework/resources/Datasources/ctas_auto_partition/ctas_tpch_multiple_partitions.ddl
framework/resources/Datasources/ctas_auto_partition/ctas_tpch_single_partition1.ddl
framework/resources/Datasources/join/crt_tbl_prtnby_nulls.ddl
framework/resources/Datasources/join/crt_tbls_partition_by_l.ddl
framework/resources/Datasources/join/crt_tbls_partition_by_r.ddl
framework/resources/Datasources/limit0/implicit_cast_with_views/create_implicit_cast_views.sql
framework/resources/Datasources/min_max_dir/scripts/refresh_metadata.ddl
framework/resources/Datasources/subqueries/create_tables.sql
framework/resources/Datasources/subqueries/create_views.sql
framework/resources/Datasources/tpcds/createViewsParquet.sql
framework/resources/Functional/complex/parquet/complex.json
framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/csv/data/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/data/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/hierarchical/plan/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/json/data/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/data/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/existing_partition_pruning/parquet/plan/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/general/data/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/general/plan/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/data/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_multiple_partitions/plan/ctas_auto_partition.json
framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition/tpch.json
framework/resources/Functional/ctas/ctas_auto_partition/tpch0.01_single_partition1/tpch.json
framework/resources/Functional/ctas/ctas_flatten/100000rows/ctas_flatten.json
framework/resources/Functional/ctas/ctas_flatten/2rows/ctas_flatten.json
framework/resources/Functional/ctas/ctas_joins_aggregates/ctas.json
framework/resources/Functional/decimal_parquet/decimal_parquet.json
framework/resources/Functional/drill_fragments/decimal/data/data.json
framework/resources/Functional/drill_fragments/decimal/plan/plan.json
framework/resources/Functional/filter/pushdown/item_star_operator/data.json
framework/resources/Functional/filter/pushdown/item_star_operator/plan.json
framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal.json
framework/resources/Functional/filter/pushdown/varchar_decimal/dfs/varchar_decimal_plan.json
framework/resources/Functional/group_by_alias/group_by_alias.json
framework/resources/Functional/int96/int96_data.json
framework/resources/Functional/int96/int96_plan.json
framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/data/ctas_auto_partition.json
framework/resources/Functional/interpreted_partition_pruning/ctas_auto_partition/hierarchical/plan/ctas_auto_partition.json
framework/resources/Functional/joins/nulleqjoin/nulleqjoin.json
framework/resources/Functional/limit0/aggregates/aggregation/bugs/data/bugs.json
framework/resources/Functional/limit0/aggregates/aggregation/bugs/plan/bugs.json
framework/resources/Functional/limit0/aggregates/aggregation/count_distinct/data/count_distinct.json
framework/resources/Functional/limit0/aggregates/aggregation/count_distinct/plan/count_distinct.json
framework/resources/Functional/limit0/aggregates/aggregation/group_by_case/data/case.json
framework/resources/Functional/limit0/aggregates/aggregation/group_by_case/plan/case.json
framework/resources/Functional/limit0/aggregates/aggregation/group_by_expression/data/group_by_expression.json
framework/resources/Functional/limit0/aggregates/aggregation/group_by_expression/plan/group_by_expression.json
framework/resources/Functional/limit0/aggregates/aggregation/multicolumn/data/multicolumn.json
framework/resources/Functional/limit0/aggregates/aggregation/multicolumn/plan/multicolumn.json
framework/resources/Functional/limit0/aggregates/aggregation/sanity/data/sanity.json
framework/resources/Functional/limit0/aggregates/aggregation/sanity/plan/sanity.json
framework/resources/Functional/limit0/aggregates/aggregation/scalar/data/scalar.json
framework/resources/Functional/limit0/aggregates/aggregation/scalar/plan/scalar.json
framework/resources/Functional/limit0/aggregates/tpcds_variants/parquet/data/aggregate.json
framework/resources/Functional/limit0/aggregates/tpcds_variants/parquet/plan/aggregate.json
framework/resources/Functional/limit0/filters/data/data.json
framework/resources/Functional/limit0/filters/plan/data.json
framework/resources/Functional/limit0/functions/data/limit0_functions.json
framework/resources/Functional/limit0/functions/plan/limit0_functions.json
framework/resources/Functional/limit0/implicit_cast_with_views/data/implicit_cast.json
framework/resources/Functional/limit0/implicit_cast_with_views/plan/implicit_cast.json
framework/resources/Functional/limit0/union/data/queries.json
framework/resources/Functional/limit0/union/plan/queries.json
framework/resources/Functional/limit0/union_all/prq_union_all/data/prq_union_all.json
framework/resources/Functional/limit0/union_all/prq_union_all/plan/prq_union_all.json
framework/resources/Functional/limit0/window_functions/aggregates/data/queries.json
framework/resources/Functional/limit0/window_functions/aggregates/plan/queries.json
framework/resources/Functional/limit0/window_functions/bugs/data/bugs.json
framework/resources/Functional/limit0/window_functions/bugs/plan/bugs.json
framework/resources/Functional/limit0/window_functions/empty_over_clause/data/empty_over_clause.json
framework/resources/Functional/limit0/window_functions/empty_over_clause/plan/empty_over_clause.json
framework/resources/Functional/limit0/window_functions/multiple_partitions/data/multiple_partitions.json
framework/resources/Functional/limit0/window_functions/multiple_partitions/plan/multiple_partitions.json
framework/resources/Functional/limit0/window_functions/tpcds_variants/data/tpcds_parquet_sf1.json
framework/resources/Functional/limit0/window_functions/tpcds_variants/plan/tpcds_parquet_sf1.json
framework/resources/Functional/metadata_caching/data/metadata_caching_small.json
framework/resources/Functional/metadata_caching/generated_caches/metadata_caching_small.json
framework/resources/Functional/metadata_caching/partition_pruning/data/metadata_caching_pp.json
framework/resources/Functional/metadata_caching/partition_pruning/plan/metadata_caching_pp.json
framework/resources/Functional/metadata_caching/plan/metadata_caching_small.json
framework/resources/Functional/min_max_dir/min_max_dir.json
framework/resources/Functional/parquet_storage/parquet_date/auto_partition/data/parquet_date.json
framework/resources/Functional/parquet_storage/parquet_date/auto_partition/plan/parquet_date.json
framework/resources/Functional/parquet_storage/parquet_date/mc_parquet_date/parquet_date.json
framework/resources/Functional/partition_pruning/dfs/parquet/data/partitionDirectory.json
framework/resources/Functional/partition_pruning/dfs/parquet/plan/regex_no_order/partitionDirectory.json
framework/resources/Functional/schema_change_empty_batch/json/empty_batch_json.json
framework/resources/Functional/subqueries/subquery_in_select/subquery_in_select.json
framework/resources/Functional/table_function/positive/data/table_function.json
framework/resources/Functional/table_stats/stats/positive/stats.json
framework/resources/Functional/table_stats/stats/query/query.json
framework/resources/Functional/text_storage/testcases/textReadGroup.json
framework/resources/Functional/tpcds/impala/json/tpcds_sf1_json.json
framework/resources/Functional/tpcds/impala/parquet/tpcds_parquet_sf1.json
framework/resources/Functional/tpcds/impala/text/tpcds_text_sf1.json
framework/resources/Functional/tpcds/sanity/json/tpcds_sf1_json.json
framework/resources/Functional/tpcds/sanity/parquet/tpcds_parquet_sf1.json
framework/resources/Functional/tpcds/sanity/text/tpcds_text_sf1.json
framework/resources/Functional/tpcds/variants/json/tpcds_json_sf1.json
framework/resources/Functional/tpcds/variants/parquet/tpcds_parquet_sf1.json
framework/resources/Functional/tpcds/variants/text/tpcds_text_sf1.json
framework/resources/Functional/udfs/udfs.json
framework/resources/Functional/window_functions/tpcds/tpcds_parquet_sf1.json
framework/resources/Functional/window_functions/tpcds_variants/tpcds_parquet_sf1.json
framework/src/main/java/org/apache/drill/test/framework/DrillTestDefaults.java
framework/src/main/java/org/apache/drill/test/framework/TestDriver.java
framework/src/main/java/org/apache/drill/test/framework/Utils.java