remake column indexes and query processing of filters (#12388)
authorClint Wylie <cwylie@apache.org>
Wed, 11 May 2022 06:27:08 +0000 (23:27 -0700)
committerGitHub <noreply@github.com>
Wed, 11 May 2022 06:27:08 +0000 (11:57 +0530)
commit9e5a940cf1fc04b00f9cb0216ec8766d8fed0ee2
treeab769891ae64a781e7fb14438f28bcac5ee31da7
parentdeb69d1bc03aef784a563260bed1d21505495439
remake column indexes and query processing of filters (#12388)

Following up on #12315, which pushed most of the logic of building ImmutableBitmap into BitmapIndex in order to hide the details of how column indexes are implemented from the Filter implementations, this PR totally refashions how Filter consume indexes. The end result, while a rather dramatic reshuffling of the existing code, should be extraordinarily flexible, eventually allowing us to model any type of index we can imagine, and providing the machinery to build the filters that use them, while also allowing for other column implementations to implement the built-in index types to provide adapters to make use indexing in the current set filters that Druid provides.
75 files changed:
benchmarks/src/test/java/org/apache/druid/benchmark/BoundFilterBenchmark.java
benchmarks/src/test/java/org/apache/druid/benchmark/DimensionPredicateFilterBenchmark.java
benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
benchmarks/src/test/java/org/apache/druid/benchmark/LikeFilterBenchmark.java
benchmarks/src/test/java/org/apache/druid/benchmark/MockColumnIndexSelector.java [moved from benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java with 53% similarity]
indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java
processing/src/main/java/org/apache/druid/query/filter/ColumnIndexSelector.java [moved from processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java with 58% similarity]
processing/src/main/java/org/apache/druid/query/filter/Filter.java
processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java
processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java
processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java
processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
processing/src/main/java/org/apache/druid/query/search/AutoStrategy.java
processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java [deleted file]
processing/src/main/java/org/apache/druid/segment/ColumnSelectorColumnIndexSelector.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/FilteredOffset.java
processing/src/main/java/org/apache/druid/segment/IndexIO.java
processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java
processing/src/main/java/org/apache/druid/segment/IntIteratorUtils.java
processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java
processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java
processing/src/main/java/org/apache/druid/segment/VirtualColumn.java
processing/src/main/java/org/apache/druid/segment/VirtualColumns.java
processing/src/main/java/org/apache/druid/segment/column/AllFalseBitmapColumnIndex.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/AllTrueBitmapColumnIndex.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/BitmapColumnIndex.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/BitmapIndex.java [deleted file]
processing/src/main/java/org/apache/druid/segment/column/BitmapIndexes.java [deleted file]
processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java
processing/src/main/java/org/apache/druid/segment/column/ColumnHolder.java
processing/src/main/java/org/apache/druid/segment/column/ColumnIndexCapabilities.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/ColumnIndexSupplier.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/DruidPredicateIndex.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/LexicographicalRangeIndex.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/SimpleColumnHolder.java
processing/src/main/java/org/apache/druid/segment/column/SimpleColumnIndexCapabilities.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/column/StringValueSetIndex.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java
processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java
processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java
processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
processing/src/main/java/org/apache/druid/segment/filter/FalseFilter.java
processing/src/main/java/org/apache/druid/segment/filter/Filters.java
processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java
processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java
processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java
processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java
processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java
processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java
processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java
processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java
processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedStringIndexSupplier.java [new file with mode: 0644]
processing/src/main/java/org/apache/druid/segment/serde/NoIndexesColumnIndexSupplier.java [moved from processing/src/main/java/org/apache/druid/segment/serde/SpatialIndexColumnPartSupplier.java with 61% similarity]
processing/src/main/java/org/apache/druid/segment/serde/NullColumnPartSerde.java
processing/src/main/java/org/apache/druid/segment/serde/StringBitmapIndexColumnPartSupplier.java [deleted file]
processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
processing/src/test/java/org/apache/druid/segment/ColumnSelectorColumnIndexSelectorTest.java [moved from processing/src/test/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelectorTest.java with 56% similarity]
processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java
processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java
processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
processing/src/test/java/org/apache/druid/segment/filter/ExtractionDimFilterTest.java
processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java
processing/src/test/java/org/apache/druid/segment/filter/FiltersTest.java [deleted file]
processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
processing/src/test/java/org/apache/druid/segment/serde/NullColumnPartSerdeTest.java
processing/src/test/java/org/apache/druid/segment/virtual/DummyStringVirtualColumn.java
processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
services/src/main/java/org/apache/druid/cli/DumpSegment.java
services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java
sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java