support Aliyun OSS service as deep storage (#9898)
authorfrank chen <frank.chen021@outlook.com>
Thu, 2 Jul 2020 05:20:53 +0000 (13:20 +0800)
committerGitHub <noreply@github.com>
Thu, 2 Jul 2020 05:20:53 +0000 (22:20 -0700)
commit60c6bd5b4c44f28f5dbff48e70c6138ce35204b6
tree080adf3483393adaa431044d200ce124b763c189
parente2c5bcc22d2c175bba51fbb2a9c90303ecdf6f44
support Aliyun OSS service as deep storage (#9898)

* init commit, all tests passed

* fix format

Signed-off-by: frank chen <frank.chen021@outlook.com>
* data stored successfully

* modify config path

* add doc

* add aliyun-oss extension to project

* remove descriptor deletion code to avoid warning message output by aliyun client

* fix warnings reported by lgtm-com

* fix ci warnings

Signed-off-by: frank chen <frank.chen021@outlook.com>
* fix errors reported by intellj inspection check

Signed-off-by: frank chen <frank.chen021@outlook.com>
* fix doc spelling check

Signed-off-by: frank chen <frank.chen021@outlook.com>
* fix dependency warnings reported by ci

Signed-off-by: frank chen <frank.chen021@outlook.com>
* fix warnings reported by CI

Signed-off-by: frank chen <frank.chen021@outlook.com>
* add package configuration to support showing extension info

Signed-off-by: frank chen <frank.chen021@outlook.com>
* add IT test cases and fix bugs

Signed-off-by: frank chen <frank.chen021@outlook.com>
* 1. code review comments adopted
2. change schema from 'aliyun-oss' to 'oss'

Signed-off-by: frank chen <frank.chen021@outlook.com>
* add license info

Signed-off-by: frank chen <frank.chen021@outlook.com>
* fix doc

Signed-off-by: frank chen <frank.chen021@outlook.com>
* exclude execution of IT testcases of OSS extension from CI

Signed-off-by: frank chen <frank.chen021@outlook.com>
* put the extensions under contrib group and add to distribution

* fix names in test cases

* add unit test to cover OssInputSource

* fix names in test cases

* fix dependency problem reported by CI

Signed-off-by: frank chen <frank.chen021@outlook.com>
45 files changed:
.travis.yml
LICENSE
distribution/pom.xml
docs/development/extensions-contrib/aliyun-oss-extensions.md [new file with mode: 0644]
docs/development/extensions.md
extensions-contrib/aliyun-oss-extensions/pom.xml [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/data/input/aliyun/OssInputSourceTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssObjectSummaryIteratorTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java [new file with mode: 0644]
extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java [new file with mode: 0644]
integration-tests/docker/environment-configs/override-examples/oss [new file with mode: 0644]
integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java [new file with mode: 0644]
integration-tests/src/test/java/org/apache/druid/tests/indexer/ITOssToOssParallelIndexTest.java [new file with mode: 0644]
pom.xml
website/.spelling