[HUDI-845] Added locking capability to allow multiple writers (#2374)
authorn3nash <nagarwal@uber.com>
Tue, 16 Mar 2021 23:43:53 +0000 (16:43 -0700)
committerGitHub <noreply@github.com>
Tue, 16 Mar 2021 23:43:53 +0000 (16:43 -0700)
commit74241947c123c860a1b0344f25cef316440a70d6
treecbe65f830d4646fe51d7913ad0d5e080a0272f6d
parentb038623ed3318404f9bc4707f005a9fc458c0adf
[HUDI-845] Added locking capability to allow multiple writers (#2374)

* [HUDI-845] Added locking capability to allow multiple writers
1. Added LockProvider API for pluggable lock methodologies
2. Added Resolution Strategy API to allow for pluggable conflict resolution
3. Added TableService client API to schedule table services
4. Added Transaction Manager for wrapping actions within transactions
88 files changed:
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitUtilities.java
hudi-client/hudi-client-common/pom.xml
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentOperation.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConflictResolutionStrategy.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleConcurrentFileWritesConflictResolutionStrategy.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedLockProvider.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataConversionUtils.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/BaseCleanActionExecutor.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/BaseCleanPlanActionExecutor.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/BaseScheduleCompactionActionExecutor.java
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/AbstractUpgradeDowngrade.java
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/FileSystemBasedLockProviderTestClass.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestZookeeperBasedLockProvider.java [new file with mode: 0644]
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java [new file with mode: 0644]
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/clean/FlinkScheduleCleanActionExecutor.java [new file with mode: 0644]
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/BaseFlinkCommitActionExecutor.java
hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java
hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java
hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/clean/JavaScheduleCleanActionExecutor.java [new file with mode: 0644]
hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/clean/SparkCleanPlanActionExecutor.java [new file with mode: 0644]
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java [new file with mode: 0644]
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiveLog.java
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestHoodieBackedMetadata.java
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java
hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java [new file with mode: 0644]
hudi-common/src/main/java/org/apache/hudi/common/lock/LockProvider.java [new file with mode: 0644]
hudi-common/src/main/java/org/apache/hudi/common/lock/LockState.java [new file with mode: 0644]
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieMetadataWrapper.java [new file with mode: 0644]
hudi-common/src/main/java/org/apache/hudi/common/model/TableServiceType.java [new file with mode: 0644]
hudi-common/src/main/java/org/apache/hudi/common/model/WriteConcurrencyMode.java [new file with mode: 0644]
hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
hudi-common/src/main/java/org/apache/hudi/common/util/ClusteringUtils.java
hudi-common/src/main/java/org/apache/hudi/common/util/CommitUtils.java
hudi-common/src/main/java/org/apache/hudi/exception/HoodieLockException.java [new file with mode: 0644]
hudi-common/src/main/java/org/apache/hudi/exception/HoodieWriteConflictException.java [new file with mode: 0644]
hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/ZookeeperTestService.java
hudi-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java
hudi-integ-test/pom.xml
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteWriter.java
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/WriterContext.java
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/ZookeeperServiceProvider.java [new file with mode: 0644]
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DFSDeltaWriterAdapter.java
hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkInsertNode.scala [new file with mode: 0644]
hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkUpsertNode.scala [new file with mode: 0644]
hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
hudi-integ-test/src/test/resources/unit-test-cow-dag-spark-datasource.yaml [new file with mode: 0644]
hudi-spark-datasource/hudi-spark/pom.xml
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveMetastoreLockProvider.java [new file with mode: 0644]
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveMetastoreLockProvider.java [new file with mode: 0644]
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java
packaging/hudi-integ-test-bundle/pom.xml
packaging/hudi-spark-bundle/pom.xml
packaging/hudi-utilities-bundle/pom.xml
pom.xml