HIVE-21103: PartitionManagementTask should not modify DN configs to avoid closing...
authorPrasanth Jayachandran <prasanthj@apache.org>
Tue, 12 Feb 2019 08:30:23 +0000 (00:30 -0800)
committerPrasanth Jayachandran <prasanthj@apache.org>
Tue, 12 Feb 2019 08:30:23 +0000 (00:30 -0800)
standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionManagementTask.java

index 313f87b..7c1e30a 100644 (file)
@@ -731,9 +731,10 @@ public class MetastoreConf {
       "metastore.partition.management.table.types", "MANAGED_TABLE,EXTERNAL_TABLE",
       "Comma separated list of table types to use for partition management"),
     PARTITION_MANAGEMENT_TASK_THREAD_POOL_SIZE("metastore.partition.management.task.thread.pool.size",
-      "metastore.partition.management.task.thread.pool.size", 5,
+      "metastore.partition.management.task.thread.pool.size", 3,
       "Partition management uses thread pool on to which tasks are submitted for discovering and retaining the\n" +
-      "partitions. This determines the size of the thread pool."),
+      "partitions. This determines the size of the thread pool. Note: Increasing the thread pool size will cause\n" +
+      "threadPoolSize * maxConnectionPoolSize connections to backend db"),
     PARTITION_MANAGEMENT_CATALOG_NAME("metastore.partition.management.catalog.name",
       "metastore.partition.management.catalog.name", "hive",
       "Automatic partition management will look for tables under the specified catalog name"),
index 901bf80..59001b5 100644 (file)
@@ -190,9 +190,6 @@ public class PartitionManagementTask implements MetastoreTaskThread {
     // when invalid path is encountered as these are background threads. We just want to skip and move on. Users will
     // have to fix the invalid paths via external means.
     conf.set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "skip");
-    // since msck runs in thread pool and each of them create their own metastore client, we don't want explosion of
-    // connections to metastore for embedded mode. Also we don't need too many db connections anyway.
-    conf.setInt(MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.getVarname(), 2);
   }
 
   private static class MsckThread implements Runnable {