AMBARI-24290 : Fix issues in AMS multiple cluster support. (#1764)
authoravijayanhwx <avijayan@hortonworks.com>
Tue, 24 Jul 2018 19:40:55 +0000 (12:40 -0700)
committerGitHub <noreply@github.com>
Tue, 24 Jul 2018 19:40:55 +0000 (12:40 -0700)
* AMBARI-24290 : Fix issues in AMS multiple cluster support.

* AMBARI-24290 : Fix issues in AMS multiple cluster support (2)

* AMBARI-24290 : Fix issues in AMS multiple cluster support (3)

ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java

index d768d0d..c90c93b 100644 (file)
@@ -17,6 +17,7 @@
  */
 package org.apache.ambari.metrics.core.timeline;
 
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
 import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
 
@@ -52,6 +53,7 @@ import org.apache.ambari.metrics.core.timeline.query.Condition;
 import org.apache.ambari.metrics.core.timeline.query.ConditionBuilder;
 import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
 import org.apache.ambari.metrics.core.timeline.query.TopNCondition;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -255,6 +257,10 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
     TimelineMetrics metrics = new TimelineMetrics();
     List<String> transientMetricNames = new ArrayList<>();
 
+    if (configuration.getTimelineMetricsMultipleClusterSupport() && StringUtils.isEmpty(instanceId)) {
+      instanceId = DEFAULT_INSTANCE_ID;
+    }
+
     List<byte[]> uuids = metricMetadataManager.getUuidsForGetMetricQuery(metricFunctions.keySet(),
       hostnames,
       applicationId,
@@ -282,7 +288,7 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
 
     Condition condition = conditionBuilder.build();
 
-    if (hostnames == null || hostnames.isEmpty()) {
+    if (CollectionUtils.isEmpty(hostnames)) {
       metrics = hBaseAccessor.getAggregateMetricRecords(condition, metricFunctions);
     } else {
       metrics = hBaseAccessor.getMetricRecords(condition, metricFunctions);
index 0946511..c0427e7 100644 (file)
@@ -29,6 +29,7 @@ import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguratio
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CONTAINER_METRICS_TTL;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DATE_TIERED_COMPACTION_POLICY;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.FIFO_COMPACTION_POLICY_CLASS;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_MAX_RETRIES;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT;
@@ -39,6 +40,7 @@ import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguratio
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_COMPACTION_CLASS_KEY;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_ENGINE_CLASS;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_EVENT_METRIC_PATTERNS;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TRANSIENT_METRIC_PATTERNS;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_DAILY_TABLE_TTL;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_HOUR_TABLE_TTL;
@@ -199,6 +201,7 @@ public class PhoenixHBaseAccessor {
   private final boolean skipBlockCacheForAggregatorsEnabled;
   private TimelineMetricMetadataManager metadataManagerInstance;
   private Set<String> eventMetricPatterns = new HashSet<>();
+  private boolean supportMultipleClusterMetrics = false;
 
   private Map<String, Integer> tableTTL = new HashMap<>();
 
@@ -258,6 +261,8 @@ public class PhoenixHBaseAccessor {
     tableTTL.put(METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME, metricsConf.getInt(CLUSTER_DAILY_TABLE_TTL, 730 * 86400)); //2 years
     tableTTL.put(METRIC_TRANSIENT_TABLE_NAME, metricsConf.getInt(METRICS_TRANSIENT_TABLE_TTL, 7 * 86400)); //7 days
 
+    this.supportMultipleClusterMetrics = Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"));
+
     if (cacheEnabled) {
       LOG.debug("Initialising and starting metrics cache committer thread...");
       metricsCommiterThread = new MetricsCacheCommitterThread(this);
@@ -906,6 +911,9 @@ public class PhoenixHBaseAccessor {
                 tm.getHostName(), tm.getAppId());
 
         if (!tm.getAppId().equals("FLUME_HANDLER")) {
+          if (supportMultipleClusterMetrics && StringUtils.isEmpty(tm.getInstanceId())) {
+            tm.setInstanceId(DEFAULT_INSTANCE_ID);
+          }
           metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName());
         }
       }
index f7fd72a..9766f0c 100644 (file)
@@ -300,6 +300,8 @@ public class TimelineMetricConfiguration {
 
   public static final String HOST_APP_ID = "HOST";
 
+  public static final String DEFAULT_INSTANCE_ID = "DEFAULT";
+
   public static final String DEFAULT_INSTANCE_PORT = "12001";
 
   public static final String AMSHBASE_METRICS_WHITESLIST_FILE = "amshbase_metrics_whitelist";
index ba93792..878dcb3 100644 (file)
@@ -99,7 +99,7 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
       skipInterpolationMetricPatterns.addAll(getJavaMetricPatterns(skipInterpolationMetricPatternStrings));
     }
 
-    this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager, true);
+    this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager);
   }
 
   @Override