[AMBARI-25077] Grafana on a workload cluster should show visualization of only it...
authoravijayanhwx <avijayan@hortonworks.com>
Wed, 2 Jan 2019 18:39:46 +0000 (10:39 -0800)
committerGitHub <noreply@github.com>
Wed, 2 Jan 2019 18:39:46 +0000 (10:39 -0800)
ambari-metrics-grafana/ambari-metrics/datasource.js
ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java

index dd64566..18e2709 100644 (file)
@@ -59,7 +59,7 @@ define([
 
       //We get a list of components and their associated metrics.
       this.initMetricAppidMapping = function () {
-        return this.doAmbariRequest({url: '/ws/v1/timeline/metrics/metadata'})
+        return this.doAmbariRequest({url: '/metadata'})
           .then(function (items) {
             items = items.data;
             allMetrics = {};
@@ -227,7 +227,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform +
+            url: '?metricNames=' + target.metric + metricTransform +
             metricAggregator + "&hostname=" + target.hosts + '&appId=' + target.app + instanceId + '&startTime=' + from +
             '&endTime=' + to + precision + seriesAggregator
           }).then(
@@ -268,7 +268,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform
+            url: '?metricNames=' + target.metric + metricTransform
             + metricAggregator + '&hostname=' + tHost + '&appId=' + target.app + instanceId + '&startTime=' + from +
             '&endTime=' + to + precision + seriesAggregator
           }).then(
@@ -302,7 +302,7 @@ define([
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           var templatedComponent = (_.isEmpty(tComponent)) ? target.app : tComponent;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform
+            url: '?metricNames=' + target.metric + metricTransform
             + metricAggregator + '&hostname=' + target.templatedHost + '&appId=' + templatedComponent + instanceId
             + '&startTime=' + from + '&endTime=' + to + precision + topN + seriesAggregator
           }).then(
@@ -317,7 +317,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + encodeURIComponent(target.queue) + metricTransform
+            url: '?metricNames=' + encodeURIComponent(target.queue) + metricTransform
             + metricAggregator + '&appId=resourcemanager' + instanceId + '&startTime=' + from +
             '&endTime=' + to + precision + seriesAggregator
           }).then(
@@ -330,7 +330,7 @@ define([
             + target.precision;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.hbMetric + instanceId + '&appId=hbase&startTime='
+            url: '?metricNames=' + target.hbMetric + instanceId + '&appId=hbase&startTime='
             + from + '&endTime=' + to + precision + seriesAggregator
           }).then(
             allHostMetricsData(target)
@@ -345,7 +345,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.kbMetric + metricTransform + instanceId
+            url: '?metricNames=' + target.kbMetric + metricTransform + instanceId
             + metricAggregator + '&appId=kafka_broker&startTime=' + from +
             '&endTime=' + to + precision + seriesAggregator
           }).then(
@@ -361,7 +361,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.nnMetric + metricTransform + instanceId
+            url: '?metricNames=' + target.nnMetric + metricTransform + instanceId
             + metricAggregator + '&appId=namenode&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
           }).then(
             allHostMetricsData(target)
@@ -377,7 +377,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.sTopoMetric + metricTransform + instanceId
+            url: '?metricNames=' + target.sTopoMetric + metricTransform + instanceId
             + metricAggregator + '&appId=nimbus&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
           }).then(
             allHostMetricsData(target)
@@ -393,7 +393,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.sCoreMetric + metricTransform + instanceId
+            url: '?metricNames=' + target.sCoreMetric + metricTransform + instanceId
             + metricAggregator + '&appId=ambari-infra-solr&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
           }).then(
             allHostMetricsData(target)
@@ -408,7 +408,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.sCollectionMetric + metricTransform + instanceId
+            url: '?metricNames=' + target.sCollectionMetric + metricTransform + instanceId
             + metricAggregator + '&appId=ambari-infra-solr&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
           }).then(
             allHostMetricsData(target)
@@ -424,7 +424,7 @@ define([
           var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
           var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
           return self.doAmbariRequest({
-            url: '/ws/v1/timeline/metrics?metricNames=' + target.sDataSourceMetric + metricTransform + instanceId
+            url: '?metricNames=' + target.sDataSourceMetric + metricTransform + instanceId
             + metricAggregator + '&appId=druid&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
           }).then(
             allHostMetricsData(target)
@@ -1111,7 +1111,7 @@ define([
        */
       this.testDatasource = function () {
         return this.doAmbariRequest({
-          url: '/ws/v1/timeline/metrics/metadata',
+          url: '/metadata',
           method: 'GET'
         }).then(function (response) {
           console.log(response);
@@ -1161,7 +1161,7 @@ define([
         }
         return this.doAmbariRequest({
           method: 'GET',
-          url: '/ws/v1/timeline/metrics/instances?' + 'appId=' + app
+          url: '/instance?' + 'appId=' + app
         }).then(function (response) {
           var clusters = [];
           var data = response.data;
@@ -1188,7 +1188,7 @@ define([
         }
         return this.doAmbariRequest({
           method: 'GET',
-          url: '/ws/v1/timeline/metrics/instances?' + 'appId=' + app + '&instanceId=' + cluster
+          url: '/instance?' + 'appId=' + app + '&instanceId=' + cluster
         }).then(function (response) {
           var hosts = [];
           var data = response.data;
index c90c93b..3619af6 100644 (file)
@@ -17,7 +17,6 @@
  */
 package org.apache.ambari.metrics.core.timeline;
 
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
 import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
 
@@ -87,6 +86,7 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
   private TimelineMetricMetadataManager metricMetadataManager;
   private MetricCollectorHAController haController;
   private boolean containerMetricsDisabled = false;
+  private String defaultInstanceId = "";
 
   /**
    * Construct the service.
@@ -218,6 +218,7 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
           "delay = " + initDelay + ", delay = " + delay);
       }
       containerMetricsDisabled = configuration.isContainerMetricsDisabled();
+      defaultInstanceId = configuration.getDefaultInstanceId();
       isInitialized = true;
     }
 
@@ -258,7 +259,7 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
     List<String> transientMetricNames = new ArrayList<>();
 
     if (configuration.getTimelineMetricsMultipleClusterSupport() && StringUtils.isEmpty(instanceId)) {
-      instanceId = DEFAULT_INSTANCE_ID;
+      instanceId = this.defaultInstanceId;
     }
 
     List<byte[]> uuids = metricMetadataManager.getUuidsForGetMetricQuery(metricFunctions.keySet(),
index b6ff202..6f1e1c0 100644 (file)
@@ -29,7 +29,6 @@ import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguratio
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CONTAINER_METRICS_TTL;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DATE_TIERED_COMPACTION_POLICY;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.FIFO_COMPACTION_POLICY_CLASS;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_MAX_RETRIES;
 import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT;
@@ -203,6 +202,7 @@ public class PhoenixHBaseAccessor {
   private TimelineMetricMetadataManager metadataManagerInstance;
   private Set<String> eventMetricPatterns = new HashSet<>();
   private boolean supportMultipleClusterMetrics = false;
+  private String defaultInstanceId = "";
 
   private Map<String, Integer> tableTTL = new HashMap<>();
 
@@ -263,6 +263,7 @@ public class PhoenixHBaseAccessor {
     tableTTL.put(METRIC_TRANSIENT_TABLE_NAME, metricsConf.getInt(METRICS_TRANSIENT_TABLE_TTL, 7 * 86400)); //7 days
 
     this.supportMultipleClusterMetrics = Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"));
+    this.defaultInstanceId = configuration.getDefaultInstanceId();
 
     if (cacheEnabled) {
       LOG.debug("Initialising and starting metrics cache committer thread...");
@@ -903,6 +904,10 @@ public class PhoenixHBaseAccessor {
 
       boolean acceptMetric = TimelineMetricsFilter.acceptMetric(tm);
 
+      if (supportMultipleClusterMetrics && StringUtils.isEmpty(tm.getInstanceId())) {
+        tm.setInstanceId(defaultInstanceId);
+      }
+
       // Write to metadata cache on successful write to store
       if (metadataManager != null) {
         metadataManager.putIfModifiedTimelineMetricMetadata(
@@ -912,9 +917,6 @@ public class PhoenixHBaseAccessor {
                 tm.getHostName(), tm.getAppId());
 
         if (!tm.getAppId().equals("FLUME_HANDLER")) {
-          if (supportMultipleClusterMetrics && StringUtils.isEmpty(tm.getInstanceId())) {
-            tm.setInstanceId(DEFAULT_INSTANCE_ID);
-          }
           metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName());
         }
       }
index 9766f0c..f8f4bb0 100644 (file)
@@ -292,6 +292,9 @@ public class TimelineMetricConfiguration {
   public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
     "timeline.metrics.support.multiple.clusters";
 
+  public static final String TIMELINE_METRICS_DEFAULT_INSTANCE_ID =
+    "timeline.metrics.default.instanceid";
+
   public static final String TIMELINE_METRICS_EVENT_METRIC_PATTERNS =
     "timeline.metrics.downsampler.event.metric.patterns";
 
@@ -512,6 +515,13 @@ public class TimelineMetricConfiguration {
     return false;
   }
 
+  public String getDefaultInstanceId() {
+    if (metricsConf != null) {
+      return metricsConf.get(TIMELINE_METRICS_DEFAULT_INSTANCE_ID, DEFAULT_INSTANCE_ID);
+    }
+    return DEFAULT_INSTANCE_ID;
+  }
+
   public String getTimelineServiceRpcAddress() {
     String defaultRpcAddress = "0.0.0.0:60200";
     if (metricsConf != null) {
index ee50051..092da51 100644 (file)
@@ -154,7 +154,7 @@ public class PhoenixTransactSQL {
       "START_TIME UNSIGNED_LONG, " +
       "SUPPORTS_AGGREGATION BOOLEAN, " +
       "IS_WHITELISTED BOOLEAN " +
-      "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID)) " +
+      "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID, INSTANCE_ID)) " +
       "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
 
   public static final String CREATE_HOSTED_APPS_METADATA_TABLE_SQL =
index b3378be..71477ec 100644 (file)
@@ -36,6 +36,7 @@ import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
@@ -288,6 +289,31 @@ public class TimelineWebServices {
   }
 
   @GET
+  @Path("metrics/{instanceId}")
+  @Produces({ MediaType.APPLICATION_JSON })
+  public TimelineMetrics getTimelineMetricsForInstance(
+    @Context HttpServletRequest req,
+    @Context HttpServletResponse res,
+    @PathParam("instanceId") String instanceId,
+    @QueryParam("metricNames") String metricNames,
+    @QueryParam("appId") String appId,
+    @QueryParam("hostname") String hostname,
+    @QueryParam("startTime") String startTime,
+    @QueryParam("endTime") String endTime,
+    @QueryParam("precision") String precision,
+    @QueryParam("limit") String limit,
+    @QueryParam("grouped") String grouped,
+    @QueryParam("topN") String topN,
+    @QueryParam("topNFunction") String topNFunction,
+    @QueryParam("isBottomN") String isBottomN,
+    @QueryParam("seriesAggregateFunction") String seriesAggregateFunction
+  ) {
+
+    return getTimelineMetrics(req, res, metricNames, appId, instanceId, hostname, startTime, endTime, precision, limit,
+      grouped, topN, topNFunction, isBottomN, seriesAggregateFunction);
+  }
+
+  @GET
   @Path("/metrics/summary")
   @Produces({ MediaType.APPLICATION_JSON })
   public TimelineMetricServiceSummary getTimelineMetricServiceSummary(
@@ -325,6 +351,28 @@ public class TimelineWebServices {
   }
 
   @GET
+  @Path("/metrics/{instanceId}/metadata")
+  @Produces({ MediaType.APPLICATION_JSON })
+  public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadataForInstance(
+    @Context HttpServletRequest req,
+    @Context HttpServletResponse res,
+    @QueryParam("appId") String appId,
+    @QueryParam("metricName") String metricPattern,
+    @QueryParam("includeAll") String includeBlacklistedMetrics
+  ) {
+    init(res);
+
+    try {
+      return timelineMetricStore.getTimelineMetricMetadata(
+        parseStr(appId),
+        parseStr(metricPattern),
+        parseBoolean(includeBlacklistedMetrics));
+    } catch (Exception e) {
+      throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+    }
+  }
+
+  @GET
   @Path("/metrics/hosts")
   @Produces({ MediaType.APPLICATION_JSON })
   public Map<String, Set<String>> getHostedAppsMetadata(
@@ -341,7 +389,7 @@ public class TimelineWebServices {
   }
 
   @GET
-  @Path("/metrics/instances")
+  @Path("/metrics/instance")
   @Produces({ MediaType.APPLICATION_JSON })
   public Map<String, Map<String, Set<String>>> getClusterHostsMetadata(
     @Context HttpServletRequest req,
@@ -359,6 +407,24 @@ public class TimelineWebServices {
   }
 
   @GET
+  @Path("/metrics/{instanceId}/instance")
+  @Produces({ MediaType.APPLICATION_JSON })
+  public Map<String, Map<String, Set<String>>> getClusterHostsMetadataForInstance(
+    @Context HttpServletRequest req,
+    @Context HttpServletResponse res,
+    @QueryParam("appId") String appId,
+    @PathParam("instanceId") String instanceId
+  ) {
+    init(res);
+
+    try {
+      return timelineMetricStore.getInstanceHostsMetadata(instanceId, appId);
+    } catch (Exception e) {
+      throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
+    }
+  }
+
+  @GET
   @Path("/metrics/uuid")
   @Produces({ MediaType.APPLICATION_JSON })
   public byte[] getUuid(