</includes>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target</directory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
<include>*-sources.jar</include>
</includes>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf</directory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory>
<outputDirectory>etc/hadoop</outputDirectory>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}</directory>
- <outputDirectory>/share/hadoop/${hadoop.component}/lib/slider</outputDirectory>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version}</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/lib/services</outputDirectory>
</fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>hadoop-yarn-slider-dist</id>
+ <id>hadoop-yarn-services-dist</id>
<formats>
<format>dir</format>
</formats>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
<version>${project.version}</version>
</dependency>
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "servicesapi" "run yarn-native-service rest server"
+ hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "service" "run a service"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
- servicesapi)
+ apiserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApplicationApiWebApp'
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
;;
service)
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*'
HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
#
# See ResourceManager for some examples
#
-#export YARN_SERVICESAPI_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-servicesapi.log-$(date +'%Y%m%d%H%M')"
+#export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ApplicationState;
-import org.apache.hadoop.yarn.service.api.records.ApplicationStatus;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
+/**
+ * The rest API endpoints for users to manage services on YARN.
+ */
@Singleton
@Path(CONTEXT_ROOT)
-public class ApplicationApiService {
+public class ApiServer {
private static final Logger LOG =
- LoggerFactory.getLogger(ApplicationApiService.class);
+ LoggerFactory.getLogger(ApiServer.class);
private static Configuration YARN_CONFIG = new YarnConfiguration();
private static ServiceClient SERVICE_CLIENT;
}
@POST
- @Path(APP_ROOT_PATH)
+ @Path(SERVICE_ROOT_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
- public Response createApplication(Application application) {
- LOG.info("POST: createApplication = {}", application);
- ApplicationStatus applicationStatus = new ApplicationStatus();
+ public Response createService(Service service) {
+ LOG.info("POST: createService = {}", service);
+ ServiceStatus serviceStatus = new ServiceStatus();
try {
- ApplicationId applicationId = SERVICE_CLIENT.actionCreate(application);
- LOG.info("Successfully created application " + application.getName()
+ ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service);
+ LOG.info("Successfully created service " + service.getName()
+ " applicationId = " + applicationId);
- applicationStatus.setState(ApplicationState.ACCEPTED);
- applicationStatus.setUri(
- CONTEXT_ROOT + APP_ROOT_PATH + "/" + application
+ serviceStatus.setState(ServiceState.ACCEPTED);
+ serviceStatus.setUri(
+ CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
.getName());
- return Response.status(Status.CREATED).entity(applicationStatus).build();
+ return Response.status(Status.CREATED).entity(serviceStatus).build();
} catch (IllegalArgumentException e) {
- applicationStatus.setDiagnostics(e.getMessage());
- return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+ serviceStatus.setDiagnostics(e.getMessage());
+ return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
.build();
} catch (Exception e) {
- String message = "Failed to create application " + application.getName();
+ String message = "Failed to create service " + service.getName();
LOG.error(message, e);
- applicationStatus.setDiagnostics(message + ": " + e.getMessage());
+ serviceStatus.setDiagnostics(message + ": " + e.getMessage());
return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
+ .entity(serviceStatus).build();
}
}
@GET
- @Path(APP_PATH)
+ @Path(SERVICE_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
- public Response getApplication(@PathParam(APP_NAME) String appName) {
- LOG.info("GET: getApplication for appName = {}", appName);
- ApplicationStatus applicationStatus = new ApplicationStatus();
-
- // app name validation
- if (!SliderUtils.isClusternameValid(appName)) {
- applicationStatus.setDiagnostics("Invalid application name: " + appName);
- applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
- .build();
- }
-
+ public Response getService(@PathParam(SERVICE_NAME) String appName) {
+ LOG.info("GET: getService for appName = {}", appName);
+ ServiceStatus serviceStatus = new ServiceStatus();
try {
- Application app = SERVICE_CLIENT.getStatus(appName);
+ Service app = SERVICE_CLIENT.getStatus(appName);
return Response.ok(app).build();
+ } catch (IllegalArgumentException e) {
+ serviceStatus.setDiagnostics(e.getMessage());
+ serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
+ return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+ .build();
} catch (Exception e) {
- LOG.error("Get application failed", e);
- applicationStatus
- .setDiagnostics("Failed to retrieve application: " + e.getMessage());
+ LOG.error("Get service failed", e);
+ serviceStatus
+ .setDiagnostics("Failed to retrieve service: " + e.getMessage());
return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
+ .entity(serviceStatus).build();
}
}
@DELETE
- @Path(APP_PATH)
+ @Path(SERVICE_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
- public Response deleteApplication(@PathParam(APP_NAME) String appName) {
- LOG.info("DELETE: deleteApplication for appName = {}", appName);
- return stopApplication(appName, true);
+ public Response deleteService(@PathParam(SERVICE_NAME) String appName) {
+ LOG.info("DELETE: deleteService for appName = {}", appName);
+ return stopService(appName, true);
}
- private Response stopApplication(String appName, boolean destroy) {
+ private Response stopService(String appName, boolean destroy) {
try {
SERVICE_CLIENT.actionStop(appName, destroy);
if (destroy) {
SERVICE_CLIENT.actionDestroy(appName);
- LOG.info("Successfully deleted application {}", appName);
+ LOG.info("Successfully deleted service {}", appName);
} else {
- LOG.info("Successfully stopped application {}", appName);
+ LOG.info("Successfully stopped service {}", appName);
}
return Response.status(Status.NO_CONTENT).build();
} catch (ApplicationNotFoundException e) {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- applicationStatus.setDiagnostics(
- "Application " + appName + " not found " + e.getMessage());
- return Response.status(Status.NOT_FOUND).entity(applicationStatus)
+ ServiceStatus serviceStatus = new ServiceStatus();
+ serviceStatus.setDiagnostics(
+ "Service " + appName + " not found " + e.getMessage());
+ return Response.status(Status.NOT_FOUND).entity(serviceStatus)
.build();
} catch (Exception e) {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- applicationStatus.setDiagnostics(e.getMessage());
+ ServiceStatus serviceStatus = new ServiceStatus();
+ serviceStatus.setDiagnostics(e.getMessage());
return Response.status(Status.INTERNAL_SERVER_ERROR)
- .entity(applicationStatus).build();
+ .entity(serviceStatus).build();
}
}
@Path(COMPONENT_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
- public Response updateComponent(@PathParam(APP_NAME) String appName,
+ public Response updateComponent(@PathParam(SERVICE_NAME) String appName,
@PathParam(COMPONENT_NAME) String componentName, Component component) {
if (component.getNumberOfContainers() < 0) {
return Response.status(Status.BAD_REQUEST).entity(
- "Application = " + appName + ", Component = " + component.getName()
+ "Service = " + appName + ", Component = " + component.getName()
+ ": Invalid number of containers specified " + component
.getNumberOfContainers()).build();
}
.get(componentName) + " to " + component.getNumberOfContainers())
.build();
} catch (YarnException | IOException e) {
- ApplicationStatus status = new ApplicationStatus();
+ ServiceStatus status = new ServiceStatus();
status.setDiagnostics(e.getMessage());
return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status)
.build();
}
@PUT
- @Path(APP_PATH)
+ @Path(SERVICE_PATH)
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
- public Response updateApplication(@PathParam(APP_NAME) String appName,
- Application updateAppData) {
- LOG.info("PUT: updateApplication for app = {} with data = {}", appName,
- updateAppData);
+ public Response updateService(@PathParam(SERVICE_NAME) String appName,
+ Service updateServiceData) {
+ LOG.info("PUT: updateService for app = {} with data = {}", appName,
+ updateServiceData);
- // Ignore the app name provided in updateAppData and always use appName
+ // Ignore the app name provided in updateServiceData and always use appName
// path param
- updateAppData.setName(appName);
+ updateServiceData.setName(appName);
// For STOP the app should be running. If already stopped then this
// operation will be a no-op. For START it should be in stopped state.
// If already running then this operation will be a no-op.
- if (updateAppData.getState() != null
- && updateAppData.getState() == ApplicationState.STOPPED) {
- return stopApplication(appName, false);
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STOPPED) {
+ return stopService(appName, false);
}
// If a START is requested
- if (updateAppData.getState() != null
- && updateAppData.getState() == ApplicationState.STARTED) {
- return startApplication(appName);
+ if (updateServiceData.getState() != null
+ && updateServiceData.getState() == ServiceState.STARTED) {
+ return startService(appName);
}
// If new lifetime value specified then update it
- if (updateAppData.getLifetime() != null
- && updateAppData.getLifetime() > 0) {
- return updateLifetime(appName, updateAppData);
+ if (updateServiceData.getLifetime() != null
+ && updateServiceData.getLifetime() > 0) {
+ return updateLifetime(appName, updateServiceData);
}
// flex a single component app
- if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil
- .hasComponent(updateAppData)) {
- Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData);
- return updateComponent(updateAppData.getName(), defaultComp.getName(),
+ if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
+ .hasComponent(updateServiceData)) {
+ Component defaultComp = ServiceApiUtil.createDefaultComponent(updateServiceData);
+ return updateComponent(updateServiceData.getName(), defaultComp.getName(),
defaultComp);
}
return Response.status(Status.NO_CONTENT).build();
}
- private Response updateLifetime(String appName, Application updateAppData) {
+ private Response updateLifetime(String appName, Service updateAppData) {
try {
String newLifeTime =
SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
- return Response.ok("Application " + appName + " lifeTime is successfully updated to "
+ return Response.ok("Service " + appName + " lifeTime is successfully updated to "
+ updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
} catch (Exception e) {
String message =
- "Failed to update application (" + appName + ") lifetime ("
+ "Failed to update service (" + appName + ") lifetime ("
+ updateAppData.getLifetime() + ")";
LOG.error(message, e);
return Response.status(Status.INTERNAL_SERVER_ERROR)
}
}
- private Response startApplication(String appName) {
+ private Response startService(String appName) {
try {
SERVICE_CLIENT.actionStart(appName);
- LOG.info("Successfully started application " + appName);
- return Response.ok("Application " + appName + " is successfully started").build();
+ LOG.info("Successfully started service " + appName);
+ return Response.ok("Service " + appName + " is successfully started").build();
} catch (Exception e) {
- String message = "Failed to start application " + appName;
+ String message = "Failed to start service " + appName;
LOG.info(message, e);
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(message + ": " + e.getMessage()).build();
--- /dev/null
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.webapp;
+
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.eclipse.jetty.webapp.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
+
+/**
+ * This class launches the web service using Hadoop HttpServer2 (which uses
+ * an embedded Jetty container). This is the entry point to your service.
+ * The Java command used to launch this app should call the main method.
+ */
+public class ApiServerWebApp extends AbstractService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(ApiServerWebApp.class);
+ private static final String SEP = ";";
+
+ // REST API server for YARN native services
+ private HttpServer2 apiServer;
+ private InetSocketAddress bindAddress;
+
+ public static void main(String[] args) throws IOException {
+ ApiServerWebApp apiWebApp = new ApiServerWebApp();
+ try {
+ apiWebApp.startWebApp();
+ } catch (Exception e) {
+ apiWebApp.close();
+ }
+ }
+
+ public ApiServerWebApp() {
+ super(ApiServerWebApp.class.getName());
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS,
+ DEFAULT_API_SERVER_ADDRESS , DEFAULT_API_SERVER_PORT);
+ logger.info("YARN API server running on " + bindAddress);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ doSecureLogin(getConfig());
+ }
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (apiServer != null) {
+ apiServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ private void doSecureLogin(org.apache.hadoop.conf.Configuration conf)
+ throws IOException {
+ SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB,
+ YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName());
+ addFilters(conf);
+ }
+
+ private void addFilters(org.apache.hadoop.conf.Configuration conf) {
+ // Always load pseudo authentication filter to parse "user.name" in an URL
+ // to identify a HTTP request's user.
+ boolean hasHadoopAuthFilterInitializer = false;
+ String filterInitializerConfKey = "hadoop.http.filter.initializers";
+ Class<?>[] initializersClasses =
+ conf.getClasses(filterInitializerConfKey);
+ List<String> targets = new ArrayList<String>();
+ if (initializersClasses != null) {
+ for (Class<?> initializer : initializersClasses) {
+ if (initializer.getName().equals(
+ AuthenticationFilterInitializer.class.getName())) {
+ hasHadoopAuthFilterInitializer = true;
+ break;
+ }
+ targets.add(initializer.getName());
+ }
+ }
+ if (!hasHadoopAuthFilterInitializer) {
+ targets.add(AuthenticationFilterInitializer.class.getName());
+ conf.set(filterInitializerConfKey, StringUtils.join(",", targets));
+ }
+ }
+
+ private void startWebApp() throws IOException {
+ URI uri = URI.create("http://" + NetUtils.getHostPortString(bindAddress));
+
+ apiServer = new HttpServer2.Builder()
+ .setName("api-server")
+ .setConf(getConfig())
+ .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
+ .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY)
+ .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+ .addEndpoint(uri).build();
+
+ String apiPackages =
+ ApiServer.class.getPackage().getName() + SEP
+ + GenericExceptionHandler.class.getPackage().getName() + SEP
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+ apiServer.addJerseyResourcePackage(apiPackages, "/*");
+
+ try {
+ logger.info("Service starting up. Logging start...");
+ apiServer.start();
+ logger.info("Server status = {}", apiServer.toString());
+ for (Configuration conf : apiServer.getWebAppContext()
+ .getConfigurations()) {
+ logger.info("Configurations = {}", conf);
+ }
+ logger.info("Context Path = {}", Collections.singletonList(
+ apiServer.getWebAppContext().getContextPath()));
+ logger.info("ResourceBase = {}", Collections.singletonList(
+ apiServer.getWebAppContext().getResourceBase()));
+ logger.info("War = {}", Collections
+ .singletonList(apiServer.getWebAppContext().getWar()));
+ } catch (Exception ex) {
+ logger.error("Hadoop HttpServer2 App **failed**", ex);
+ throw ex;
+ }
+ }
+}
+++ /dev/null
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.webapp;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URI;
-import java.util.Arrays;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.eclipse.jetty.webapp.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class launches the web application using Hadoop HttpServer2 (which uses
- * an embedded Jetty container). This is the entry point to your application.
- * The Java command used to launch this app should call the main method.
- */
-public class ApplicationApiWebApp extends AbstractService {
- private static final Logger logger = LoggerFactory
- .getLogger(ApplicationApiWebApp.class);
- private static final String SEP = ";";
-
- // REST API server for YARN native services
- private HttpServer2 applicationApiServer;
-
- public static void main(String[] args) throws IOException {
- ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp();
- try {
- apiWebApp.startWebApp();
- } catch (Exception e) {
- if (apiWebApp != null) {
- apiWebApp.close();
- }
- }
- }
-
- public ApplicationApiWebApp() {
- super(ApplicationApiWebApp.class.getName());
- }
-
- @Override
- protected void serviceStart() throws Exception {
- startWebApp();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (applicationApiServer != null) {
- applicationApiServer.stop();
- }
- super.serviceStop();
- }
-
- protected void startWebApp() throws IOException {
- // The port that we should run on can be set into an environment variable
- // Look for that variable and default to 9191 if it isn't there.
- String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT);
- if (StringUtils.isEmpty(webPort)) {
- webPort = "9191";
- }
-
- String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST);
- if (StringUtils.isEmpty(webHost)) {
- webHost = InetAddress.getLocalHost().getHostName();
- }
- logger.info("YARN native services REST API running on host {} and port {}",
- webHost, webPort);
- logger.info("Configuration = {}", getConfig());
-
- applicationApiServer = new HttpServer2.Builder()
- .setName("services-rest-api")
- .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
-
- String apiPackages =
- ApplicationApiService.class.getPackage().getName() + SEP
- + GenericExceptionHandler.class.getPackage().getName() + SEP
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
- applicationApiServer.addJerseyResourcePackage(apiPackages, "/*");
-
- try {
- logger.info("Application starting up. Logging start...");
- applicationApiServer.start();
- logger.info("Server status = {}", applicationApiServer.toString());
- for (Configuration conf : applicationApiServer.getWebAppContext()
- .getConfigurations()) {
- logger.info("Configurations = {}", conf);
- }
- logger.info("Context Path = {}", Arrays.asList(applicationApiServer
- .getWebAppContext().getContextPath()));
- logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer
- .getWebAppContext().getResourceBase()));
- logger.info("War = {}",
- Arrays.asList(applicationApiServer.getWebAppContext().getWar()));
- } catch (Exception ex) {
- logger.error("Hadoop HttpServer2 App **failed**", ex);
- throw ex;
- }
- }
-}
Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like.
- To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component apps to complex multi-component assemblies needing orchestration.
+ To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component service to complex multi-component assemblies needing orchestration.
We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management.
- http
- https
# will be prefixed to all paths
-basePath: /services/v1/
+basePath: /ws/v1/
consumes:
- application/json
produces:
- application/json
paths:
- /applications:
+ /services:
get:
- summary: List of applications/services running in the cluster
- description: Get a list of all currently running applications (response includes a minimal projection of the application info). For more details do a GET on a specific application name.
+ summary: List of services running in the cluster
+ description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name.
responses:
200:
- description: An array of applications
+ description: An array of services
schema:
type: array
items:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
post:
- summary: Create an application/service
- description: Create an application. The request JSON is an Application object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the application creation request. There is no guarantee that the application will actually reach a RUNNING state. Resource availability and several other factors determines if the application will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the application and determine its state.
+ summary: Create a service
+ description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state.
parameters:
- - name: Application
+ - name: Service
in: body
- description: Application request object
+ description: Service request object
required: true
schema:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
responses:
202:
description: Request accepted
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
- /applications/{app_name}:
+ /services/{service_name}:
get:
- summary: Get an application/service details
- description: Return the details (including containers) of a running application
+ summary: Get service details
+ description: Return the details (including containers) of a running service
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
200:
- description: An application object
+ description: a service object
schema:
type: object
items:
- $ref: '#/definitions/Application'
+ $ref: '#/definitions/Service'
examples:
- app_name: logsearch
+ service_name: logsearch
artifact:
id: logsearch:latest
type: docker
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
put:
- summary: Update an application/service or upgrade the binary version of the components of a running application
- description: Update the runtime properties of an application. As of now, only update of lifetime and number of instances (flexing) of the components of an application is supported. The PUT operation is also used to orchestrate an upgrade of the application containers to a newer version of their artifacts.
+ summary: Update a service or upgrade the binary version of the components of a running service
+ description: Update the runtime properties of a service. As of now, only update of lifetime and number of instances (flexing) of the components of a service is supported. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts.
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
204:
description: Update or upgrade was successful
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
delete:
- summary: Destroy application/service
- description: Destroy an application and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
+ summary: Destroy service
+ description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
parameters:
- - name: app_name
+ - name: service_name
in: path
- description: Application name
+ description: Service name
required: true
type: string
responses:
204:
description: Destroy was successful
404:
- description: Application does not exist
+ description: Service does not exist
default:
description: Unexpected error
schema:
- $ref: '#/definitions/ApplicationStatus'
+ $ref: '#/definitions/ServiceStatus'
definitions:
- Application:
- description: An Application resource has the following attributes.
+ Service:
+ description: a service resource has the following attributes.
required:
- name
properties:
name:
type: string
- description: A unique application name. If Registry DNS is enabled, the max length is 63 characters.
+ description: A unique service name. If Registry DNS is enabled, the max length is 63 characters.
id:
type: string
- description: A unique application id.
+ description: A unique service id.
artifact:
- description: Artifact of single-component applications.
+ description: Artifact of single-component service.
$ref: '#/definitions/Artifact'
resource:
- description: Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.
+ description: Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.
$ref: '#/definitions/Resource'
launch_command:
type: string
- description: The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.
+ description: The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.
launch_time:
type: string
format: date
- description: The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
+ description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
number_of_containers:
type: integer
format: int64
- description: Number of containers for each app-component in the application. Each app-component can further override this app-level global default.
+ description: Number of containers for each component in the service. Each component can further override this service-level global default.
number_of_running_containers:
type: integer
format: int64
- description: In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
+ description: In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
lifetime:
type: integer
format: int64
- description: Life time (in seconds) of the application from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
+ description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
placement_policy:
- description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
+ description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
$ref: '#/definitions/PlacementPolicy'
components:
- description: Components of an application.
+ description: Components of a service.
type: array
items:
$ref: '#/definitions/Component'
configuration:
- description: Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.
+ description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.
$ref: '#/definitions/Configuration'
containers:
- description: Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.
+ description: Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.
type: array
items:
$ref: '#/definitions/Container'
state:
- description: State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.
- $ref: '#/definitions/ApplicationState'
+ description: State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.
+ $ref: '#/definitions/ServiceState'
quicklinks:
type: object
- description: A blob of key-value pairs of quicklinks to be exported for an application.
+ description: A blob of key-value pairs of quicklinks to be exported for a service.
additionalProperties:
type: string
queue:
type: string
- description: The YARN queue that this application should be submitted to.
+ description: The YARN queue that this service should be submitted to.
Resource:
description:
- Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.
+ Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.
properties:
profile:
type: string
type: string
description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.
PlacementPolicy:
- description: Placement policy of an instance of an application. This feature is in the works in YARN-4902.
+ description: Placement policy of an instance of a service. This feature is in the works in YARN-6592.
properties:
label:
type: string
- description: Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.
+ description: Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.
Artifact:
- description: Artifact of an application component. If not specified, component will just run the bare launch command and no artifact will be localized.
+ description: Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized.
required:
- id
properties:
id:
type: string
- description: Artifact id. Examples are package location uri for tarball based apps, image name for docker, name of application, etc.
+ description: Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc.
type:
type: string
- description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For APPLICATION type, the application specified will be read and its components will be added into this application. The original component with artifact type APPLICATION will be removed (any properties specified in the original component will be ignored).
+ description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored).
enum:
- DOCKER
- TARBALL
- - APPLICATION
+ - SERVICE
default: DOCKER
uri:
type: string
description: Artifact location to support multiple artifact stores (optional).
Component:
- description: One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.
+ description: One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.
required:
- name
properties:
name:
type: string
- description: Name of the application component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
+ description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
dependencies:
type: array
items:
type: string
- description: An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.
+ description: An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG.
readiness_check:
- description: Readiness check for this app-component.
+ description: Readiness check for this component.
$ref: '#/definitions/ReadinessCheck'
artifact:
- description: Artifact of the component (optional). If not specified, the application level global artifact takes effect.
+ description: Artifact of the component (optional). If not specified, the service level global artifact takes effect.
$ref: '#/definitions/Artifact'
launch_command:
type: string
description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).
resource:
- description: Resource of this component (optional). If not specified, the application level global resource takes effect.
+ description: Resource of this component (optional). If not specified, the service level global resource takes effect.
$ref: '#/definitions/Resource'
number_of_containers:
type: integer
format: int64
- description: Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.
+ description: Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.
run_privileged_container:
type: boolean
description: Run all containers of this component in privileged mode (YARN-4262).
placement_policy:
- description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.
+ description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.
$ref: '#/definitions/PlacementPolicy'
configuration:
- description: Config properties for this app-component.
+ description: Config properties for this component.
$ref: '#/definitions/Configuration'
quicklinks:
type: array
items:
type: string
- description: A list of quicklink keys defined at the application level, and to be resolved by this component.
+ description: A list of quicklink keys defined at the service level, and to be resolved by this component.
ReadinessCheck:
- description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.
+ description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.
required:
- type
properties:
additionalProperties:
type: string
artifact:
- description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET
+ description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET
$ref: '#/definitions/Artifact'
Configuration:
- description: Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+ description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
properties:
properties:
type: object
- description: A blob of key-value pairs of common application properties.
+ description: A blob of key-value pairs of common service properties.
additionalProperties:
type: string
env:
type: object
- description: A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.
+ description: A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.
additionalProperties:
type: string
files:
- description: Array of list of files that needs to be created and made available as volumes in the application component containers.
+ description: Array of list of files that needs to be created and made available as volumes in the service component containers.
type: array
items:
$ref: '#/definitions/ConfigFile'
ConfigFile:
- description: A config file that needs to be created and made available as a volume in an application component container.
+ description: A config file that needs to be created and made available as a volume in a service component container.
properties:
type:
type: string
type: object
description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.
Container:
- description: An instance of a running application container.
+ description: An instance of a running service container.
properties:
id:
type: string
- description: Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.
+ description: Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.
launch_time:
type: string
format: date
type: string
description: The bare node or host in which the container is running, e.g. cn008.example.com.
state:
- description: State of the container of an application.
+ description: State of the container of a service.
$ref: '#/definitions/ContainerState'
component_name:
type: string
privileged_container:
type: boolean
description: Container running in privileged mode or not.
- ApplicationState:
- description: The current state of an application.
+ ServiceState:
+ description: The current state of a service.
properties:
state:
type: string
- description: enum of the state of the application
+ description: enum of the state of the service
enum:
- ACCEPTED
- STARTED
- STOPPED
- FAILED
ContainerState:
- description: The current state of the container of an application.
+ description: The current state of the container of a service.
properties:
state:
type: string
description: enum of the state of the container
enum:
- INIT
+ - STARTED
- READY
- ApplicationStatus:
- description: The current status of a submitted application, returned as a response to the GET API.
+ ServiceStatus:
+ description: The current status of a submitted service, returned as a response to the GET API.
properties:
diagnostics:
type: string
- description: Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.
+ description: Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.
state:
- description: Application state.
- $ref: '#/definitions/ApplicationState'
+ description: Service state.
+ $ref: '#/definitions/ServiceState'
code:
type: integer
format: int32
- description: An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.
+ description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.
+++ /dev/null
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export SLIDER_VERSION=${project.version}
-export HDP_VERSION=${HDP_VERSION:-$SLIDER_VERSION}
-export SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
-export LIB_PARENT_DIR=`dirname $SCRIPT_DIR`
-export JAVA_HOME=${JAVA_HOME:-/usr/jdk64/jdk1.8.0_40}
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
-export REST_SERVICE_PORT=${REST_SERVICE_PORT:-9191}
-export APP_RUNAS_USER=${APP_RUNAS_USER:-root}
-export REST_SERVICE_LOG_DIR=${REST_SERVICE_LOG_DIR:-/tmp/}
-export JAVA_OPTS="-Xms256m -Xmx1024m -XX:+PrintGC -Xloggc:$REST_SERVICE_LOG_DIR/gc.log"
-$JAVA_HOME/bin/java $JAVA_OPTS -cp .:$HADOOP_CONF_DIR:$LIB_PARENT_DIR/services-api/*:$LIB_PARENT_DIR/slider/* -DREST_SERVICE_LOG_DIR=$REST_SERVICE_LOG_DIR -Dlog4j.configuration=log4j-server.properties -Dslider.libdir=$LIB_PARENT_DIR/slider org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp 1>>$REST_SERVICE_LOG_DIR/restservice-out.log 2>&1
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider</artifactId>
+ <artifactId>hadoop-yarn-services</artifactId>
<version>3.0.0-beta1-SNAPSHOT</version>
</parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-slider-core</artifactId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
<packaging>jar</packaging>
- <name>Apache Hadoop YARN Slider Core</name>
+ <name>Apache Hadoop YARN Services Core</name>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
@Override
public GetStatusResponseProto getStatus(GetStatusRequestProto request)
throws IOException, YarnException {
- String stat = ServiceApiUtil.jsonSerDeser.toJson(context.application);
+ String stat = ServiceApiUtil.jsonSerDeser.toJson(context.service);
return GetStatusResponseProto.newBuilder().setStatus(stat).build();
}
import com.google.common.cache.LoadingCache;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
public class ServiceContext {
- public Application application = null;
+ public Service service = null;
public SliderFileSystem fs;
public String serviceHdfsDir = "";
public ApplicationAttemptId attemptId;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.servicemonitor.ServiceMonitor;
+import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
ContainerId amContainerId = getAMContainerId();
ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId();
- LOG.info("Application attemptId: " + attemptId);
+ LOG.info("Service AppAttemptId: " + attemptId);
context.attemptId = attemptId;
// configure AM to wait forever for RM
protected void loadApplicationJson(ServiceContext context,
SliderFileSystem fs) throws IOException {
- context.application = ServiceApiUtil
- .loadApplicationFrom(fs, new Path(amArgs.getAppDefPath()));
- LOG.info(context.application.toString());
+ context.service = ServiceApiUtil
+ .loadServiceFrom(fs, new Path(amArgs.getAppDefPath()));
+ LOG.info(context.service.toString());
}
@Override
* limitations under the License.
*/
-package org.apache.hadoop.yarn.service.metrics;
+package org.apache.hadoop.yarn.service;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
@Metric("containers preempted")
public MutableGaugeInt containersPreempted;
- @Metric("containers exceeded limits")
- public MutableGaugeInt containersLimitsExceeded;
-
@Metric("containers surplus")
public MutableGaugeInt surplusContainers;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants;
-import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.ServiceApiConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.service.component.ComponentEvent;
import org.apache.hadoop.yarn.service.component.ComponentEventType;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
-import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
/**
private static final Logger LOG =
LoggerFactory.getLogger(ServiceScheduler.class);
- private Application app;
+ private Service app;
// component_name -> component
private final Map<String, Component> componentsByName =
private ContainerLaunchService containerLaunchService;
public ServiceScheduler(ServiceContext context) {
- super(context.application.getName());
+ super(context.service.getName());
this.context = context;
}
public void buildInstance(ServiceContext context, Configuration configuration)
throws YarnException {
- app = context.application;
+ app = context.service;
executorService = Executors.newScheduledThreadPool(10);
RegistryOperations registryClient = RegistryOperationsFactory
.createInstance("ServiceScheduler", configuration);
serviceMetrics = ServiceMetrics
.register(app.getName(), "Metrics for service");
serviceMetrics.tag("type", "Metrics type [component or service]", "service");
- serviceMetrics.tag("appId", "Application id for service", app.getId());
+ serviceMetrics.tag("appId", "Service id for service", app.getId());
amRMClient = createAMRMClient();
addIfService(amRMClient);
+ "Navigate to the failed component for more details.";
amRMClient
.unregisterApplicationMaster(FinalApplicationStatus.ENDED, msg, "");
- LOG.info("Application " + app.getName()
+ LOG.info("Service " + app.getName()
+ " unregistered with RM, with attemptId = " + context.attemptId
+ ", diagnostics = " + diagnostics);
super.serviceStop();
}
private void registerServiceInstance(ApplicationAttemptId attemptId,
- Application application) throws IOException {
- LOG.info("Registering " + attemptId + ", " + application.getName()
+ Service service) throws IOException {
+ LOG.info("Registering " + attemptId + ", " + service.getName()
+ " into registry");
ServiceRecord serviceRecord = new ServiceRecord();
serviceRecord.set(YarnRegistryAttributes.YARN_ID,
context.clientAMService.getBindAddress()));
// set any provided attributes
- setUserProvidedServiceRecordAttributes(application.getConfiguration(),
+ setUserProvidedServiceRecordAttributes(service.getConfiguration(),
serviceRecord);
executorService.submit(new Runnable() {
return componentsByName;
}
- public Application getApp() {
+ public Service getApp() {
return app;
}
* limitations under the License.
*/
-package org.apache.hadoop.yarn.service.api.constants;
+package org.apache.hadoop.yarn.service.api;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability;
/**
- * Artifact of an application component.
+ * Artifact of an service component.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "Artifact of an application component")
+@ApiModel(description = "Artifact of an service component")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Artifact implements Serializable {
private String id = null;
public enum TypeEnum {
- DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
+ DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");
private String value;
private String uri = null;
/**
- * Artifact id. Examples are package location uri for tarball based apps,
+ * Artifact id. Examples are package location uri for tarball based services,
* image name for docker, etc.
**/
public Artifact id(String id) {
return this;
}
- @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based apps, image name for docker, etc.")
+ @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based services, image name for docker, etc.")
@JsonProperty("id")
public String getId() {
return id;
private String uri;
/**
- * Resource location, e.g. \
- * "/applications/helloworld/containers/container_e3751_1458061340047_0008_01_000002\
- * "
+ * Resource location for a service, e.g.
+ * /ws/v1/services/helloworld
+ *
**/
public String getUri() {
return uri;
import org.apache.hadoop.classification.InterfaceStability;
/**
- * One or more components of the application. If the application is HBase say,
+ * One or more components of the service. If the service is HBase say,
* then the component can be a simple role like master or regionserver. If the
- * application is a complex business webapp then a component can be other
- * applications say Kafka or Storm. Thereby it opens up the support for complex
- * and nested applications.
+ * service is a complex business webapp then a component can be other
+ * services say Kafka or Storm. Thereby it opens up the support for complex
+ * and nested services.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.")
+@ApiModel(description = "One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
Collections.synchronizedList(new ArrayList<Container>());
/**
- * Name of the application component (mandatory).
+ * Name of the service component (mandatory).
**/
public Component name(String name) {
this.name = name;
return this;
}
- @ApiModelProperty(example = "null", required = true, value = "Name of the application component (mandatory).")
+ @ApiModelProperty(example = "null", required = true, value = "Name of the service component (mandatory).")
@JsonProperty("name")
public String getName() {
return name;
}
/**
- * An array of application components which should be in READY state (as
+ * An array of service components which should be in READY state (as
* defined by readiness check), before this component can be started. The
- * dependencies across all components of an application should be represented
+ * dependencies across all components of a service should be represented
* as a DAG.
**/
public Component dependencies(List<String> dependencies) {
return this;
}
- @ApiModelProperty(example = "null", value = "An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.")
+ @ApiModelProperty(example = "null", value = "An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an service should be represented as a DAG.")
@JsonProperty("dependencies")
public List<String> getDependencies() {
return dependencies;
}
/**
- * Readiness check for this app-component.
+ * Readiness check for this component.
**/
public Component readinessCheck(ReadinessCheck readinessCheck) {
this.readinessCheck = readinessCheck;
return this;
}
- @ApiModelProperty(example = "null", value = "Readiness check for this app-component.")
+ @ApiModelProperty(example = "null", value = "Readiness check for this component.")
@JsonProperty("readiness_check")
public ReadinessCheck getReadinessCheck() {
return readinessCheck;
}
/**
- * Artifact of the component (optional). If not specified, the application
+ * Artifact of the component (optional). If not specified, the service
* level global artifact takes effect.
**/
public Component artifact(Artifact artifact) {
return this;
}
- @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the application level global artifact takes effect.")
+ @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the service level global artifact takes effect.")
@JsonProperty("artifact")
public Artifact getArtifact() {
return artifact;
}
/**
- * Resource of this component (optional). If not specified, the application
+ * Resource of this component (optional). If not specified, the service
* level global resource takes effect.
**/
public Component resource(Resource resource) {
return this;
}
- @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the application level global resource takes effect.")
+ @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the service level global resource takes effect.")
@JsonProperty("resource")
public Resource getResource() {
return resource;
}
/**
- * Number of containers for this app-component (optional). If not specified,
- * the application level global number_of_containers takes effect.
+ * Number of containers for this component (optional). If not specified,
+ * the service level global number_of_containers takes effect.
**/
public Component numberOfContainers(Long numberOfContainers) {
this.numberOfContainers = numberOfContainers;
return this;
}
- @ApiModelProperty(example = "null", value = "Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.")
+ @ApiModelProperty(example = "null", value = "Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.")
@JsonProperty("number_of_containers")
public Long getNumberOfContainers() {
return numberOfContainers;
this.numberOfContainers = numberOfContainers;
}
- @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
+ @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
@JsonProperty("containers")
public List<Container> getContainers() {
return containers;
/**
* Advanced scheduling and placement policies for all containers of this
- * component (optional). If not specified, the app level placement_policy
+ * component (optional). If not specified, the service level placement_policy
* takes effect. Refer to the description at the global level for more
* details.
**/
return this;
}
- @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.")
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.")
@JsonProperty("placement_policy")
public PlacementPolicy getPlacementPolicy() {
return placementPolicy;
}
/**
- * Config properties for this app-component.
+ * Config properties for this component.
**/
public Component configuration(Configuration configuration) {
this.configuration = configuration;
return this;
}
- @ApiModelProperty(example = "null", value = "Config properties for this app-component.")
+ @ApiModelProperty(example = "null", value = "Config properties for this component.")
@JsonProperty("configuration")
public Configuration getConfiguration() {
return configuration;
}
/**
- * A list of quicklink keys defined at the application level, and to be
+ * A list of quicklink keys defined at the service level, and to be
* resolved by this component.
**/
public Component quicklinks(List<String> quicklinks) {
return this;
}
- @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the application level, and to be resolved by this component.")
+ @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the service level, and to be resolved by this component.")
@JsonProperty("quicklinks")
public List<String> getQuicklinks() {
return quicklinks;
/**
* A config file that needs to be created and made available as a volume in an
- * application component container.
+ * service component container.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "A config file that needs to be created and made available as a volume in an application component container.")
+@ApiModel(description = "A config file that needs to be created and made available as a volume in an service component container.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
/**
* The absolute path that this configuration file should be mounted as, in the
- * application container.
+ * service container.
**/
public ConfigFile destFile(String destFile) {
this.destFile = destFile;
return this;
}
- @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the application container.")
+ @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the service container.")
@JsonProperty("dest_file")
public String getDestFile() {
return destFile;
import java.util.Objects;
/**
- * Set of configuration properties that can be injected into the application
+ * Set of configuration properties that can be injected into the service
* components via envs, files and custom pluggable helper docker containers.
* Files of several standard formats like xml, properties, json, yaml and
* templates will be supported.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
+@ApiModel(description = "Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Configuration implements Serializable {
private List<ConfigFile> files = new ArrayList<ConfigFile>();
/**
- * A blob of key-value pairs of common application properties.
+ * A blob of key-value pairs of common service properties.
**/
public Configuration properties(Map<String, String> properties) {
this.properties = properties;
return this;
}
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common application properties.")
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common service properties.")
@JsonProperty("properties")
public Map<String, String> getProperties() {
return properties;
/**
* A blob of key-value pairs which will be appended to the default system
- * properties and handed off to the application at start time. All placeholder
+ * properties and handed off to the service at start time. All placeholder
* references to properties will be substituted before injection.
**/
public Configuration env(Map<String, String> env) {
return this;
}
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.")
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.")
@JsonProperty("env")
public Map<String, String> getEnv() {
return env;
/**
* Array of list of files that needs to be created and made available as
- * volumes in the application component containers.
+ * volumes in the service component containers.
**/
public Configuration files(List<ConfigFile> files) {
this.files = files;
return this;
}
- @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the application component containers.")
+ @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the service component containers.")
@JsonProperty("files")
public List<ConfigFile> getFiles() {
return files;
import org.apache.hadoop.classification.InterfaceStability;
/**
- * An instance of a running application container.
+ * An instance of a running service container.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "An instance of a running application container")
+@ApiModel(description = "An instance of a running service container")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
private Boolean privilegedContainer = null;
/**
- * Unique container id of a running application, e.g.
+ * Unique container id of a running service, e.g.
* container_e3751_1458061340047_0008_01_000002.
**/
public Container id(String id) {
return this;
}
- @ApiModelProperty(example = "null", value = "Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.")
+ @ApiModelProperty(example = "null", value = "Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002.")
@JsonProperty("id")
public String getId() {
return id;
}
/**
- * State of the container of an application.
+ * State of the container of an service.
**/
public Container state(ContainerState state) {
this.state = state;
return this;
}
- @ApiModelProperty(example = "null", value = "State of the container of an application.")
+ @ApiModelProperty(example = "null", value = "State of the container of an service.")
@JsonProperty("state")
public ContainerState getState() {
return state;
import org.apache.hadoop.classification.InterfaceStability;
/**
- * Placement policy of an instance of an application. This feature is in the
+ * Placement policy of an instance of an service. This feature is in the
* works in YARN-4902.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "Placement policy of an instance of an application. This feature is in the works in YARN-4902.")
+@ApiModel(description = "Placement policy of an instance of an service. This feature is in the works in YARN-4902.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public class PlacementPolicy implements Serializable {
private static final long serialVersionUID = 4341110649551172231L;
private String label = null;
/**
- * Assigns an app to a named partition of the cluster where the application
- * desires to run (optional). If not specified all apps are submitted to a
- * default label of the app owner. One or more labels can be setup for each
- * application owner account with required constraints like no-preemption,
+ * Assigns a service to a named partition of the cluster where the service
+ * desires to run (optional). If not specified all services are submitted to
+ * a default label of the service owner. One or more labels can be setup for
+ * each service owner account with required constraints like no-preemption,
* sla-99999, preemption-ok, etc.
**/
public PlacementPolicy label(String label) {
return this;
}
- @ApiModelProperty(example = "null", value = "Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
+ @ApiModelProperty(example = "null", value = "Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
@JsonProperty("label")
public String getLabel() {
return label;
/**
* A custom command or a pluggable helper container to determine the readiness
- * of a container of a component. Readiness for every application is different.
+ * of a container of a component. Readiness for every service is different.
* Hence the need for a simple interface, with scope to support advanced
* usecases.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.")
+@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
* specified, this helper container typically hosts the http uri and
* encapsulates the complex scripts required to perform actual container
* readiness check. At the end it is expected to respond a 204 No content just
- * like the simplified use case. This pluggable framework benefits application
- * owners who can run applications without any packaging modifications. Note,
+ * like the simplified use case. This pluggable framework benefits service
+ * owners who can run services without any packaging modifications. Note,
* artifacts of type docker only is supported for now.
**/
public ReadinessCheck artifact(Artifact artifact) {
return this;
}
- @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now.")
+ @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now.")
@JsonProperty("artifact")
public Artifact getArtifact() {
return artifact;
/**
* Resource determines the amount of resources (vcores, memory, network, etc.)
* usable by a container. This field determines the resource to be applied for
- * all the containers of a component or application. The resource specified at
- * the app (or global) level can be overriden at the component level. Only one
- * of profile OR cpu & memory are exepected. It raises a validation
+ * all the containers of a component or service. The resource specified at
+ * the service (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu & memory are expected. It raises a validation
* exception otherwise.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
+@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public class Resource extends BaseResource implements Cloneable {
private static final long serialVersionUID = -6431667797380250037L;
if (this.memory == null) {
return 0;
}
- return Long.valueOf(memory);
+ return Long.parseLong(memory);
}
@Override
import java.util.Objects;
/**
- * An Application resource has the following attributes.
+ * An Service resource has the following attributes.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "An Application resource has the following attributes.")
+@ApiModel(description = "An Service resource has the following attributes.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
"lifetime", "containers" })
-public class Application extends BaseResource {
+public class Service extends BaseResource {
private static final long serialVersionUID = -4491694636566094885L;
private String name = null;
private List<Component> components = new ArrayList<>();
private Configuration configuration = new Configuration();
private List<Container> containers = new ArrayList<>();
- private ApplicationState state = null;
+ private ServiceState state = null;
private Map<String, String> quicklinks = new HashMap<>();
private String queue = null;
/**
- * A unique application name.
+ * A unique service name.
**/
- public Application name(String name) {
+ public Service name(String name) {
this.name = name;
return this;
}
- @ApiModelProperty(example = "null", required = true, value = "A unique application name.")
+ @ApiModelProperty(example = "null", required = true, value = "A unique service name.")
@JsonProperty("name")
public String getName() {
return name;
}
/**
- * A unique application id.
+ * A unique service id.
**/
- public Application id(String id) {
+ public Service id(String id) {
this.id = id;
return this;
}
- @ApiModelProperty(example = "null", value = "A unique application id.")
+ @ApiModelProperty(example = "null", value = "A unique service id.")
@JsonProperty("id")
public String getId() {
return id;
}
/**
- * Artifact of single-component applications. Mandatory if components
+ * Artifact of single-component services. Mandatory if components
* attribute is not specified.
**/
- public Application artifact(Artifact artifact) {
+ public Service artifact(Artifact artifact) {
this.artifact = artifact;
return this;
}
- @ApiModelProperty(example = "null", value = "Artifact of single-component applications. Mandatory if components attribute is not specified.")
+ @ApiModelProperty(example = "null", value = "Artifact of single-component services. Mandatory if components attribute is not specified.")
@JsonProperty("artifact")
public Artifact getArtifact() {
return artifact;
}
/**
- * Resource of single-component applications or the global default for
- * multi-component applications. Mandatory if it is a single-component
- * application and if cpus and memory are not specified at the Application
+ * Resource of single-component services or the global default for
+ * multi-component services. Mandatory if it is a single-component
+ * service and if cpus and memory are not specified at the Service
* level.
**/
- public Application resource(Resource resource) {
+ public Service resource(Resource resource) {
this.resource = resource;
return this;
}
- @ApiModelProperty(example = "null", value = "Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.")
+ @ApiModelProperty(example = "null", value = "Resource of single-component services or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level.")
@JsonProperty("resource")
public Resource getResource() {
return resource;
}
/**
- * The custom launch command of an application component (optional). If not
- * specified for applications with docker images say, it will default to the
+ * The custom launch command of an service component (optional). If not
+ * specified for services with docker images say, it will default to the
* default start command of the image. If there is a single component in this
- * application, you can specify this without the need to have a 'components'
+ * service, you can specify this without the need to have a 'components'
* section.
**/
- public Application launchCommand(String launchCommand) {
+ public Service launchCommand(String launchCommand) {
this.launchCommand = launchCommand;
return this;
}
- @ApiModelProperty(example = "null", value = "The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.")
+ @ApiModelProperty(example = "null", value = "The custom launch command of an service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section.")
@JsonProperty("launch_command")
public String getLaunchCommand() {
return launchCommand;
}
/**
- * The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
+ * The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.
**/
- public Application launchTime(Date launchTime) {
+ public Service launchTime(Date launchTime) {
this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
return this;
}
- @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
+ @ApiModelProperty(example = "null", value = "The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.")
@JsonProperty("launch_time")
public Date getLaunchTime() {
return launchTime == null ? null : (Date) launchTime.clone();
}
/**
- * Number of containers for each app-component in the application. Each
- * app-component can further override this app-level global default.
+ * Number of containers for each component in the service. Each
+ * component can further override this service-level global default.
**/
- public Application numberOfContainers(Long numberOfContainers) {
+ public Service numberOfContainers(Long numberOfContainers) {
this.numberOfContainers = numberOfContainers;
return this;
}
- @ApiModelProperty(example = "null", value = "Number of containers for each app-component in the application. Each app-component can further override this app-level global default.")
+ @ApiModelProperty(example = "null", value = "Number of containers for each component in the service. Each component can further override this service-level global default.")
@JsonProperty("number_of_containers")
public Long getNumberOfContainers() {
return numberOfContainers;
/**
* In get response this provides the total number of running containers for
- * this application (across all components) at the time of request. Note, a
+ * this service (across all components) at the time of request. Note, a
* subsequent request can return a different number as and when more
* containers get allocated until it reaches the total number of containers or
* if a flex request has been made between the two requests.
**/
- public Application numberOfRunningContainers(Long numberOfRunningContainers) {
+ public Service numberOfRunningContainers(Long numberOfRunningContainers) {
this.numberOfRunningContainers = numberOfRunningContainers;
return this;
}
- @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
+ @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
@JsonProperty("number_of_running_containers")
public Long getNumberOfRunningContainers() {
return numberOfRunningContainers;
}
/**
- * Life time (in seconds) of the application from the time it reaches the
+ * Life time (in seconds) of the service from the time it reaches the
* RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For
* unlimited lifetime do not set a lifetime value.
**/
- public Application lifetime(Long lifetime) {
+ public Service lifetime(Long lifetime) {
this.lifetime = lifetime;
return this;
}
- @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
+ @ApiModelProperty(example = "null", value = "Life time (in seconds) of the service from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
@JsonProperty("lifetime")
public Long getLifetime() {
return lifetime;
/**
* Advanced scheduling and placement policies (optional). If not specified, it
- * defaults to the default placement policy of the app owner. The design of
+ * defaults to the default placement policy of the service owner. The design of
* placement policies are in the works. It is not very clear at this point,
- * how policies in conjunction with labels be exposed to application owners.
+ * how policies in conjunction with labels be exposed to service owners.
* This is a placeholder for now. The advanced structure of this attribute
* will be determined by YARN-4902.
**/
- public Application placementPolicy(PlacementPolicy placementPolicy) {
+ public Service placementPolicy(PlacementPolicy placementPolicy) {
this.placementPolicy = placementPolicy;
return this;
}
- @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
@JsonProperty("placement_policy")
public PlacementPolicy getPlacementPolicy() {
return placementPolicy;
}
/**
- * Components of an application.
+ * Components of an service.
**/
- public Application components(List<Component> components) {
+ public Service components(List<Component> components) {
this.components = components;
return this;
}
- @ApiModelProperty(example = "null", value = "Components of an application.")
+ @ApiModelProperty(example = "null", value = "Components of an service.")
@JsonProperty("components")
public List<Component> getComponents() {
return components;
}
/**
- * Config properties of an application. Configurations provided at the
- * application/global level are available to all the components. Specific
+ * Config properties of an service. Configurations provided at the
+ * service/global level are available to all the components. Specific
* properties can be overridden at the component level.
**/
- public Application configuration(Configuration configuration) {
+ public Service configuration(Configuration configuration) {
this.configuration = configuration;
return this;
}
- @ApiModelProperty(example = "null", value = "Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.")
+ @ApiModelProperty(example = "null", value = "Config properties of an service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level.")
@JsonProperty("configuration")
public Configuration getConfiguration() {
return configuration;
}
/**
- * Containers of a started application. Specifying a value for this attribute
+ * Containers of a started service. Specifying a value for this attribute
* for the POST payload raises a validation error. This blob is available only
- * in the GET response of a started application.
+ * in the GET response of a started service.
**/
- public Application containers(List<Container> containers) {
+ public Service containers(List<Container> containers) {
this.containers = containers;
return this;
}
- @ApiModelProperty(example = "null", value = "Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
+ @ApiModelProperty(example = "null", value = "Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
@JsonProperty("containers")
public List<Container> getContainers() {
return containers;
}
/**
- * State of the application. Specifying a value for this attribute for the
+ * State of the service. Specifying a value for this attribute for the
* POST payload raises a validation error. This attribute is available only in
- * the GET response of a started application.
+ * the GET response of a started service.
**/
- public Application state(ApplicationState state) {
+ public Service state(ServiceState state) {
this.state = state;
return this;
}
- @ApiModelProperty(example = "null", value = "State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.")
+ @ApiModelProperty(example = "null", value = "State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service.")
@JsonProperty("state")
- public ApplicationState getState() {
+ public ServiceState getState() {
return state;
}
- public void setState(ApplicationState state) {
+ public void setState(ServiceState state) {
this.state = state;
}
/**
- * A blob of key-value pairs of quicklinks to be exported for an application.
+ * A blob of key-value pairs of quicklinks to be exported for an service.
**/
- public Application quicklinks(Map<String, String> quicklinks) {
+ public Service quicklinks(Map<String, String> quicklinks) {
this.quicklinks = quicklinks;
return this;
}
- @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an application.")
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an service.")
@JsonProperty("quicklinks")
public Map<String, String> getQuicklinks() {
return quicklinks;
}
/**
- * The YARN queue that this application should be submitted to.
+ * The YARN queue that this service should be submitted to.
**/
- public Application queue(String queue) {
+ public Service queue(String queue) {
this.queue = queue;
return this;
}
- @ApiModelProperty(example = "null", value = "The YARN queue that this application should be submitted to.")
+ @ApiModelProperty(example = "null", value = "The YARN queue that this service should be submitted to.")
@JsonProperty("queue")
public String getQueue() {
return queue;
if (o == null || getClass() != o.getClass()) {
return false;
}
- Application application = (Application) o;
- return Objects.equals(this.name, application.name);
+ Service service = (Service) o;
+ return Objects.equals(this.name, service.name);
}
@Override
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("class Application {\n");
+ sb.append("class Service {\n");
sb.append(" name: ").append(toIndentedString(name)).append("\n");
sb.append(" id: ").append(toIndentedString(id)).append("\n");
import org.apache.hadoop.classification.InterfaceStability;
/**
- * The current state of an application.
+ * The current state of an service.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "The current state of an application.")
+@ApiModel(description = "The current state of an service.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public enum ApplicationState {
+public enum ServiceState {
ACCEPTED, STARTED, READY, STOPPED, FAILED;
}
import org.apache.hadoop.classification.InterfaceStability;
/**
- * The current status of a submitted application, returned as a response to the
+ * The current status of a submitted service, returned as a response to the
* GET API.
**/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-@ApiModel(description = "The current status of a submitted application, returned as a response to the GET API.")
+@ApiModel(description = "The current status of a submitted service, returned as a response to the GET API.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ApplicationStatus extends BaseResource {
+public class ServiceStatus extends BaseResource {
private static final long serialVersionUID = -3469885905347851034L;
private String diagnostics = null;
- private ApplicationState state = null;
+ private ServiceState state = null;
private Integer code = null;
/**
* Diagnostic information (if any) for the reason of the current state of the
- * application. It typically has a non-null value, if the application is in a
+ * service. It typically has a non-null value, if the service is in a
* non-running state.
**/
- public ApplicationStatus diagnostics(String diagnostics) {
+ public ServiceStatus diagnostics(String diagnostics) {
this.diagnostics = diagnostics;
return this;
}
- @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.")
+ @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state.")
@JsonProperty("diagnostics")
public String getDiagnostics() {
return diagnostics;
}
/**
- * Application state.
+ * Service state.
**/
- public ApplicationStatus state(ApplicationState state) {
+ public ServiceStatus state(ServiceState state) {
this.state = state;
return this;
}
- @ApiModelProperty(example = "null", value = "Application state.")
+ @ApiModelProperty(example = "null", value = "Service state.")
@JsonProperty("state")
- public ApplicationState getState() {
+ public ServiceState getState() {
return state;
}
- public void setState(ApplicationState state) {
+ public void setState(ServiceState state) {
this.state = state;
}
/**
- * An error code specific to a scenario which app owners should be able to use
+ * An error code specific to a scenario which service owners should be able to use
* to understand the failure in addition to the diagnostic information.
**/
- public ApplicationStatus code(Integer code) {
+ public ServiceStatus code(Integer code) {
this.code = code;
return this;
}
- @ApiModelProperty(example = "null", value = "An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.")
+ @ApiModelProperty(example = "null", value = "An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.")
@JsonProperty("code")
public Integer getCode() {
return code;
if (o == null || getClass() != o.getClass()) {
return false;
}
- ApplicationStatus applicationStatus = (ApplicationStatus) o;
- return Objects.equals(this.diagnostics, applicationStatus.diagnostics)
- && Objects.equals(this.state, applicationStatus.state)
- && Objects.equals(this.code, applicationStatus.code);
+ ServiceStatus serviceStatus = (ServiceStatus) o;
+ return Objects.equals(this.diagnostics, serviceStatus.diagnostics)
+ && Objects.equals(this.state, serviceStatus.state)
+ && Objects.equals(this.code, serviceStatus.code);
}
@Override
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("class ApplicationStatus {\n");
+ sb.append("class ServiceStatus {\n");
sb.append(" diagnostics: ").append(toIndentedString(diagnostics))
.append("\n");
import java.net.InetSocketAddress;
+import static org.apache.hadoop.io.retry.RetryPolicies.TRY_ONCE_THEN_FAIL;
+
public class ClientAMProxy extends ServerProxy{
public static <T> T createProxy(final Configuration conf,
final Class<T> protocol, final UserGroupInformation ugi,
final YarnRPC rpc, final InetSocketAddress serverAddress) {
-
- RetryPolicy retryPolicy =
- createRetryPolicy(conf, YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS,
- 15 * 60 * 1000, YarnServiceConf.CLIENT_AM_RETRY_MAX_INTERVAL_MS,
- 2 * 1000);
Configuration confClone = new Configuration(conf);
confClone.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
confClone.setInt(CommonConfigurationKeysPublic.
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
+ RetryPolicy retryPolicy;
+
+ if (conf.getLong(YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS, 0) == 0) {
+ // by default no retry
+ retryPolicy = TRY_ONCE_THEN_FAIL;
+ } else {
+ retryPolicy =
+ createRetryPolicy(conf, YarnServiceConf.CLIENT_AM_RETRY_MAX_WAIT_MS,
+ 15 * 60 * 1000, YarnServiceConf.CLIENT_AM_RETRY_MAX_INTERVAL_MS,
+ 2 * 1000);
+ }
return createRetriableProxy(confClone, protocol, ugi, rpc, serverAddress,
retryPolicy);
}
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileNotFoundException;
+
import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
public class ServiceCLI {
client.actionCreate(args.getActionCreateArgs());
break;
case ACTION_STATUS:
- Application app = client.getStatus(args.getClusterName());
+ Service app = client.getStatus(args.getClusterName());
System.out.println(app);
break;
case ACTION_FLEX:
- client.actionFlexByCLI(args);
+ try {
+ client.actionFlexByCLI(args);
+ } catch (FileNotFoundException e) {
+ System.err.println(
+ args.getClusterName() + " doesn't exist: " + e.getMessage());
+ return -1;
+ }
break;
case ACTION_STOP:
client.actionStop(args.getClusterName(), false);
import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.ServiceMaster;
-import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
@InterfaceAudience.Public
private static final Logger LOG =
LoggerFactory.getLogger(ServiceClient.class);
private SliderFileSystem fs;
- private YarnClient yarnClient;
+ //TODO disable retry so that client / rest API doesn't block?
+ protected YarnClient yarnClient;
// Avoid looking up applicationId from fs all the time.
private Map<String, ApplicationId> cachedAppIds = new ConcurrentHashMap<>();
- private Map<String, ClientAMProtocol> cachedAMProxies = new ConcurrentHashMap<>();
private RegistryOperations registryClient;
private CuratorFramework curatorClient;
private static EnumSet<YarnApplicationState> terminatedStates =
EnumSet.of(FINISHED, FAILED, KILLED);
private static EnumSet<YarnApplicationState> liveStates =
- EnumSet.of(NEW, NEW_SAVING, SUBMITTED, RUNNING);
+ EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING);
+ private static EnumSet<YarnApplicationState> preRunningStates =
+ EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED);
public ServiceClient() {
super(ServiceClient.class.getName());
super.serviceStop();
}
- private Application loadAppJsonFromLocalFS(
+ private Service loadAppJsonFromLocalFS(
AbstractClusterBuildingActionArgs args) throws IOException {
File file = args.getAppDef();
Path filePath = new Path(file.getAbsolutePath());
LOG.info("Loading app json from: " + filePath);
- Application application = ServiceApiUtil.jsonSerDeser
+ Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
if (args.lifetime > 0) {
- application.setLifetime(args.lifetime);
+ service.setLifetime(args.lifetime);
}
- application.setName(args.getClusterName());
- return application;
+ service.setName(args.getClusterName());
+ return service;
}
public int actionBuild(AbstractClusterBuildingActionArgs args)
return actionBuild(loadAppJsonFromLocalFS(args));
}
- public int actionBuild(Application application)
+ public int actionBuild(Service service)
throws YarnException, IOException {
- Path appDir = checkAppNotExistOnHdfs(application);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
- createDirAndPersistApp(appDir, application);
+ Path appDir = checkAppNotExistOnHdfs(service);
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ createDirAndPersistApp(appDir, service);
return EXIT_SUCCESS;
}
return EXIT_SUCCESS;
}
- public ApplicationId actionCreate(Application application)
+ public ApplicationId actionCreate(Service service)
throws IOException, YarnException {
- String appName = application.getName();
- validateClusterName(appName);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
- verifyNoLiveAppInRM(appName, "create");
- Path appDir = checkAppNotExistOnHdfs(application);
+ String serviceName = service.getName();
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ verifyNoLiveAppInRM(serviceName, "create");
+ Path appDir = checkAppNotExistOnHdfs(service);
// Write the definition first and then submit - AM will read the definition
- createDirAndPersistApp(appDir, application);
- ApplicationId appId = submitApp(application);
- cachedAppIds.put(appName, appId);
- application.setId(appId.toString());
+ createDirAndPersistApp(appDir, service);
+ ApplicationId appId = submitApp(service);
+ cachedAppIds.put(serviceName, appId);
+ service.setId(appId.toString());
// update app definition with appId
- persistAppDef(appDir, application);
+ persistAppDef(appDir, service);
return appId;
}
ActionFlexArgs flexArgs = args.getActionFlexArgs();
Map<String, Long> componentCounts =
new HashMap<>(flexArgs.getComponentMap().size());
- Application persistedApp =
- ServiceApiUtil.loadApplication(fs, flexArgs.getClusterName());
- if (!StringUtils.isEmpty(persistedApp.getId())) {
- cachedAppIds.put(persistedApp.getName(),
- ApplicationId.fromString(persistedApp.getId()));
+ Service persistedService =
+ ServiceApiUtil.loadService(fs, flexArgs.getClusterName());
+ if (!StringUtils.isEmpty(persistedService.getId())) {
+ cachedAppIds.put(persistedService.getName(),
+ ApplicationId.fromString(persistedService.getId()));
+ } else {
+ throw new YarnException(persistedService.getName()
+ + " appId is null, may be not submitted to YARN yet");
}
+
for (Map.Entry<String, String> entry : flexArgs.getComponentMap()
.entrySet()) {
String compName = entry.getKey();
- ServiceApiUtil.validateCompName(compName);
- Component component = persistedApp.getComponent(compName);
+ ServiceApiUtil.validateNameFormat(compName, getConfig());
+ Component component = persistedService.getComponent(compName);
if (component == null) {
throw new IllegalArgumentException(entry.getKey() + " does not exist !");
}
if (componentCounts.size() == 0) {
actionHelp(ACTION_FLEX, args);
}
- flexComponents(args.getClusterName(), componentCounts, persistedApp);
+ flexComponents(args.getClusterName(), componentCounts, persistedService);
return EXIT_SUCCESS;
}
}
// Called by Rest Service
- public Map<String, Long> flexByRestService(String appName,
+ public Map<String, Long> flexByRestService(String serviceName,
Map<String, Long> componentCounts) throws YarnException, IOException {
// load app definition
- Application persistedApp = ServiceApiUtil.loadApplication(fs, appName);
- cachedAppIds.put(persistedApp.getName(),
- ApplicationId.fromString(persistedApp.getId()));
- return flexComponents(appName, componentCounts, persistedApp);
+ Service persistedService = ServiceApiUtil.loadService(fs, serviceName);
+ if (StringUtils.isEmpty(persistedService.getId())) {
+ throw new YarnException(
+ serviceName + " appId is null, may be not submitted to YARN yet");
+ }
+ cachedAppIds.put(persistedService.getName(),
+ ApplicationId.fromString(persistedService.getId()));
+ return flexComponents(serviceName, componentCounts, persistedService);
}
- private Map<String, Long> flexComponents(String appName,
- Map<String, Long> componentCounts, Application persistedApp)
+ private Map<String, Long> flexComponents(String serviceName,
+ Map<String, Long> componentCounts, Service persistedService)
throws YarnException, IOException {
- validateClusterName(appName);
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
Map<String, Long> original = new HashMap<>(componentCounts.size());
FlexComponentsRequestProto.Builder requestBuilder =
FlexComponentsRequestProto.newBuilder();
- for (Component persistedComp : persistedApp.getComponents()) {
+ for (Component persistedComp : persistedService.getComponents()) {
String name = persistedComp.getName();
if (componentCounts.containsKey(persistedComp.getName())) {
original.put(name, persistedComp.getNumberOfContainers());
throw new YarnException("Components " + componentCounts.keySet()
+ " do not exist in app definition.");
}
- ServiceApiUtil.jsonSerDeser
- .save(fs.getFileSystem(), ServiceApiUtil.getAppJsonPath(fs, appName),
- persistedApp, true);
- ClientAMProtocol proxy = getAMProxy(appName);
- if (proxy == null) {
- String message = appName + " is not running";
+ jsonSerDeser
+ .save(fs.getFileSystem(), ServiceApiUtil.getServiceJsonPath(fs, serviceName),
+ persistedService, true);
+
+ ApplicationReport appReport =
+ yarnClient.getApplicationReport(getAppId(serviceName));
+ if (appReport.getYarnApplicationState() != RUNNING) {
+ String message =
+ serviceName + " is at " + appReport.getYarnApplicationState()
+ + " state, flex can only be invoked when service is running";
LOG.error(message);
throw new YarnException(message);
}
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ throw new YarnException(serviceName + " AM hostname is empty");
+ }
+ ClientAMProtocol proxy =
+ createAMProxy(appReport.getHost(), appReport.getRpcPort());
proxy.flexComponents(requestBuilder.build());
for (Map.Entry<String, Long> entry : original.entrySet()) {
LOG.info("[COMPONENT {}]: number of containers changed from {} to {}",
return original;
}
- public int actionStop(String appName, boolean waitForAppStopped)
+ public int actionStop(String serviceName, boolean waitForAppStopped)
throws YarnException, IOException {
- validateClusterName(appName);
- getAppId(appName);
- ApplicationId currentAppId = cachedAppIds.get(appName);
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ApplicationId currentAppId = getAppId(serviceName);
ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
if (terminatedStates.contains(report.getYarnApplicationState())) {
- LOG.info("Application {} is already in a terminated state {}", appName,
+ LOG.info("Service {} is already in a terminated state {}", serviceName,
report.getYarnApplicationState());
return EXIT_SUCCESS;
}
- LOG.info("Stopping application {}, with appId = {}", appName, currentAppId);
+ if (preRunningStates.contains(report.getYarnApplicationState())) {
+ String msg = serviceName + " is at " + report.getYarnApplicationState()
+ + ", forcefully killed by user!";
+ yarnClient.killApplication(currentAppId, msg);
+ LOG.info(msg);
+ return EXIT_SUCCESS;
+ }
+ if (StringUtils.isEmpty(report.getHost())) {
+ throw new YarnException(serviceName + " AM hostname is empty");
+ }
+ LOG.info("Stopping service {}, with appId = {}", serviceName, currentAppId);
try {
- ClientAMProtocol proxy = getAMProxy(appName, report);
- cachedAppIds.remove(appName);
- cachedAMProxies.remove(appName);
+ ClientAMProtocol proxy =
+ createAMProxy(report.getHost(), report.getRpcPort());
+ cachedAppIds.remove(serviceName);
if (proxy != null) {
// try to stop the app gracefully.
StopRequestProto request = StopRequestProto.newBuilder().build();
proxy.stop(request);
- LOG.info("Application " + appName + " is being gracefully stopped...");
+ LOG.info("Service " + serviceName + " is being gracefully stopped...");
} else {
yarnClient.killApplication(currentAppId,
- appName + " is forcefully killed by user!");
- LOG.info("Forcefully kill the application: " + appName);
+ serviceName + " is forcefully killed by user!");
+ LOG.info("Forcefully kill the service: " + serviceName);
return EXIT_SUCCESS;
}
Thread.sleep(2000);
report = yarnClient.getApplicationReport(currentAppId);
if (terminatedStates.contains(report.getYarnApplicationState())) {
- LOG.info("Application " + appName + " is stopped.");
+ LOG.info("Service " + serviceName + " is stopped.");
break;
}
// Forcefully kill after 10 seconds.
if ((System.currentTimeMillis() - startTime) > 10000) {
LOG.info("Stop operation timeout stopping, forcefully kill the app "
- + appName);
+ + serviceName);
yarnClient.killApplication(currentAppId,
"Forcefully kill the app by user");
break;
}
if (++pollCount % 10 == 0) {
- LOG.info("Waiting for application " + appName + " to be stopped.");
+ LOG.info("Waiting for service " + serviceName + " to be stopped.");
}
}
} catch (IOException | YarnException | InterruptedException e) {
- LOG.info("Failed to stop " + appName
+ LOG.info("Failed to stop " + serviceName
+ " gracefully, forcefully kill the app.");
yarnClient.killApplication(currentAppId, "Forcefully kill the app");
}
return EXIT_SUCCESS;
}
- public int actionDestroy(String appName) throws Exception {
- validateClusterName(appName);
- verifyNoLiveAppInRM(appName, "Destroy");
- Path appDir = fs.buildClusterDirPath(appName);
+ public int actionDestroy(String serviceName) throws Exception {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ verifyNoLiveAppInRM(serviceName, "Destroy");
+ Path appDir = fs.buildClusterDirPath(serviceName);
FileSystem fileSystem = fs.getFileSystem();
// remove from the appId cache
- cachedAppIds.remove(appName);
- cachedAMProxies.remove(appName);
+ cachedAppIds.remove(serviceName);
if (fileSystem.exists(appDir)) {
if (fileSystem.delete(appDir, true)) {
- LOG.info("Successfully deleted application dir for " + appName + ": "
+ LOG.info("Successfully deleted service dir for " + serviceName + ": "
+ appDir);
} else {
String message =
- "Failed to delete application + " + appName + " at: " + appDir;
+ "Failed to delete service + " + serviceName + " at: " + appDir;
LOG.info(message);
throw new YarnException(message);
}
}
- deleteZKNode(appName);
- String registryPath = ServiceRegistryUtils.registryPathForInstance(appName);
+ deleteZKNode(serviceName);
+ String registryPath = ServiceRegistryUtils.registryPathForInstance(serviceName);
try {
getRegistryClient().delete(registryPath, true);
} catch (IOException e) {
LOG.warn("Error deleting registry entry {}", registryPath, e);
}
- LOG.info("Destroyed cluster {}", appName);
+ LOG.info("Destroyed cluster {}", serviceName);
return EXIT_SUCCESS;
}
throw new UsageException(CommonArgs.usage(args, actionName));
}
- private void verifyNoLiveAppInRM(String appname, String action)
+ private void verifyNoLiveAppInRM(String serviceName, String action)
throws IOException, YarnException {
Set<String> types = new HashSet<>(1);
types.add(YarnServiceConstants.APP_TYPE);
Set<String> tags = null;
- if (appname != null) {
- tags = Collections.singleton(SliderUtils.createNameTag(appname));
+ if (serviceName != null) {
+ tags = Collections.singleton(SliderUtils.createNameTag(serviceName));
}
GetApplicationsRequest request = GetApplicationsRequest.newInstance();
request.setApplicationTypes(types);
List<ApplicationReport> reports = yarnClient.getApplications(request);
if (!reports.isEmpty()) {
throw new YarnException(
- "Failed to " + action + " application, as " + appname
+ "Failed to " + action + " service, as " + serviceName
+ " already exists.");
}
}
- private ApplicationId submitApp(Application app)
+ private ApplicationId submitApp(Service app)
throws IOException, YarnException {
- String appName = app.getName();
+ String serviceName = app.getName();
Configuration conf = getConfig();
Path appRootDir = fs.buildClusterDirPath(app.getName());
// copy local slideram-log4j.properties to hdfs and add to localResources
boolean hasAMLog4j =
- addAMLog4jResource(appName, conf, localResources);
+ addAMLog4jResource(serviceName, conf, localResources);
// copy jars to hdfs and add to localResources
- addJarResource(appName, localResources);
+ addJarResource(serviceName, localResources);
// add keytab if in secure env
- addKeytabResourceIfSecure(fs, localResources, conf, appName);
+ addKeytabResourceIfSecure(fs, localResources, conf, serviceName);
if (LOG.isDebugEnabled()) {
printLocalResources(localResources);
}
// create AM CLI
String cmdStr =
- buildCommandLine(appName, conf, appRootDir, hasAMLog4j);
+ buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j);
submissionContext.setResource(Resource.newInstance(YarnServiceConf
.getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
app.getConfiguration(), conf), 1));
queue = conf.get(YARN_QUEUE, "default");
}
submissionContext.setQueue(queue);
- submissionContext.setApplicationName(appName);
+ submissionContext.setApplicationName(serviceName);
submissionContext.setApplicationType(YarnServiceConstants.APP_TYPE);
Set<String> appTags =
- AbstractClientProvider.createApplicationTags(appName, null, null);
+ AbstractClientProvider.createApplicationTags(serviceName, null, null);
if (!appTags.isEmpty()) {
submissionContext.setApplicationTags(appTags);
}
LOG.debug(builder.toString());
}
- private String buildCommandLine(String appName, Configuration conf,
+ private String buildCommandLine(String serviceName, Configuration conf,
Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
CLI.forceIPv4().headless();
CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
}
CLI.add(ServiceMaster.class.getCanonicalName());
- CLI.add(ACTION_CREATE, appName);
+ CLI.add(ACTION_CREATE, serviceName);
//TODO debugAM CLI.add(Arguments.ARG_DEBUG)
- CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, appName + ".json"));
+ CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, serviceName + ".json"));
// pass the registry binding
CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
return env;
}
- protected Path addJarResource(String appName,
+ protected Path addJarResource(String serviceName,
Map<String, LocalResource> localResources)
throws IOException, SliderException {
- Path libPath = fs.buildClusterDirPath(appName);
+ Path libPath = fs.buildClusterDirPath(serviceName);
ProviderUtils
.addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
libPath, "lib", false);
return libPath;
}
- private boolean addAMLog4jResource(String appName, Configuration conf,
+ private boolean addAMLog4jResource(String serviceName, Configuration conf,
Map<String, LocalResource> localResources)
throws IOException, BadClusterStateException {
boolean hasAMLog4j = false;
new File(hadoopConfDir, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
if (localFile.exists()) {
Path localFilePath = createLocalPath(localFile);
- Path appDirPath = fs.buildClusterDirPath(appName);
+ Path appDirPath = fs.buildClusterDirPath(serviceName);
Path remoteConfPath =
new Path(appDirPath, YarnServiceConstants.SUBMITTED_CONF_DIR);
Path remoteFilePath =
return hasAMLog4j;
}
- public int actionStart(String appName) throws YarnException, IOException {
- validateClusterName(appName);
- Path appDir = checkAppExistOnHdfs(appName);
- Application application = ServiceApiUtil.loadApplication(fs, appName);
- ServiceApiUtil.validateAndResolveApplication(application, fs, getConfig());
+ public int actionStart(String serviceName) throws YarnException, IOException {
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ Path appDir = checkAppExistOnHdfs(serviceName);
+ Service service = ServiceApiUtil.loadService(fs, serviceName);
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
// see if it is actually running and bail out;
- verifyNoLiveAppInRM(appName, "thaw");
- ApplicationId appId = submitApp(application);
- application.setId(appId.toString());
+ verifyNoLiveAppInRM(serviceName, "thaw");
+ ApplicationId appId = submitApp(service);
+ service.setId(appId.toString());
// write app definition on to hdfs
- createDirAndPersistApp(appDir, application);
+ createDirAndPersistApp(appDir, service);
return 0;
}
- private Path checkAppNotExistOnHdfs(Application application)
+ private Path checkAppNotExistOnHdfs(Service service)
throws IOException, SliderException {
- Path appDir = fs.buildClusterDirPath(application.getName());
+ Path appDir = fs.buildClusterDirPath(service.getName());
fs.verifyDirectoryNonexistent(
- new Path(appDir, application.getName() + ".json"));
+ new Path(appDir, service.getName() + ".json"));
return appDir;
}
- private Path checkAppExistOnHdfs(String appName)
+ private Path checkAppExistOnHdfs(String serviceName)
throws IOException, SliderException {
- Path appDir = fs.buildClusterDirPath(appName);
- fs.verifyPathExists(new Path(appDir, appName + ".json"));
+ Path appDir = fs.buildClusterDirPath(serviceName);
+ fs.verifyPathExists(new Path(appDir, serviceName + ".json"));
return appDir;
}
- private void createDirAndPersistApp(Path appDir, Application application)
+ private void createDirAndPersistApp(Path appDir, Service service)
throws IOException, SliderException {
FsPermission appDirPermission = new FsPermission("750");
fs.createWithPermissions(appDir, appDirPermission);
- persistAppDef(appDir, application);
+ persistAppDef(appDir, service);
}
- private void persistAppDef(Path appDir, Application application)
+ private void persistAppDef(Path appDir, Service service)
throws IOException {
- Path appJson = new Path(appDir, application.getName() + ".json");
- ServiceApiUtil.jsonSerDeser
- .save(fs.getFileSystem(), appJson, application, true);
+ Path appJson = new Path(appDir, service.getName() + ".json");
+ jsonSerDeser
+ .save(fs.getFileSystem(), appJson, service, true);
LOG.info(
- "Persisted application " + application.getName() + " at " + appJson);
+ "Persisted service " + service.getName() + " at " + appJson);
}
private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
Map<String, LocalResource> localResource, Configuration conf,
- String appName) throws IOException, BadConfigException {
+ String serviceName) throws IOException, BadConfigException {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
}
conf.get(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
String keytabDir = conf.get(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
Path keytabPath =
- fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName);
+ fileSystem.buildKeytabPath(keytabDir, amKeytabName, serviceName);
if (fileSystem.getFileSystem().exists(keytabPath)) {
LocalResource keytabRes =
fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);
} else {
LOG.warn("The AM will be "
+ "started without a kerberos authenticated identity. "
- + "The application is therefore not guaranteed to remain "
+ + "The service is therefore not guaranteed to remain "
+ "operational beyond 24 hours.");
}
}
}
}
- public String updateLifetime(String appName, long lifetime)
+ public String updateLifetime(String serviceName, long lifetime)
throws YarnException, IOException {
- getAppId(appName);
- ApplicationId currentAppId = cachedAppIds.get(appName);
+ ApplicationId currentAppId = getAppId(serviceName);
ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
if (report == null) {
- throw new YarnException("Application not found for " + appName);
+ throw new YarnException("Service not found for " + serviceName);
}
ApplicationId appId = report.getApplicationId();
- LOG.info("Updating lifetime of an application: appName = " + appName
+ LOG.info("Updating lifetime of an service: serviceName = " + serviceName
+ ", appId = " + appId + ", lifetime = " + lifetime);
Map<ApplicationTimeoutType, String> map = new HashMap<>();
String newTimeout =
UpdateApplicationTimeoutsRequest.newInstance(appId, map);
yarnClient.updateApplicationTimeouts(request);
LOG.info(
- "Successfully updated lifetime for an application: appName = " + appName
+ "Successfully updated lifetime for an service: serviceName = " + serviceName
+ ", appId = " + appId + ". New expiry time in ISO8601 format is "
+ newTimeout);
return newTimeout;
}
- public Application getStatus(String appName)
+ public ServiceState convertState(FinalApplicationStatus status) {
+ switch (status) {
+ case UNDEFINED:
+ return ServiceState.ACCEPTED;
+ case FAILED:
+ case KILLED:
+ return ServiceState.FAILED;
+ case ENDED:
+ case SUCCEEDED:
+ return ServiceState.STOPPED;
+ }
+ return ServiceState.ACCEPTED;
+ }
+
+ public Service getStatus(String serviceName)
throws IOException, YarnException {
- validateClusterName(appName);
- ApplicationId currentAppId = getAppId(appName);
+ ServiceApiUtil.validateNameFormat(serviceName, getConfig());
+ ApplicationId currentAppId = getAppId(serviceName);
ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
- ClientAMProtocol amProxy = getAMProxy(appName, appReport);
- Application appSpec;
- if (amProxy != null) {
- GetStatusResponseProto response =
- amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
- appSpec = ServiceApiUtil.jsonSerDeser.fromJson(response.getStatus());
- } else {
- appSpec = new Application();
- appSpec.setName(appName);
- }
+ Service appSpec = new Service();
+ appSpec.setName(serviceName);
+ appSpec.setState(convertState(appReport.getFinalApplicationStatus()));
ApplicationTimeout lifetime =
appReport.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
if (lifetime != null) {
appSpec.setLifetime(lifetime.getRemainingTime());
}
+
+ if (appReport.getYarnApplicationState() != RUNNING) {
+ LOG.info("Service {} is at {} state", serviceName,
+ appReport.getYarnApplicationState());
+ return appSpec;
+ }
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ LOG.warn(serviceName + " AM hostname is empty");
+ return appSpec;
+ }
+ ClientAMProtocol amProxy =
+ createAMProxy(appReport.getHost(), appReport.getRpcPort());
+ GetStatusResponseProto response =
+ amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
+ appSpec = jsonSerDeser.fromJson(response.getStatus());
+
return appSpec;
}
}
}
- // Get AMProxy with the appReport provided
- protected ClientAMProtocol getAMProxy(String appName, ApplicationReport report)
- throws IOException {
- if (!cachedAMProxies.containsKey(appName) && !StringUtils
- .isEmpty(report.getHost())) {
- insertAMProxy(appName, report.getHost(), report.getRpcPort());
- }
- return cachedAMProxies.get(appName);
- }
-
- // Get AMProxy without appReport provided - it'll getAppReport from RM
- protected ClientAMProtocol getAMProxy(String appName)
- throws IOException, YarnException {
- ApplicationId currentAppId = getAppId(appName);
-
- if (cachedAMProxies.containsKey(appName)) {
- return cachedAMProxies.get(appName);
- } else {
- ApplicationReport appReport =
- yarnClient.getApplicationReport(currentAppId);
- String host = appReport.getHost();
- int port = appReport.getRpcPort();
- if (!StringUtils.isEmpty(host)) {
- return insertAMProxy(appName, host, port);
- }
- return null;
- }
- }
-
- private ClientAMProtocol insertAMProxy(String appName, String host, int port)
+ protected ClientAMProtocol createAMProxy(String host, int port)
throws IOException {
InetSocketAddress address =
NetUtils.createSocketAddrForHost(host, port);
- ClientAMProtocol amProxy =
- ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
+ return ClientAMProxy.createProxy(getConfig(), ClientAMProtocol.class,
UserGroupInformation.getCurrentUser(), rpc, address);
- cachedAMProxies.put(appName, amProxy);
- return amProxy;
}
- private synchronized ApplicationId getAppId(String appName)
+ private synchronized ApplicationId getAppId(String serviceName)
throws IOException, YarnException {
- if (cachedAppIds.containsKey(appName)) {
- return cachedAppIds.get(appName);
+ if (cachedAppIds.containsKey(serviceName)) {
+ return cachedAppIds.get(serviceName);
}
- Application persistedApp = ServiceApiUtil.loadApplication(fs, appName);
- if (persistedApp == null) {
- throw new YarnException("Application " + appName
+ Service persistedService = ServiceApiUtil.loadService(fs, serviceName);
+ if (persistedService == null) {
+ throw new YarnException("Service " + serviceName
+ " doesn't exist on hdfs. Please check if the app exists in RM");
}
- ApplicationId currentAppId = ApplicationId.fromString(persistedApp.getId());
- cachedAppIds.put(appName, currentAppId);
+ ApplicationId currentAppId = ApplicationId.fromString(persistedService.getId());
+ cachedAppIds.put(serviceName, currentAppId);
return currentAppId;
}
}
public abstract class AbstractClusterBuildingActionArgs
extends AbstractActionArgs {
@Parameter(names = {ARG_APPDEF},
- description = "Template application definition file in JSON format.")
+ description = "Template service definition file in JSON format.")
public File appDef;
public File getAppDef() {
}
@Parameter(names = {
- ARG_QUEUE }, description = "Queue to submit the application")
+ ARG_QUEUE }, description = "Queue to submit the service")
public String queue;
@Parameter(names = {
- ARG_LIFETIME }, description = "Lifetime of the application from the time of request")
+ ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
public long lifetime;
@ParametersDelegate
public boolean install;
@Parameter(names = {ARG_NAME},
- description = "The name of the application")
+ description = "The name of the service")
public String name;
@Parameter(names = {ARG_PACKAGE},
}
@Parameter(names = {ARG_LIVE},
- description = "verify that the application is running")
+ description = "verify that the service is running")
public boolean live;
@Parameter(names = {ARG_STATE},
- description = "verify that the application is in the specific YARN state")
+ description = "verify that the service is in the specific YARN state")
public String state = "";
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any application report")
+ description = "output file for any service report")
public File out;
}
}
@Parameter(names = {ARG_LIVE},
- description = "List only live application instances")
+ description = "List only live service instances")
public boolean live;
@Parameter(names = {ARG_STATE},
public boolean verbose = false;
@Parameter(names = {ARG_CONTAINERS},
- description = "List containers of an application instance")
+ description = "List containers of a service instance")
public boolean containers;
@Parameter(names = {ARG_VERSION},
public boolean internal;
@Parameter(names = {ARG_USER},
- description = "the name of the user whose application is being resolved")
+ description = "the name of the user whose service is being resolved")
public String user;
/**
public String output;
@Parameter(names = {ARG_LIFETIME},
- description = "Lifetime of the application from the time of request")
+ description = "Lifetime of the service from the time of request")
public boolean lifetime;
public String getOutput() {
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Parameter(names = {ARG_LIFETIME},
- description = "Life time of the application since application started at"
+ description = "Life time of the service since service started at"
+ " running state")
public long lifetime;
projects / hadoop.git / commitdiff
