AMATERASU-24 - Refactor Spark out of Amaterasu executor to it's own project
authorArun Manivannan <arun@arunma.com>
Sun, 24 Jun 2018 09:24:21 +0000 (17:24 +0800)
committerArun Manivannan <arun@arunma.com>
Sun, 24 Jun 2018 09:24:21 +0000 (17:24 +0800)
17 files changed:
common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala
gradle/wrapper/gradle-wrapper.properties
gradlew
gradlew.bat
leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java
leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
leader/src/main/scala/org/apache/amaterasu/leader/dsl/JobParser.scala
leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/AmaterasuScheduler.scala
leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
leader/src/main/scala/org/apache/amaterasu/leader/utilities/ActiveReportListener.scala
leader/src/main/scala/org/apache/amaterasu/leader/utilities/HttpServer.scala
leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
leader/src/main/scripts/amaterasu.properties
leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala
leader/src/test/scala/org/apache/amaterasu/utilities/HttpServerTests.scala

index 8a44019..fe69260 100755 (executable)
@@ -16,9 +16,9 @@
  */
 package org.apache.amaterasu.common.execution.actions
 
-import NotificationLevel.NotificationLevel
-import NotificationType.NotificationType
 import com.fasterxml.jackson.annotation.JsonProperty
+import org.apache.amaterasu.common.execution.actions.NotificationLevel.NotificationLevel
+import org.apache.amaterasu.common.execution.actions.NotificationType.NotificationType
 
 abstract class Notifier {
 
index 4b125b8..85f3050 100644 (file)
@@ -1,6 +1,6 @@
-#Fri Jan 27 12:21:51 AEDT 2017
+#Sun Jun 24 01:04:07 SGT 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip
diff --git a/gradlew b/gradlew
index 3efb0e9..9aa616c 100755 (executable)
--- a/gradlew
+++ b/gradlew
@@ -1,20 +1,4 @@
 #!/usr/bin/env bash
-#
-#    Licensed to the Apache Software Foundation (ASF) under one or more
-#    contributor license agreements.  See the NOTICE file distributed with
-#    this work for additional information regarding copyright ownership.
-#    The ASF licenses this file to You under the Apache License, Version 2.0
-#    (the "License"); you may not use this file except in compliance with
-#    the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 
 ##############################################################################
 ##
index 718266c..f955316 100644 (file)
@@ -1,21 +1,3 @@
-rem
-rem
-rem    Licensed to the Apache Software Foundation (ASF) under one or more
-rem    contributor license agreements.  See the NOTICE file distributed with
-rem    this work for additional information regarding copyright ownership.
-rem    The ASF licenses this file to You under the Apache License, Version 2.0
-rem    (the "License"); you may not use this file except in compliance with
-rem    the License.  You may obtain a copy of the License at
-rem
-rem       http://www.apache.org/licenses/LICENSE-2.0
-rem
-rem    Unless required by applicable law or agreed to in writing, software
-rem    distributed under the License is distributed on an "AS IS" BASIS,
-rem    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-rem    See the License for the specific language governing permissions and
-rem    limitations under the License.
-rem
-
 @if "%DEBUG%" == "" @echo off
 @rem ##########################################################################
 @rem
index be0fc05..38a9c38 100644 (file)
  */
 package org.apache.amaterasu.leader.yarn;
 
-import org.apache.commons.cli.*;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
 
 public class ArgsParser {
     private static Options getOptions() {
index e3c2812..8f16ee7 100644 (file)
@@ -31,7 +31,16 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -43,11 +52,19 @@ import org.apache.log4j.LogManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.jms.*;
+import javax.jms.Connection;
+import javax.jms.JMSException;
+import javax.jms.MessageConsumer;
+import javax.jms.Session;
+import javax.jms.Topic;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 
 import static java.lang.System.exit;
 
index 8ef1c7a..aba6210 100755 (executable)
@@ -18,12 +18,12 @@ package org.apache.amaterasu.leader.dsl
 
 import java.util.concurrent.BlockingQueue
 
-import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import com.fasterxml.jackson.databind.node.ArrayNode
+import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import org.apache.amaterasu.common.dataobjects.ActionData
-import org.apache.amaterasu.leader.execution.actions.{Action, ErrorAction, SequentialAction}
 import org.apache.amaterasu.leader.execution.JobManager
+import org.apache.amaterasu.leader.execution.actions.{Action, ErrorAction, SequentialAction}
 import org.apache.curator.framework.CuratorFramework
 
 import scala.collection.JavaConverters._
index 8c487c1..f6dea22 100644 (file)
@@ -19,7 +19,6 @@ package org.apache.amaterasu.leader.frameworks.spark
 import java.io.File
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
-import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.leader.utilities.{DataLoader, MemoryFormatParser}
 import org.apache.amaterasu.sdk.frameworks.FrameworkSetupProvider
 import org.apache.amaterasu.sdk.frameworks.configuration.DriverConfiguration
index f2f2c00..4b1a74c 100755 (executable)
@@ -17,7 +17,6 @@
 package org.apache.amaterasu.leader.mesos.schedulers
 
 import org.apache.amaterasu.common.logging.Logging
-
 import org.apache.mesos.Protos.{Resource, Value}
 import org.apache.mesos.Scheduler
 
index 87a8f5d..2c2e8af 100755 (executable)
  */
 package org.apache.amaterasu.leader.mesos.schedulers
 
-import java.io.File
 import java.util
 import java.util.concurrent.locks.ReentrantLock
 import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue}
-import java.util.{Collections, Properties, UUID}
+import java.util.{Collections, UUID}
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
@@ -185,7 +184,7 @@ class JobScheduler extends AmaterasuScheduler {
                     .setExtract(true)
                     .build())
                   .addUris(URI.newBuilder()
-                    .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/Miniconda2-latest-Linux-x86_64.sh")
+                    .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/miniconda.sh")
                     .setExecutable(false)
                     .setExtract(false)
                     .build())
index 2664665..b3ffaad 100644 (file)
@@ -17,9 +17,7 @@
 package org.apache.amaterasu.leader.utilities
 
 import javax.jms.{Message, MessageListener, TextMessage}
-
 import net.liftweb.json._
-import net.liftweb.json.JsonDSL._
 import org.apache.amaterasu.common.execution.actions.{Notification, NotificationLevel, NotificationType}
 
 class ActiveReportListener extends MessageListener {
index 2e01963..5c48329 100644 (file)
@@ -18,19 +18,14 @@ package org.apache.amaterasu.leader.utilities
 
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.log4j.{BasicConfigurator, Level, Logger}
-import org.eclipse.jetty.server.{Handler, Server, ServerConnector}
 import org.eclipse.jetty.server.handler._
-import org.eclipse.jetty.servlet.{DefaultServlet, ServletContextHandler, ServletHolder}
-import org.eclipse.jetty.toolchain.test.MavenTestingUtils
-import org.eclipse.jetty.util.thread.QueuedThreadPool
+import org.eclipse.jetty.server.{Server, ServerConnector}
 import org.eclipse.jetty.util.log.StdErrLog
-import org.eclipse.jetty.util.resource.Resource
 import org.jsoup.Jsoup
 import org.jsoup.select.Elements
 
 import scala.collection.JavaConverters._
 import scala.io.{BufferedSource, Source}
-import scala.text.Document
 
 /**
   * Created by kirupa
index d3bcb7d..406c150 100644 (file)
@@ -253,7 +253,7 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging {
           s"java -cp spark/jars/*:executor.jar:spark-runner.jar:spark-runtime.jar:spark/conf/:${config.YARN.hadoopHomeDir}/conf/ " +
             "-Xmx1G " +
             "-Dscala.usejavacp=true " +
-            "-Dhdp.version=2.6.5.0-292 " +
+            "-Dhdp.version=2.6.1.0-129 " +
             "org.apache.amaterasu.executor.yarn.executors.ActionsExecutorLauncher " +
             s"'${jobManager.jobId}' '${config.master}' '${actionData.name}' '${URLEncoder.encode(taskData, "UTF-8")}' '${URLEncoder.encode(execData, "UTF-8")}' '${actionData.id}-${container.getId.getContainerId}' '$address' " +
             s"1> ${ApplicationConstants.LOG_DIR_EXPANSION_VAR}/stdout " +
index 70da38e..b178f52 100644 (file)
@@ -32,10 +32,9 @@ import org.apache.hadoop.yarn.util.Records
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
 import scala.collection.concurrent
-import scala.concurrent.Future
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.{Future, _}
 import scala.util.{Failure, Success}
-import scala.concurrent._
-import ExecutionContext.Implicits.global
 
 class YarnRMCallbackHandler(nmClient: NMClientAsync,
                             jobManager: JobManager,
index cb9e896..0bd7afd 100755 (executable)
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-zk=192.168.1.90
-version=0.2.0-incubating-rc3
-master=192.168.1.90
-user=amaterasu
+zk=127.0.0.1
+version=0.2.0-incubating-rc4
+master=192.168.33.11
+user=root
 mode=yarn
 webserver.port=8000
 webserver.root=dist
@@ -25,9 +25,5 @@ yarn.jarspath=hdfs:///apps/amaterasu
 spark.home=/usr/hdp/current/spark2-client
 #spark.home=/opt/cloudera/parcels/SPARK2-2.1.0.cloudera2-1.cdh5.7.0.p0.171658/lib/spark2
 yarn.hadoop.home.dir=/etc/hadoop
-#spark.opts.spark.yarn.am.extraJavaOptions="-Dhdp.version=2.6.1.0-129"
-#spark.opts.spark.driver.extraJavaOptions="-Dhdp.version=2.6.1.0-129"
-spark.opts.spark.yarn.am.extraJavaOptions="-Dhdp.version=2.6.5.0-292"
-spark.opts.spark.driver.extraJavaOptions="-Dhdp.version=2.6.5.0-292"
-amaterasu.executor.extra.java.opts = "-Xmx1G -Dscala.usejavacp=true -Dhdp.version=2.6.5.0-292"
-
+spark.opts.spark.yarn.am.extraJavaOptions="-Dhdp.version=2.6.1.0-129"
+spark.opts.spark.driver.extraJavaOptions="-Dhdp.version=2.6.1.0-129"
index ac5af36..af42677 100755 (executable)
@@ -17,8 +17,8 @@
 package org.apache.amaterasu.leader.mesos
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
-import org.apache.amaterasu.leader.mesos.schedulers.ClusterScheduler
 import org.apache.amaterasu.leader.Kami
+import org.apache.amaterasu.leader.mesos.schedulers.ClusterScheduler
 import org.scalatest._
 
 class ClusterSchedulerTests extends FlatSpec with Matchers {
index 25769b6..0e321f0 100644 (file)
@@ -19,14 +19,8 @@ package org.apache.amaterasu.utilities
 
 import java.io.File
 
-import org.apache.amaterasu.leader.utilities.HttpServer
-import org.jsoup.Jsoup
-import org.jsoup.select.Elements
 import org.scalatest.{FlatSpec, Matchers}
 
-import scala.collection.JavaConverters._
-import scala.io.Source
-
 
 class HttpServerTests extends FlatSpec with Matchers {