AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own project (Master)
authorArun Manivannan <arun@arunma.com>
Sun, 1 Jul 2018 07:31:53 +0000 (15:31 +0800)
committerArun Manivannan <arun@arunma.com>
Sun, 1 Jul 2018 07:31:54 +0000 (15:31 +0800)
71 files changed:
.gitignore
build.gradle
common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala
executor/build.gradle
executor/src/main/scala/org/apache/amaterasu/executor/mesos/executors/MesosActionsExecutor.scala
executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/ActionsExecutor.scala
frameworks/spark/runner/build.gradle [new file with mode: 0644]
frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java [moved from executor/src/main/java/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkEntryPoint.java with 92% similarity]
frameworks/spark/runner/src/main/resources/codegen.py [moved from executor/src/test/resources/codegen.py with 100% similarity]
frameworks/spark/runner/src/main/resources/runtime.py [moved from executor/src/test/resources/runtime.py with 100% similarity]
frameworks/spark/runner/src/main/resources/spark-version-info.properties [new file with mode: 0644]
frameworks/spark/runner/src/main/resources/spark_intp.py [moved from executor/src/main/resources/spark_intp.py with 100% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRunnersProvider.scala with 93% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkExecutionQueue.scala with 94% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkResultQueue.scala with 85% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkRunner.scala with 95% similarity, mode: 0644]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/ResultQueue.scala with 94% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala [moved from executor/src/main/scala/org/apache/spark/repl/amaterasu/AmaSparkILoop.scala with 95% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala [moved from executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala with 94% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala [moved from executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkScalaRunner.scala with 97% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRRunner.scala with 69% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkSql/SparkSqlRunner.scala with 96% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv [moved from executor/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/json/SparkSqlTestData.json [moved from executor/src/test/resources/SparkSql/json/SparkSqlTestData.json with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc [moved from executor/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc [moved from executor/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/_SUCCESS [moved from executor/src/test/resources/SparkSql/parquet/_SUCCESS with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/_common_metadata [moved from executor/src/test/resources/SparkSql/parquet/_common_metadata with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/_metadata [moved from executor/src/test/resources/SparkSql/parquet/_metadata with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet [moved from executor/src/test/resources/tmp/job/start/x/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet [moved from executor/src/test/resources/tmp/job/start/x/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet with 100% similarity]
frameworks/spark/runner/src/test/resources/amaterasu.properties [moved from executor/src/test/resources/amaterasu.properties with 100% similarity]
frameworks/spark/runner/src/test/resources/codegen.py [moved from executor/src/main/resources/codegen.py with 100% similarity]
frameworks/spark/runner/src/test/resources/py4j-0.10.4-src.zip [moved from executor/src/test/resources/py4j-0.10.4-src.zip with 100% similarity]
frameworks/spark/runner/src/test/resources/py4j.tar.gz [moved from executor/src/test/resources/py4j.tar.gz with 100% similarity]
frameworks/spark/runner/src/test/resources/pyspark-with-amacontext.py [moved from executor/src/test/resources/pyspark-with-amacontext.py with 100% similarity]
frameworks/spark/runner/src/test/resources/pyspark.tar.gz [moved from executor/src/test/resources/pyspark.tar.gz with 100% similarity]
frameworks/spark/runner/src/test/resources/pyspark.zip [moved from executor/src/test/resources/pyspark.zip with 100% similarity]
frameworks/spark/runner/src/test/resources/runtime.py [moved from executor/src/main/resources/runtime.py with 100% similarity]
frameworks/spark/runner/src/test/resources/simple-pyspark.py [moved from executor/src/test/resources/simple-pyspark.py with 100% similarity]
frameworks/spark/runner/src/test/resources/simple-python-err.py [moved from executor/src/test/resources/simple-python-err.py with 100% similarity]
frameworks/spark/runner/src/test/resources/simple-python.py [moved from executor/src/test/resources/simple-python.py with 100% similarity]
frameworks/spark/runner/src/test/resources/simple-spark.scala [moved from executor/src/test/resources/simple-spark.scala with 83% similarity]
frameworks/spark/runner/src/test/resources/spark-version-info.properties [moved from executor/src/main/resources/spark-version-info.properties with 100% similarity]
frameworks/spark/runner/src/test/resources/spark_intp.py [moved from executor/src/test/resources/spark_intp.py with 100% similarity]
frameworks/spark/runner/src/test/resources/step-2.scala [moved from executor/src/test/resources/step-2.scala with 94% similarity]
frameworks/spark/runner/src/test/resources/tmp/job/start/x/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet [moved from executor/src/test/resources/SparkSql/parquet/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet with 100% similarity]
frameworks/spark/runner/src/test/resources/tmp/job/start/x/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet [moved from executor/src/test/resources/SparkSql/parquet/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet with 100% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala [moved from executor/src/test/scala/org/apache/amaterasu/spark/SparkTestsSuite.scala with 90% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala [moved from executor/src/test/scala/org/apache/amaterasu/RunnersTests/RunnersLoadingTests.scala with 96% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala [moved from executor/src/test/scala/org/apache/amaterasu/spark/PySparkRunnerTests.scala with 96% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala [moved from executor/src/test/scala/org/apache/amaterasu/spark/SparkScalaRunnerTests.scala with 92% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala [moved from executor/src/test/scala/org/apache/amaterasu/spark/SparkSqlRunnerTests.scala with 96% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala [moved from executor/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala with 100% similarity]
frameworks/spark/runtime/build.gradle [new file with mode: 0644]
frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/runtime/AmaContext.scala with 93% similarity, mode: 0644]
leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java
leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
leader/src/main/scala/org/apache/amaterasu/leader/dsl/JobParser.scala
leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/AmaterasuScheduler.scala
leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
leader/src/main/scala/org/apache/amaterasu/leader/utilities/ActiveReportListener.scala
leader/src/main/scala/org/apache/amaterasu/leader/utilities/HttpServer.scala
leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
leader/src/main/scripts/ama-start-mesos.sh
leader/src/main/scripts/ama-start-yarn.sh
leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala
leader/src/test/scala/org/apache/amaterasu/utilities/HttpServerTests.scala
settings.gradle

index fabb847..dbb51cb 100755 (executable)
@@ -48,3 +48,5 @@ bin/
 amaterasu-executor/
 project/project/
 executor/spark-warehouse/
+repo
+repo/**
index 0f11347..00e44ea 100644 (file)
@@ -25,10 +25,6 @@ allprojects {
     version '0.2.0-incubating-rc4'
 }
 
-project(':leader')
-project(':common')
-project(':executor')
-
 task copyLeagalFiles(type: Copy) {
     from "./DISCLAIMER", "./LICENSE", "./NOTICE"
     into "${buildDir}/amaterasu"
index 8a44019..fe69260 100755 (executable)
@@ -16,9 +16,9 @@
  */
 package org.apache.amaterasu.common.execution.actions
 
-import NotificationLevel.NotificationLevel
-import NotificationType.NotificationType
 import com.fasterxml.jackson.annotation.JsonProperty
+import org.apache.amaterasu.common.execution.actions.NotificationLevel.NotificationLevel
+import org.apache.amaterasu.common.execution.actions.NotificationType.NotificationType
 
 abstract class Notifier {
 
index 21bc2b0..09e269c 100644 (file)
@@ -54,7 +54,6 @@ dependencies {
 
     compile group: 'org.scala-lang', name: 'scala-library', version: '2.11.8'
     compile group: 'org.scala-lang', name: 'scala-reflect', version: '2.11.8'
-    compile group: 'org.scala-lang', name: 'scala-compiler', version: '2.11.8'
     compile group: 'io.netty', name: 'netty-all', version: '4.0.42.Final'
     compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.5'
     compile group: 'org.apache.maven', name: 'maven-core', version: '3.0.5'
@@ -75,18 +74,7 @@ dependencies {
     compile project(':common')
     compile project(':amaterasu-sdk')
 
-    //runtime dependency for spark
-    provided('org.apache.spark:spark-repl_2.11:2.2.1')
-    provided('org.apache.spark:spark-core_2.11:2.2.1')
-
-    testCompile project(':common')
-    testCompile "gradle.plugin.com.github.maiflai:gradle-scalatest:0.14"
-    testRuntime 'org.pegdown:pegdown:1.1.0'
-    testCompile 'junit:junit:4.11'
-    testCompile 'org.scalatest:scalatest_2.11:3.0.2'
-    testCompile 'org.scala-lang:scala-library:2.11.8'
-    testCompile('org.apache.spark:spark-repl_2.11:2.2.1')
-    testCompile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.9'
+
 
 }
 
index 9ab75be..90c2001 100755 (executable)
@@ -26,7 +26,6 @@ import org.apache.amaterasu.executor.common.executors.ProvidersFactory
 import org.apache.mesos.Protos._
 import org.apache.mesos.protobuf.ByteString
 import org.apache.mesos.{Executor, ExecutorDriver, MesosExecutorDriver}
-import org.apache.spark.SparkContext
 
 import scala.collection.JavaConverters._
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -37,7 +36,6 @@ class MesosActionsExecutor extends Executor with Logging {
 
   var master: String = _
   var executorDriver: ExecutorDriver = _
-  var sc: SparkContext = _
   var jobId: String = _
   var actionName: String = _
   //  var sparkScalaRunner: SparkScalaRunner = _
@@ -83,7 +81,7 @@ class MesosActionsExecutor extends Executor with Logging {
     notifier = new MesosNotifier(driver)
     notifier.info(s"Executor ${executorInfo.getExecutorId.getValue} registered")
     val outStream = new ByteArrayOutputStream()
-    providersFactory = ProvidersFactory(data, jobId, outStream, notifier, executorInfo.getExecutorId.getValue, hostName, propFile = "./amaterasu.properties")
+    providersFactory = ProvidersFactory(data, jobId, outStream, notifier, executorInfo.getExecutorId.getValue, hostName, "./amaterasu.properties")
 
   }
 
index f4f553c..b5f8700 100644 (file)
@@ -19,24 +19,18 @@ package org.apache.amaterasu.executor.yarn.executors
 import java.io.ByteArrayOutputStream
 import java.net.{InetAddress, URLDecoder}
 
-import scala.collection.JavaConverters._
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import org.apache.amaterasu.common.dataobjects.{ExecData, TaskData}
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.executor.common.executors.{ActiveNotifier, ProvidersFactory}
-import org.apache.hadoop.net.NetUtils
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.spark.SparkContext
 
-import scala.reflect.internal.util.ScalaClassLoader
-import scala.reflect.internal.util.ScalaClassLoader.URLClassLoader
+import scala.collection.JavaConverters._
 
 
 class ActionsExecutor extends Logging {
 
   var master: String = _
-  var sc: SparkContext = _
   var jobId: String = _
   var actionName: String = _
   var taskData: TaskData = _
diff --git a/frameworks/spark/runner/build.gradle b/frameworks/spark/runner/build.gradle
new file mode 100644 (file)
index 0000000..cc6c902
--- /dev/null
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+    id 'com.github.johnrengelman.shadow' version '1.2.4'
+    id 'com.github.maiflai.scalatest' version '0.6-5-g9065d91'
+    id 'scala'
+    id 'java'
+}
+
+shadowJar {
+    zip64 true
+}
+
+repositories {
+    maven {
+        url "https://plugins.gradle.org/m2/"
+    }
+    mavenCentral()
+}
+
+test {
+    maxParallelForks = 1
+    forkEvery = 1
+}
+
+configurations {
+    provided
+}
+
+sourceSets {
+    main.compileClasspath += configurations.provided
+    test.compileClasspath += configurations.provided
+    test.runtimeClasspath += configurations.provided
+}
+
+dependencies {
+
+    compile project(':executor')
+    compile project(':spark-runtime')
+    compile group: 'org.scala-lang', name: 'scala-library', version: '2.11.8'
+    compile group: 'org.scala-lang', name: 'scala-reflect', version: '2.11.8'
+    compile group: 'org.scala-lang', name: 'scala-compiler', version: '2.11.8'
+    compile group: 'io.netty', name: 'netty-all', version: '4.0.42.Final'
+    compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.5'
+    compile group: 'org.apache.maven', name: 'maven-core', version: '3.0.5'
+    compile group: 'org.reflections', name: 'reflections', version: '0.9.10'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.6.5'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.6.5'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.6.5'
+    compile group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.11', version: '2.6.5'
+    compile group: 'net.liftweb', name: 'lift-json_2.11', version: '3.2.0'
+
+    compile('com.jcabi:jcabi-aether:0.10.1') {
+        exclude group: 'org.jboss.netty'
+    }
+    compile('org.apache.activemq:activemq-client:5.15.2') {
+        exclude group: 'org.jboss.netty'
+    }
+
+    //compile project(':common')
+    //compile project(':amaterasu-sdk')
+
+    //runtime dependency for spark
+    provided('org.apache.spark:spark-repl_2.11:2.2.1')
+    provided('org.apache.spark:spark-core_2.11:2.2.1')
+
+    testCompile project(':common')
+    testCompile "gradle.plugin.com.github.maiflai:gradle-scalatest:0.14"
+    testRuntime 'org.pegdown:pegdown:1.1.0'
+    testCompile 'junit:junit:4.11'
+    testCompile 'org.scalatest:scalatest_2.11:3.0.2'
+    testCompile 'org.scala-lang:scala-library:2.11.8'
+    testCompile('org.apache.spark:spark-repl_2.11:2.2.1')
+    testCompile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.9'
+
+}
+
+sourceSets {
+    test {
+        resources.srcDirs += [file('src/test/resources')]
+    }
+
+    main {
+        scala {
+            srcDirs = ['src/main/scala', 'src/main/java']
+        }
+        java {
+            srcDirs = []
+        }
+    }
+}
+
+test {
+
+    maxParallelForks = 1
+}
+
+task copyToHome(type: Copy) {
+    dependsOn shadowJar
+    from 'build/libs'
+    into '../../../build/amaterasu/dist'
+    from 'build/resources/main'
+    into '../../../build/amaterasu/dist'
+}
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark;
+package org.apache.amaterasu.framework.spark.runner.pyspark;
 
-import org.apache.amaterasu.executor.runtime.AmaContext;
 import org.apache.amaterasu.common.runtime.Environment;
-
+import org.apache.amaterasu.framework.spark.runtime.AmaContext;
+import org.apache.spark.SparkConf;
 import org.apache.spark.SparkEnv;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.SQLContext;
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
-
 import org.apache.spark.sql.SparkSession;
 import py4j.GatewayServer;
 
@@ -35,7 +32,7 @@ import java.util.concurrent.ConcurrentHashMap;
 public class PySparkEntryPoint {
 
     //private static Boolean started = false;
-    private static PySparkExecutionQueue queue = new PySparkExecutionQueue();
+    private static  PySparkExecutionQueue queue = new PySparkExecutionQueue();
     private static ConcurrentHashMap<String, ResultQueue> resultQueues = new ConcurrentHashMap<>();
 
     private static int port = 0;
diff --git a/frameworks/spark/runner/src/main/resources/spark-version-info.properties b/frameworks/spark/runner/src/main/resources/spark-version-info.properties
new file mode 100644 (file)
index 0000000..ce0b312
--- /dev/null
@@ -0,0 +1,11 @@
+version=2.1.0-SNAPSHOT
+
+user=root
+
+revision=738b4cc548ca48c010b682b8bc19a2f7e1947cfe
+
+branch=master
+
+date=2016-07-27T11:23:21Z
+
+url=https://github.com/apache/spark.git
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark
+package org.apache.amaterasu.framework.spark.runner
 
 import java.io._
 
@@ -24,10 +24,10 @@ import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.execution.dependencies.{Dependencies, PythonDependencies, PythonPackage}
 import org.apache.amaterasu.common.logging.Logging
-import org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark.PySparkRunner
-import org.apache.amaterasu.executor.execution.actions.runners.spark.SparkSql.SparkSqlRunner
+import org.apache.amaterasu.framework.spark.runner.pyspark.PySparkRunner
+import org.apache.amaterasu.framework.spark.runner.repl.{SparkRunnerHelper, SparkScalaRunner}
+import org.apache.amaterasu.framework.spark.runner.sparksql.SparkSqlRunner
 import org.apache.amaterasu.sdk.{AmaterasuRunner, RunnersProvider}
-import org.apache.spark.repl.amaterasu.runners.spark.{SparkRunnerHelper, SparkScalaRunner}
 import org.eclipse.aether.util.artifact.JavaScopes
 import org.sonatype.aether.repository.RemoteRepository
 import org.sonatype.aether.util.artifact.DefaultArtifact
@@ -113,7 +113,7 @@ class SparkRunnersProvider extends RunnersProvider with Logging {
 
     this.clusterConfig.mode match {
       case "yarn" => Seq("sh", "-c", "export HOME=$PWD && ./miniconda.sh -b -p miniconda") ! shellLoger
-      case "mesos" => Seq("sh", "Miniconda2-latest-Linux-x86_64.sh", "-b", "-p", "miniconda") ! shellLoger
+      case "mesos" => Seq("sh", "miniconda.sh", "-b", "-p", "miniconda") ! shellLoger
     }
 
     Seq("bash", "-c", "export HOME=$PWD && ./miniconda/bin/python -m conda install -y conda-build") ! shellLoger
@@ -14,9 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark
+package org.apache.amaterasu.framework.spark.runner.pyspark
 
-import org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark.ResultType.ResultType
+import org.apache.amaterasu.framework.spark.runner.pyspark.ResultType.ResultType
 
 object ResultType extends Enumeration {
   type ResultType = Value
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark
+package org.apache.amaterasu.framework.spark.runner.pyspark
 
 import java.io.File
 import java.util
@@ -128,7 +128,7 @@ object PySparkRunner {
         }
         else {
           sparkCmd = Seq(pysparkPath, intpPath, port.toString)
-        }
+    }
         var pysparkPython = "/usr/bin/python"
 
         if (pyDeps != null &&
@@ -136,9 +136,9 @@ object PySparkRunner {
           pysparkPython = "./miniconda/bin/python"
         }
         val proc = Process(sparkCmd, None,
-          "PYTHONPATH" -> pypath,
-          "PYSPARK_PYTHON" -> pysparkPython,
-        "PYTHONHASHSEED" -> 0.toString)
+      "PYTHONPATH" -> pypath,
+      "PYSPARK_PYTHON" -> pysparkPython,
+      "PYTHONHASHSEED" -> 0.toString)
 
         proc.run(shellLoger)
     }
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.repl.amaterasu.runners.spark
+package org.apache.amaterasu.framework.spark.runner.repl
 
 import java.io.{ByteArrayOutputStream, File, PrintWriter}
+import java.nio.file.{Files, Paths}
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.common.utils.FileUtils
-import org.apache.spark.repl.amaterasu.AmaSparkILoop
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.util.Utils
 import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
 
 import scala.tools.nsc.GenericRunnerSettings
 import scala.tools.nsc.interpreter.IMain
@@ -34,8 +33,9 @@ import scala.tools.nsc.interpreter.IMain
 object SparkRunnerHelper extends Logging {
 
   private val conf = new SparkConf()
-  private val rootDir = conf.getOption("spark.repl.classdir").getOrElse(Utils.getLocalDir(conf))
-  private val outputDir = Utils.createTempDir(root = rootDir, namePrefix = "repl")
+  private val rootDir = conf.get("spark.repl.classdir", System.getProperty("java.io.tmpdir"))
+  private val outputDir = Files.createTempDirectory(Paths.get(rootDir), "repl").toFile
+  outputDir.deleteOnExit()
 
   private var sparkSession: SparkSession = _
 
@@ -145,7 +145,7 @@ object SparkRunnerHelper extends Logging {
       case "yarn" =>
         conf.set("spark.home", config.spark.home)
           // TODO: parameterize those
-          .setJars(s"executor.jar" +: jars)
+          .setJars(Seq("executor.jar", "spark-runner.jar", "spark-runtime.jar") ++ jars)
           .set("spark.history.kerberos.keytab", "/etc/security/keytabs/spark.headless.keytab")
           .set("spark.driver.extraLibraryPath", "/usr/hdp/current/hadoop-client/lib/native:/usr/hdp/current/hadoop-client/lib/native/Linux-amd64-64")
           .set("spark.yarn.queue", "default")
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.repl.amaterasu.runners.spark
+package org.apache.amaterasu.framework.spark.runner.repl
 
 import java.io.ByteArrayOutputStream
 import java.util
@@ -22,7 +22,7 @@ import java.util
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
-import org.apache.amaterasu.executor.runtime.AmaContext
+import org.apache.amaterasu.framework.spark.runtime.AmaContext
 import org.apache.amaterasu.sdk.AmaterasuRunner
 import org.apache.spark.sql.{Dataset, SparkSession}
 
@@ -142,7 +142,7 @@ class SparkScalaRunner(var env: Environment,
     interpreter.interpret("import org.apache.spark.sql.SQLContext")
     interpreter.interpret("import org.apache.spark.sql.{ Dataset, SparkSession }")
     interpreter.interpret("import org.apache.spark.sql.SaveMode")
-    interpreter.interpret("import org.apache.amaterasu.executor.runtime.AmaContext")
+    interpreter.interpret("import org.apache.amaterasu.framework.spark.runtime.AmaContext")
     interpreter.interpret("import org.apache.amaterasu.common.runtime.Environment")
 
     // creating a map (_contextStore) to hold the different spark contexts
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark
+package org.apache.amaterasu.framework.spark.runner.sparkr
 
 import java.io.ByteArrayOutputStream
 import java.util
@@ -28,21 +28,21 @@ import org.apache.spark.SparkContext
 
 class SparkRRunner extends Logging with AmaterasuRunner {
 
-  override def getIdentifier = "spark-r"
+    override def getIdentifier = "spark-r"
 
-  override def executeSource(actionSource: String, actionName: String, exports: util.Map[String, String]): Unit = {
-  }
+    override def executeSource(actionSource: String, actionName: String, exports: util.Map[String, String]): Unit = {
+    }
 }
 
 object SparkRRunner {
-  def apply(
-    env: Environment,
-    jobId: String,
-    sparkContext: SparkContext,
-    outStream: ByteArrayOutputStream,
-    notifier: Notifier,
-    jars: Seq[String]
-  ): SparkRRunner = {
-    new SparkRRunner()
-  }
+    def apply(
+               env: Environment,
+               jobId: String,
+               sparkContext: SparkContext,
+               outStream: ByteArrayOutputStream,
+               notifier: Notifier,
+               jars: Seq[String]
+             ): SparkRRunner = {
+        new SparkRRunner()
+    }
 }
\ No newline at end of file
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.SparkSql
+package org.apache.amaterasu.framework.spark.runner.sparksql
 
 import java.io.File
 import java.util
@@ -22,10 +22,11 @@ import java.util
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
-import org.apache.amaterasu.executor.runtime.AmaContext
+import org.apache.amaterasu.framework.spark.runtime.AmaContext
 import org.apache.amaterasu.sdk.AmaterasuRunner
 import org.apache.commons.io.FilenameUtils
 import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+
 import scala.collection.JavaConverters._
 
 /**
@@ -101,8 +102,8 @@ class SparkSqlRunner extends Logging with AmaterasuRunner {
 
         try{
 
-        result = spark.sql(parsedQuery)
-        notifier.success(parsedQuery)
+          result = spark.sql(parsedQuery)
+          notifier.success(parsedQuery)
         } catch {
           case e: Exception => notifier.error(parsedQuery, e.getMessage)
         }
@@ -14,8 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import org.apache.amaterasu.executor.runtime.AmaContext
-import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+import org.apache.spark.sql.{DataFrame, SaveMode}
 
 val data = Seq(1,3,4,5,6)
 
@@ -23,8 +22,6 @@ val data = Seq(1,3,4,5,6)
 val sc = AmaContext.sc
 val rdd = sc.parallelize(data)
 val sqlContext = AmaContext.spark
-
-import sqlContext.implicits._
 val x: DataFrame = rdd.toDF()
 
 x.write.mode(SaveMode.Overwrite)
\ No newline at end of file
similarity index 94%
rename from executor/src/test/resources/step-2.scala
rename to frameworks/spark/runner/src/test/resources/step-2.scala
index a3d034c..86fd048 100755 (executable)
@@ -1,3 +1,5 @@
+
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -14,7 +16,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import org.apache.amaterasu.executor.runtime.AmaContext
 
 
 val highNoDf = AmaContext.getDataFrame("start", "x").where("age > 20")
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.spark
+package org.apache.amaterasu.framework.spark
 
 import java.io.{ByteArrayOutputStream, File}
 
-import org.apache.amaterasu.RunnersTests.RunnersLoadingTests
 import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.common.execution.dependencies._
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
+import org.apache.amaterasu.framework.spark.runner.RunnersLoadingTests
+import org.apache.amaterasu.framework.spark.runner.pyspark.PySparkRunnerTests
+import org.apache.amaterasu.framework.spark.runner.repl.{SparkScalaRunner, SparkScalaRunnerTests}
+import org.apache.amaterasu.framework.spark.runner.sparksql.SparkSqlRunnerTests
 import org.apache.amaterasu.utilities.TestNotifier
-import org.apache.spark.repl.amaterasu.runners.spark.SparkScalaRunner
 import org.apache.spark.sql.SparkSession
 import org.scalatest._
 
-
-
 import scala.collection.mutable.ListBuffer
 
 
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.RunnersTests
+package org.apache.amaterasu.framework.spark.runner
 
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.spark
+package org.apache.amaterasu.framework.spark.runner.pyspark
 
 import java.io.File
 
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
-import org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark.PySparkRunner
 import org.apache.log4j.{Level, Logger}
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FlatSpec, Matchers}
 
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.spark
+package org.apache.amaterasu.framework.spark.runner.repl
 
-
-import scala.collection.JavaConverters._
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
-import org.apache.amaterasu.executor.runtime.AmaContext
-import org.apache.spark.repl.amaterasu.runners.spark.SparkScalaRunner
+import org.apache.amaterasu.framework.spark.runtime.AmaContext
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FlatSpec, Matchers}
 
+import scala.collection.JavaConverters._
 import scala.io.Source
 
 @DoNotDiscover
@@ -33,6 +31,7 @@ class SparkScalaRunnerTests extends FlatSpec with Matchers with BeforeAndAfterAl
 
   "SparkScalaRunner" should "execute the simple-spark.scala" in {
 
+
     val sparkRunner =factory.getRunner("spark", "scala").get.asInstanceOf[SparkScalaRunner]
     val script = getClass.getResource("/simple-spark.scala").getPath
     val sourceCode = Source.fromFile(script).getLines().mkString("\n")
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.spark
+package org.apache.amaterasu.framework.spark.runner.sparksql
 
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
-import org.apache.amaterasu.executor.execution.actions.runners.spark.SparkSql.SparkSqlRunner
 import org.apache.amaterasu.utilities.TestNotifier
-import org.apache.log4j.Logger
-import org.apache.log4j.Level
+import org.apache.log4j.{Level, Logger}
 import org.apache.spark.sql.{SaveMode, SparkSession}
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FlatSpec, Matchers}
 
 import scala.collection.JavaConverters._
 
-/**
-  * Created by kirupa on 10/12/16.
-  */
 @DoNotDiscover
 class SparkSqlRunnerTests extends FlatSpec with Matchers with BeforeAndAfterAll {
 
diff --git a/frameworks/spark/runtime/build.gradle b/frameworks/spark/runtime/build.gradle
new file mode 100644 (file)
index 0000000..9bba2e4
--- /dev/null
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+    id 'com.github.johnrengelman.shadow' version '1.2.4'
+    id 'com.github.maiflai.scalatest' version '0.6-5-g9065d91'
+    id 'scala'
+    id 'java'
+}
+
+shadowJar {
+    zip64 true
+}
+
+repositories {
+    maven {
+        url "https://plugins.gradle.org/m2/"
+    }
+    mavenCentral()
+}
+
+test {
+    maxParallelForks = 1
+    forkEvery = 1
+}
+
+configurations {
+    provided
+    runtime.exclude module: 'hadoop-common'
+    runtime.exclude module: 'hadoop-yarn-api'
+    runtime.exclude module: 'hadoop-yarn-client'
+    runtime.exclude module: 'hadoop-hdfs'
+    runtime.exclude module: 'mesos'
+    runtime.exclude module: 'scala-compiler'
+}
+
+sourceSets {
+    main.compileClasspath += configurations.provided
+    test.compileClasspath += configurations.provided
+    test.runtimeClasspath += configurations.provided
+}
+
+dependencies {
+
+    compile project(':executor')
+    provided('org.apache.spark:spark-repl_2.11:2.2.1')
+    provided('org.apache.spark:spark-core_2.11:2.2.1')
+
+}
+
+sourceSets {
+    test {
+        resources.srcDirs += [file('src/test/resources')]
+    }
+
+    main {
+        scala {
+            srcDirs = ['src/main/scala', 'src/main/java']
+        }
+        java {
+            srcDirs = []
+        }
+    }
+}
+
+test {
+
+    maxParallelForks = 1
+}
+
+task copyToHome(type: Copy) {
+    from 'build/libs'
+    into '../../../build/amaterasu/dist'
+    from 'build/resources/main'
+    into '../../../build/amaterasu/dist'
+}
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.runtime
+package org.apache.amaterasu.framework.spark.runtime
 
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.spark.SparkContext
-import org.apache.spark.sql._
+import org.apache.spark.sql.{DataFrame, Dataset, Encoder, SparkSession}
 
 object AmaContext extends Logging {
 
@@ -40,15 +40,11 @@ object AmaContext extends Logging {
   }
 
   def getDataFrame(actionName: String, dfName: String, format: String = "parquet"): DataFrame = {
-
     spark.read.format(format).load(s"${env.workingDir}/$jobId/$actionName/$dfName")
-
   }
 
   def getDataset[T: Encoder](actionName: String, dfName: String, format: String = "parquet"): Dataset[T] = {
-
     getDataFrame(actionName, dfName, format).as[T]
-
   }
 
 }
index be0fc05..38a9c38 100644 (file)
  */
 package org.apache.amaterasu.leader.yarn;
 
-import org.apache.commons.cli.*;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
 
 public class ArgsParser {
     private static Options getOptions() {
index e3c2812..8f16ee7 100644 (file)
@@ -31,7 +31,16 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -43,11 +52,19 @@ import org.apache.log4j.LogManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.jms.*;
+import javax.jms.Connection;
+import javax.jms.JMSException;
+import javax.jms.MessageConsumer;
+import javax.jms.Session;
+import javax.jms.Topic;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 
 import static java.lang.System.exit;
 
index 8ef1c7a..aba6210 100755 (executable)
@@ -18,12 +18,12 @@ package org.apache.amaterasu.leader.dsl
 
 import java.util.concurrent.BlockingQueue
 
-import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import com.fasterxml.jackson.databind.node.ArrayNode
+import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import org.apache.amaterasu.common.dataobjects.ActionData
-import org.apache.amaterasu.leader.execution.actions.{Action, ErrorAction, SequentialAction}
 import org.apache.amaterasu.leader.execution.JobManager
+import org.apache.amaterasu.leader.execution.actions.{Action, ErrorAction, SequentialAction}
 import org.apache.curator.framework.CuratorFramework
 
 import scala.collection.JavaConverters._
index 8c487c1..f6dea22 100644 (file)
@@ -19,7 +19,6 @@ package org.apache.amaterasu.leader.frameworks.spark
 import java.io.File
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
-import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.leader.utilities.{DataLoader, MemoryFormatParser}
 import org.apache.amaterasu.sdk.frameworks.FrameworkSetupProvider
 import org.apache.amaterasu.sdk.frameworks.configuration.DriverConfiguration
index f2f2c00..4b1a74c 100755 (executable)
@@ -17,7 +17,6 @@
 package org.apache.amaterasu.leader.mesos.schedulers
 
 import org.apache.amaterasu.common.logging.Logging
-
 import org.apache.mesos.Protos.{Resource, Value}
 import org.apache.mesos.Scheduler
 
index 87a8f5d..2c2e8af 100755 (executable)
  */
 package org.apache.amaterasu.leader.mesos.schedulers
 
-import java.io.File
 import java.util
 import java.util.concurrent.locks.ReentrantLock
 import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue}
-import java.util.{Collections, Properties, UUID}
+import java.util.{Collections, UUID}
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
@@ -185,7 +184,7 @@ class JobScheduler extends AmaterasuScheduler {
                     .setExtract(true)
                     .build())
                   .addUris(URI.newBuilder()
-                    .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/Miniconda2-latest-Linux-x86_64.sh")
+                    .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/miniconda.sh")
                     .setExecutable(false)
                     .setExtract(false)
                     .build())
index 2664665..b3ffaad 100644 (file)
@@ -17,9 +17,7 @@
 package org.apache.amaterasu.leader.utilities
 
 import javax.jms.{Message, MessageListener, TextMessage}
-
 import net.liftweb.json._
-import net.liftweb.json.JsonDSL._
 import org.apache.amaterasu.common.execution.actions.{Notification, NotificationLevel, NotificationType}
 
 class ActiveReportListener extends MessageListener {
index 2e01963..9dea8de 100644 (file)
@@ -18,22 +18,16 @@ package org.apache.amaterasu.leader.utilities
 
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.log4j.{BasicConfigurator, Level, Logger}
-import org.eclipse.jetty.server.{Handler, Server, ServerConnector}
 import org.eclipse.jetty.server.handler._
-import org.eclipse.jetty.servlet.{DefaultServlet, ServletContextHandler, ServletHolder}
-import org.eclipse.jetty.toolchain.test.MavenTestingUtils
-import org.eclipse.jetty.util.thread.QueuedThreadPool
+import org.eclipse.jetty.server.{Server, ServerConnector}
 import org.eclipse.jetty.util.log.StdErrLog
-import org.eclipse.jetty.util.resource.Resource
 import org.jsoup.Jsoup
 import org.jsoup.select.Elements
 
 import scala.collection.JavaConverters._
 import scala.io.{BufferedSource, Source}
-import scala.text.Document
 
 /**
-  * Created by kirupa
   * Implementation of Jetty Web server to server Amaterasu libraries and other distribution files
   */
 object HttpServer extends Logging {
index 1828100..406c150 100644 (file)
@@ -250,7 +250,7 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging {
         val commands: List[String] = List(
           "/bin/bash ./miniconda.sh -b -p $PWD/miniconda && ",
           s"/bin/bash spark/bin/load-spark-env.sh && ",
-          s"java -cp spark/jars/*:executor.jar:spark/conf/:${config.YARN.hadoopHomeDir}/conf/ " +
+          s"java -cp spark/jars/*:executor.jar:spark-runner.jar:spark-runtime.jar:spark/conf/:${config.YARN.hadoopHomeDir}/conf/ " +
             "-Xmx1G " +
             "-Dscala.usejavacp=true " +
             "-Dhdp.version=2.6.1.0-129 " +
@@ -266,22 +266,37 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging {
         ctx.setCommands(commands)
         ctx.setTokens(allTokens)
 
+        val yarnJarPath = new Path(config.YARN.hdfsJarsPath)
+
+        //TODO Arun - Remove the hardcoding of the dist path
+        /*  val resources = mutable.Map[String, LocalResource]()
+          val binaryFileIter = fs.listFiles(new Path(s"${config.YARN.hdfsJarsPath}/dist"), false)
+          while (binaryFileIter.hasNext) {
+            val eachFile = binaryFileIter.next().getPath
+            resources (eachFile.getName) = setLocalResourceFromPath(fs.makeQualified(eachFile))
+          }
+          resources("log4j.properties") = setLocalResourceFromPath(fs.makeQualified(new Path(s"${config.YARN.hdfsJarsPath}/log4j.properties")))
+          resources ("amaterasu.properties") = setLocalResourceFromPath(fs.makeQualified(new Path(s"${config.YARN.hdfsJarsPath}/amaterasu.properties")))*/
+
         val resources = mutable.Map[String, LocalResource](
-          "executor.jar" -> executorJar,
-          "amaterasu.properties" -> propFile,
+          "executor.jar" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path(s"/dist/executor-${config.version}-all.jar"))),
+          "spark-runner.jar" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path(s"/dist/spark-runner-${config.version}-all.jar"))),
+          "spark-runtime.jar" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path(s"/dist/spark-runtime-${config.version}.jar"))),
+          "amaterasu.properties" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/amaterasu.properties"))),
+          "log4j.properties" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/log4j.properties"))),
           // TODO: Nadav/Eyal all of these should move to the executor resource setup
-          "miniconda.sh" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/Miniconda2-latest-Linux-x86_64.sh"))),
-          "codegen.py" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/codegen.py"))),
-          "runtime.py" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/runtime.py"))),
-          "spark-version-info.properties" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/spark-version-info.properties"))),
-          "spark_intp.py" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/spark_intp.py"))))
+          "miniconda.sh" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/miniconda.sh"))),
+          "codegen.py" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/codegen.py"))),
+          "runtime.py" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/runtime.py"))),
+          "spark-version-info.properties" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/spark-version-info.properties"))),
+          "spark_intp.py" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/spark_intp.py"))))
 
         val frameworkFactory = FrameworkProvidersFactory(env, config)
         val framework = frameworkFactory.getFramework(actionData.groupId)
 
         //adding the framework and executor resources
-        setupResources(framework.getGroupIdentifier, resources, framework.getGroupIdentifier)
-        setupResources(s"${framework.getGroupIdentifier}/${actionData.typeId}", resources, s"${framework.getGroupIdentifier}-${actionData.typeId}")
+        setupResources(yarnJarPath, framework.getGroupIdentifier, resources, framework.getGroupIdentifier)
+        setupResources(yarnJarPath, s"${framework.getGroupIdentifier}/${actionData.typeId}", resources, s"${framework.getGroupIdentifier}-${actionData.typeId}")
 
         ctx.setLocalResources(resources)
 
@@ -327,9 +342,9 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging {
     ByteBuffer.wrap(dob.getData, 0, dob.getLength)
   }
 
-  private def setupResources(frameworkPath: String, countainerResources: mutable.Map[String, LocalResource], resourcesPath: String): Unit = {
+  private def setupResources(yarnJarPath: Path, frameworkPath: String, countainerResources: mutable.Map[String, LocalResource], resourcesPath: String): Unit = {
 
-    val sourcePath = Path.mergePaths(jarPath, new Path(s"/$resourcesPath"))
+    val sourcePath = Path.mergePaths(yarnJarPath, new Path(s"/$resourcesPath"))
 
     if (fs.exists(sourcePath)) {
 
index 70da38e..b178f52 100644 (file)
@@ -32,10 +32,9 @@ import org.apache.hadoop.yarn.util.Records
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
 import scala.collection.concurrent
-import scala.concurrent.Future
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.{Future, _}
 import scala.util.{Failure, Success}
-import scala.concurrent._
-import ExecutionContext.Implicits.global
 
 class YarnRMCallbackHandler(nmClient: NMClientAsync,
                             jobManager: JobManager,
index 18dbed9..e01ea42 100755 (executable)
@@ -126,9 +126,9 @@ if ! ls ${BASEDIR}/dist/spark*.tgz 1> /dev/null 2>&1; then
     #wget https://d3kbcqa49mib13.cloudfront.net/spark-2.2.1-bin-hadoop2.7.tgz -P ${BASEDIR}/dist
     wget http://apache.mirror.digitalpacific.com.au/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz -P ${BASEDIR}/dist
 fi
-if [ ! -f ${BASEDIR}/dist/Miniconda2-latest-Linux-x86_64.sh ]; then
+if [ ! -f ${BASEDIR}/dist/miniconda.sh ]; then
     echo "${bold}Fetching miniconda distributable ${NC}"
-    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -P ${BASEDIR}/dist
+    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh -P ${BASEDIR}/dist
 fi
 cp ${BASEDIR}/amaterasu.properties ${BASEDIR}/dist
 eval $CMD | grep "===>"
index 8aa58f1..f6af18f 100755 (executable)
@@ -136,9 +136,9 @@ fi
 
 echo $CMD
 
-if [ ! -f ${BASEDIR}/dist/Miniconda2-latest-Linux-x86_64.sh ]; then
+if [ ! -f ${BASEDIR}/dist/miniconda.sh ]; then
     echo "${bold}Fetching miniconda distributable ${NC}"
-    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -P ${BASEDIR}/dist
+    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O ${BASEDIR}/dist/miniconda.sh
 fi
 
 
index ac5af36..af42677 100755 (executable)
@@ -17,8 +17,8 @@
 package org.apache.amaterasu.leader.mesos
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
-import org.apache.amaterasu.leader.mesos.schedulers.ClusterScheduler
 import org.apache.amaterasu.leader.Kami
+import org.apache.amaterasu.leader.mesos.schedulers.ClusterScheduler
 import org.scalatest._
 
 class ClusterSchedulerTests extends FlatSpec with Matchers {
index 25769b6..0e321f0 100644 (file)
@@ -19,14 +19,8 @@ package org.apache.amaterasu.utilities
 
 import java.io.File
 
-import org.apache.amaterasu.leader.utilities.HttpServer
-import org.jsoup.Jsoup
-import org.jsoup.select.Elements
 import org.scalatest.{FlatSpec, Matchers}
 
-import scala.collection.JavaConverters._
-import scala.io.Source
-
 
 class HttpServerTests extends FlatSpec with Matchers {
 
index 1056e01..c222795 100644 (file)
  * limitations under the License.
  */
 include 'leader'
-include 'executor'
+project(':leader')
+
 include 'common'
+project(':common')
+
+include 'executor'
+project(':executor')
+
 include 'sdk'
 findProject(':sdk')?.name = 'amaterasu-sdk'
 
+//Spark
+include 'spark-runner'
+project(':spark-runner').projectDir=file("frameworks/spark/runner")
+include 'spark-runtime'
+project(':spark-runtime').projectDir=file("frameworks/spark/runtime")
+