created spark-dispatcher and leader-common projects
authorYaniv Rodenski <roadan@gmail.com>
Tue, 17 Jul 2018 09:23:10 +0000 (19:23 +1000)
committerYaniv Rodenski <roadan@gmail.com>
Tue, 17 Jul 2018 09:23:10 +0000 (19:23 +1000)
27 files changed:
frameworks/spark/dispatcher/build.gradle [new file with mode: 0644]
frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/SparkSetupProvider.scala [moved from leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala with 95% similarity]
frameworks/spark/runner/src/main/java/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkEntryPoint.java [moved from frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java with 96% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/SparkRunnersProvider.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala with 94% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkExecutionQueue.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala with 95% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkResultQueue.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala with 88% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkRunner.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala with 98% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/ResultQueue.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala with 95% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/AmaSparkILoop.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala with 95% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkRunnerHelper.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala with 99% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkScalaRunner.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala with 97% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/sparkr/SparkRRunner.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala with 96% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/sparksql/SparkSqlRunner.scala [moved from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala with 97% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/RunnersLoadingTests.scala [moved from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala with 96% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/SparkTestsSuite.scala [moved from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala with 90% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkRunnerTests.scala [moved from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala with 98% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkScalaRunnerTests.scala [moved from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala with 94% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/sparksql/SparkSqlRunnerTests.scala [moved from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala with 99% similarity]
frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/frameworks/spark/runtime/AmaContext.scala [moved from frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala with 97% similarity]
leader-common/build.gradle [new file with mode: 0644]
leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/DataLoader.scala [moved from leader/src/main/scala/org/apache/amaterasu/leader/utilities/DataLoader.scala with 97% similarity]
leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/MemoryFormatParser.scala [moved from leader/src/main/scala/org/apache/amaterasu/leader/utilities/MemoryFormatParser.scala with 95% similarity]
leader/build.gradle
leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
settings.gradle

diff --git a/frameworks/spark/dispatcher/build.gradle b/frameworks/spark/dispatcher/build.gradle
new file mode 100644 (file)
index 0000000..a95d958
--- /dev/null
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+    id 'com.github.johnrengelman.shadow' version '1.2.4'
+    id 'com.github.maiflai.scalatest' version '0.22'
+    id 'scala'
+}
+
+shadowJar {
+    zip64 true
+}
+
+repositories {
+    maven {
+        url "https://plugins.gradle.org/m2/"
+    }
+    mavenCentral()
+}
+
+dependencies {
+    compile 'org.scala-lang:scala-library:2.11.8'
+//    compile group: 'org.scala-lang', name: 'scala-reflect', version: '2.11.8'
+//    compile group: 'org.scala-lang', name: 'scala-compiler', version: '2.11.8'
+
+    compile project(':common')
+    compile project(':leader-common')
+    compile project(':amaterasu-sdk')
+}
+
+task copyToHomeBin(type: Copy) {
+    dependsOn shadowJar
+    from 'build/libs'
+    into '../../../build/amaterasu/bin'
+}
+
+task copyToHome() {
+    dependsOn copyToHomeBin
+}
\ No newline at end of file
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.leader.frameworks.spark
+package org.apache.amaterasu.frameworks.spark.dispatcher
 
 import java.io.File
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
-import org.apache.amaterasu.leader.utilities.{DataLoader, MemoryFormatParser}
+import org.apache.amaterasu.leader.common.utilities.{DataLoader, MemoryFormatParser}
 import org.apache.amaterasu.sdk.frameworks.FrameworkSetupProvider
 import org.apache.amaterasu.sdk.frameworks.configuration.DriverConfiguration
 
 import scala.collection.mutable
 
+import scala.collection.mutable
+
 class SparkSetupProvider extends FrameworkSetupProvider {
 
 
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runner.pyspark;
+package org.apache.amaterasu.frameworks.spark.runner.pyspark;
 
 import org.apache.amaterasu.common.runtime.Environment;
-import org.apache.amaterasu.framework.spark.runtime.AmaContext;
+import org.apache.amaterasu.frameworks.spark.runtime.AmaContext;
 import org.apache.spark.SparkConf;
 import org.apache.spark.SparkEnv;
 import org.apache.spark.api.java.JavaSparkContext;
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runner
+package org.apache.amaterasu.frameworks.spark.runner
 
 import java.io._
 
@@ -24,9 +24,10 @@ import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.execution.dependencies.{Dependencies, PythonDependencies, PythonPackage}
 import org.apache.amaterasu.common.logging.Logging
-import org.apache.amaterasu.framework.spark.runner.pyspark.PySparkRunner
-import org.apache.amaterasu.framework.spark.runner.repl.{SparkRunnerHelper, SparkScalaRunner}
-import org.apache.amaterasu.framework.spark.runner.sparksql.SparkSqlRunner
+import org.apache.amaterasu.frameworks.spark.runner.repl.{SparkRunnerHelper, SparkScalaRunner}
+import org.apache.amaterasu.frameworks.spark.runner.sparksql.SparkSqlRunner
+import org.apache.amaterasu.frameworks.spark.runner.pyspark.PySparkRunner
+import org.apache.amaterasu.frameworks.spark.runner.repl.{SparkRunnerHelper, SparkScalaRunner}
 import org.apache.amaterasu.sdk.{AmaterasuRunner, RunnersProvider}
 import org.eclipse.aether.util.artifact.JavaScopes
 import org.sonatype.aether.repository.RemoteRepository
@@ -14,9 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runner.pyspark
+package org.apache.amaterasu.frameworks.spark.runner.pyspark
 
-import org.apache.amaterasu.framework.spark.runner.pyspark.ResultType.ResultType
+import org.apache.amaterasu.frameworks.spark.runner.pyspark.ResultType.ResultType
 
 object ResultType extends Enumeration {
   type ResultType = Value
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runner.repl
+package org.apache.amaterasu.frameworks.spark.runner.repl
 
 import java.io.ByteArrayOutputStream
 import java.util
@@ -22,7 +22,7 @@ import java.util
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
-import org.apache.amaterasu.framework.spark.runtime.AmaContext
+import org.apache.amaterasu.frameworks.spark.runtime.AmaContext
 import org.apache.amaterasu.sdk.AmaterasuRunner
 import org.apache.spark.sql.{Dataset, SparkSession}
 
@@ -142,7 +142,7 @@ class SparkScalaRunner(var env: Environment,
     interpreter.interpret("import org.apache.spark.sql.SQLContext")
     interpreter.interpret("import org.apache.spark.sql.{ Dataset, SparkSession }")
     interpreter.interpret("import org.apache.spark.sql.SaveMode")
-    interpreter.interpret("import org.apache.amaterasu.framework.spark.runtime.AmaContext")
+    interpreter.interpret("import org.apache.amaterasu.frameworks.spark.runtime.AmaContext")
     interpreter.interpret("import org.apache.amaterasu.common.runtime.Environment")
 
     // creating a map (_contextStore) to hold the different spark contexts
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runner.sparksql
+package org.apache.amaterasu.frameworks.spark.runner.sparksql
 
 import java.io.File
 import java.util
@@ -22,7 +22,7 @@ import java.util
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
-import org.apache.amaterasu.framework.spark.runtime.AmaContext
+import org.apache.amaterasu.frameworks.spark.runtime.AmaContext
 import org.apache.amaterasu.sdk.AmaterasuRunner
 import org.apache.commons.io.FilenameUtils
 import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runner
+package org.apache.amaterasu.frameworks.spark.runner
 
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark
+package org.apache.amaterasu.frameworks.spark.runner
 
 import java.io.{ByteArrayOutputStream, File}
 
@@ -22,10 +22,9 @@ import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.common.execution.dependencies._
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
-import org.apache.amaterasu.framework.spark.runner.RunnersLoadingTests
-import org.apache.amaterasu.framework.spark.runner.pyspark.PySparkRunnerTests
-import org.apache.amaterasu.framework.spark.runner.repl.{SparkScalaRunner, SparkScalaRunnerTests}
-import org.apache.amaterasu.framework.spark.runner.sparksql.SparkSqlRunnerTests
+import org.apache.amaterasu.frameworks.spark.runner.pyspark.PySparkRunnerTests
+import org.apache.amaterasu.frameworks.spark.runner.repl.{SparkScalaRunner, SparkScalaRunnerTests}
+import org.apache.amaterasu.frameworks.spark.runner.sparksql.SparkSqlRunnerTests
 import org.apache.amaterasu.utilities.TestNotifier
 import org.apache.spark.sql.SparkSession
 import org.scalatest._
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runner.repl
+package org.apache.amaterasu.frameworks.spark.runner.repl
 
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
-import org.apache.amaterasu.framework.spark.runtime.AmaContext
+import org.apache.amaterasu.frameworks.spark.runtime.AmaContext
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FlatSpec, Matchers}
 
 import scala.collection.JavaConverters._
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runner.sparksql
+package org.apache.amaterasu.frameworks.spark.runner.sparksql
 
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.framework.spark.runtime
+package org.apache.amaterasu.frameworks.spark.runtime
 
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
diff --git a/leader-common/build.gradle b/leader-common/build.gradle
new file mode 100644 (file)
index 0000000..3f3ac98
--- /dev/null
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+    id "com.github.johnrengelman.shadow" version "1.2.4"
+    id 'com.github.maiflai.scalatest' version '0.22'
+    id 'scala'
+    id 'java'
+}
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+    maven {
+        url "https://plugins.gradle.org/m2/"
+    }
+    mavenCentral()
+}
+
+dependencies {
+    compile 'org.scala-lang:scala-library:2.11.8'
+
+    compile project(':common')
+
+    compile group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.11', version: '2.6.3'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.6.4'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.6.4'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.6.4'
+    compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.6.4'
+}
\ No newline at end of file
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.leader.utilities
+package org.apache.amaterasu.leader.common.utilities
 
 import java.io.{File, FileInputStream}
 import java.nio.file.{Files, Paths}
index 3f9276e..a0de6f5 100644 (file)
@@ -34,9 +34,9 @@ repositories {
 dependencies {
     compile 'org.scala-lang:scala-library:2.11.8'
     compile group: 'org.scala-lang', name: 'scala-reflect', version: '2.11.8'
-//    compile group: 'org.scala-lang', name: 'scala-compiler', version: '2.11.8'
 
     compile project(':common')
+    compile project(':leader-common')
     compile project(':amaterasu-sdk')
 
     compile group: 'com.github.scopt', name: 'scopt_2.11', version: '3.3.0'
@@ -52,7 +52,6 @@ dependencies {
     compile group: 'org.eclipse.jetty', name: 'jetty-http', version: '9.2.19.v20160908'
     compile group: 'org.eclipse.jetty', name: 'jetty-io', version: '9.2.19.v20160908'
     compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: '9.2.19.v20160908'
-    //compile group: 'javax.servlet', name: 'javax.servlet-api', version: '3.1.0'
     compile group: 'org.eclipse.jetty.toolchain', name: 'jetty-test-helper', version: '4.0'
     compile group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '4.2.0.201601211800-r'
     compile group: 'org.yaml', name: 'snakeyaml', version: '1.18'
index 19b7ec3..1bb82ff 100755 (executable)
@@ -29,9 +29,10 @@ import org.apache.amaterasu.common.configuration.enums.ActionStatus.ActionStatus
 import org.apache.amaterasu.common.dataobjects.ActionData
 import org.apache.amaterasu.common.execution.actions.NotificationLevel.NotificationLevel
 import org.apache.amaterasu.common.execution.actions.{Notification, NotificationLevel, NotificationType}
+import org.apache.amaterasu.leader.common.utilities.DataLoader
 import org.apache.amaterasu.leader.execution.frameworks.FrameworkProvidersFactory
 import org.apache.amaterasu.leader.execution.{JobLoader, JobManager}
-import org.apache.amaterasu.leader.utilities.{DataLoader, HttpServer}
+import org.apache.amaterasu.leader.utilities.HttpServer
 import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
 import org.apache.curator.retry.ExponentialBackoffRetry
 import org.apache.log4j.LogManager
index 406c150..c94a3ba 100644 (file)
@@ -21,16 +21,17 @@ import java.net.{InetAddress, ServerSocket, URLEncoder}
 import java.nio.ByteBuffer
 import java.util
 import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue}
-
 import javax.jms.Session
+
 import org.apache.activemq.ActiveMQConnectionFactory
 import org.apache.activemq.broker.BrokerService
 import org.apache.amaterasu.common.configuration.ClusterConfig
 import org.apache.amaterasu.common.dataobjects.ActionData
 import org.apache.amaterasu.common.logging.Logging
+import org.apache.amaterasu.leader.common.utilities.DataLoader
 import org.apache.amaterasu.leader.execution.frameworks.FrameworkProvidersFactory
 import org.apache.amaterasu.leader.execution.{JobLoader, JobManager}
-import org.apache.amaterasu.leader.utilities.{ActiveReportListener, Args, DataLoader}
+import org.apache.amaterasu.leader.utilities.{ActiveReportListener, Args}
 import org.apache.curator.framework.recipes.barriers.DistributedBarrier
 import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
 import org.apache.curator.retry.ExponentialBackoffRetry
index b178f52..24f28cc 100644 (file)
@@ -23,8 +23,8 @@ import java.util.concurrent.ConcurrentHashMap
 import com.google.gson.Gson
 import org.apache.amaterasu.common.configuration.ClusterConfig
 import org.apache.amaterasu.common.logging.Logging
+import org.apache.amaterasu.leader.common.utilities.DataLoader
 import org.apache.amaterasu.leader.execution.JobManager
-import org.apache.amaterasu.leader.utilities.DataLoader
 import org.apache.hadoop.yarn.api.records._
 import org.apache.hadoop.yarn.client.api.async.{AMRMClientAsync, NMClientAsync}
 import org.apache.hadoop.yarn.util.Records
index c222795..9343001 100644 (file)
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+// Core
 include 'leader'
 project(':leader')
 
+include 'leader-common'
+project(':leader-common')
+
 include 'common'
 project(':common')
 
@@ -26,9 +31,13 @@ project(':executor')
 include 'sdk'
 findProject(':sdk')?.name = 'amaterasu-sdk'
 
-//Spark
+// Frameworks
+// Spark
 include 'spark-runner'
 project(':spark-runner').projectDir=file("frameworks/spark/runner")
 include 'spark-runtime'
 project(':spark-runtime').projectDir=file("frameworks/spark/runtime")
+include 'spark-dispatcher'
+project(':spark-dispatcher').projectDir=file("frameworks/spark/dispatcher")
+