AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own project (Master)
authorArun Manivannan <arun@arunma.com>
Sun, 1 Jul 2018 07:31:53 +0000 (15:31 +0800)
committerArun Manivannan <arun@arunma.com>
Sun, 1 Jul 2018 07:31:54 +0000 (15:31 +0800)
commit47e1c112de45b20483426363e1daae0080eb770f
treeb2e4b25b8a685cdadee27a043954a4474ec20694
parent172f60add078466ac3d99af3adf109633db3967b
AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own project (Master)
71 files changed:
.gitignore
build.gradle
common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala
executor/build.gradle
executor/src/main/scala/org/apache/amaterasu/executor/mesos/executors/MesosActionsExecutor.scala
executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/ActionsExecutor.scala
frameworks/spark/runner/build.gradle [new file with mode: 0644]
frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java [moved from executor/src/main/java/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkEntryPoint.java with 92% similarity]
frameworks/spark/runner/src/main/resources/codegen.py [moved from executor/src/test/resources/codegen.py with 100% similarity]
frameworks/spark/runner/src/main/resources/runtime.py [moved from executor/src/test/resources/runtime.py with 100% similarity]
frameworks/spark/runner/src/main/resources/spark-version-info.properties [new file with mode: 0644]
frameworks/spark/runner/src/main/resources/spark_intp.py [moved from executor/src/main/resources/spark_intp.py with 100% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRunnersProvider.scala with 93% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkExecutionQueue.scala with 94% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkResultQueue.scala with 85% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkRunner.scala with 95% similarity, mode: 0644]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/ResultQueue.scala with 94% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala [moved from executor/src/main/scala/org/apache/spark/repl/amaterasu/AmaSparkILoop.scala with 95% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala [moved from executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala with 94% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala [moved from executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkScalaRunner.scala with 97% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRRunner.scala with 69% similarity]
frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkSql/SparkSqlRunner.scala with 96% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv [moved from executor/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/json/SparkSqlTestData.json [moved from executor/src/test/resources/SparkSql/json/SparkSqlTestData.json with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc [moved from executor/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc [moved from executor/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/_SUCCESS [moved from executor/src/test/resources/SparkSql/parquet/_SUCCESS with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/_common_metadata [moved from executor/src/test/resources/SparkSql/parquet/_common_metadata with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/_metadata [moved from executor/src/test/resources/SparkSql/parquet/_metadata with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet [moved from executor/src/test/resources/tmp/job/start/x/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet with 100% similarity]
frameworks/spark/runner/src/test/resources/SparkSql/parquet/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet [moved from executor/src/test/resources/tmp/job/start/x/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet with 100% similarity]
frameworks/spark/runner/src/test/resources/amaterasu.properties [moved from executor/src/test/resources/amaterasu.properties with 100% similarity]
frameworks/spark/runner/src/test/resources/codegen.py [moved from executor/src/main/resources/codegen.py with 100% similarity]
frameworks/spark/runner/src/test/resources/py4j-0.10.4-src.zip [moved from executor/src/test/resources/py4j-0.10.4-src.zip with 100% similarity]
frameworks/spark/runner/src/test/resources/py4j.tar.gz [moved from executor/src/test/resources/py4j.tar.gz with 100% similarity]
frameworks/spark/runner/src/test/resources/pyspark-with-amacontext.py [moved from executor/src/test/resources/pyspark-with-amacontext.py with 100% similarity]
frameworks/spark/runner/src/test/resources/pyspark.tar.gz [moved from executor/src/test/resources/pyspark.tar.gz with 100% similarity]
frameworks/spark/runner/src/test/resources/pyspark.zip [moved from executor/src/test/resources/pyspark.zip with 100% similarity]
frameworks/spark/runner/src/test/resources/runtime.py [moved from executor/src/main/resources/runtime.py with 100% similarity]
frameworks/spark/runner/src/test/resources/simple-pyspark.py [moved from executor/src/test/resources/simple-pyspark.py with 100% similarity]
frameworks/spark/runner/src/test/resources/simple-python-err.py [moved from executor/src/test/resources/simple-python-err.py with 100% similarity]
frameworks/spark/runner/src/test/resources/simple-python.py [moved from executor/src/test/resources/simple-python.py with 100% similarity]
frameworks/spark/runner/src/test/resources/simple-spark.scala [moved from executor/src/test/resources/simple-spark.scala with 83% similarity]
frameworks/spark/runner/src/test/resources/spark-version-info.properties [moved from executor/src/main/resources/spark-version-info.properties with 100% similarity]
frameworks/spark/runner/src/test/resources/spark_intp.py [moved from executor/src/test/resources/spark_intp.py with 100% similarity]
frameworks/spark/runner/src/test/resources/step-2.scala [moved from executor/src/test/resources/step-2.scala with 94% similarity]
frameworks/spark/runner/src/test/resources/tmp/job/start/x/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet [moved from executor/src/test/resources/SparkSql/parquet/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet with 100% similarity]
frameworks/spark/runner/src/test/resources/tmp/job/start/x/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet [moved from executor/src/test/resources/SparkSql/parquet/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet with 100% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala [moved from executor/src/test/scala/org/apache/amaterasu/spark/SparkTestsSuite.scala with 90% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala [moved from executor/src/test/scala/org/apache/amaterasu/RunnersTests/RunnersLoadingTests.scala with 96% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala [moved from executor/src/test/scala/org/apache/amaterasu/spark/PySparkRunnerTests.scala with 96% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala [moved from executor/src/test/scala/org/apache/amaterasu/spark/SparkScalaRunnerTests.scala with 92% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala [moved from executor/src/test/scala/org/apache/amaterasu/spark/SparkSqlRunnerTests.scala with 96% similarity]
frameworks/spark/runner/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala [moved from executor/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala with 100% similarity]
frameworks/spark/runtime/build.gradle [new file with mode: 0644]
frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala [moved from executor/src/main/scala/org/apache/amaterasu/executor/runtime/AmaContext.scala with 93% similarity, mode: 0644]
leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java
leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
leader/src/main/scala/org/apache/amaterasu/leader/dsl/JobParser.scala
leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/AmaterasuScheduler.scala
leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
leader/src/main/scala/org/apache/amaterasu/leader/utilities/ActiveReportListener.scala
leader/src/main/scala/org/apache/amaterasu/leader/utilities/HttpServer.scala
leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
leader/src/main/scripts/ama-start-mesos.sh
leader/src/main/scripts/ama-start-yarn.sh
leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala
leader/src/test/scala/org/apache/amaterasu/utilities/HttpServerTests.scala
settings.gradle