fixing review comments
authorNadav Har Tzvi <nadavha@apache.org>
Sun, 27 May 2018 05:54:42 +0000 (08:54 +0300)
committerNadav Har Tzvi <nadavha@apache.org>
Sun, 27 May 2018 05:54:42 +0000 (08:54 +0300)
executor/src/main/resources/spark_intp.py
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRunnersProvider.scala
executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala
executor/src/test/resources/spark_intp.py

index 16e5e10..f3c9fc0 100755 (executable)
@@ -24,7 +24,18 @@ import zipimport
 sys.path.append(os.getcwd())
 from runtime import AmaContext, Environment
 
+# os.chdir(os.getcwd() + '/build/resources/test/')
+# import zipfile
+# zip = zipfile.ZipFile('pyspark.zip')
+# zip.extractall()
+# zip = zipfile.ZipFile('py4j-0.10.4-src.zip', 'r')
+# zip.extractall()
+# sys.path.insert(1, os.getcwd() + '/executor/src/test/resources/pyspark')
+# sys.path.insert(1, os.getcwd() + '/executor/src/test/resources/py4j')
 
+# py4j_path = 'spark-2.2.1-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip'
+# py4j_importer = zipimport.zipimporter(py4j_path)
+# py4j = py4j_importer.load_module('py4j')
 from py4j.java_gateway import JavaGateway, GatewayClient, java_import
 from py4j.protocol import Py4JJavaError
 from pyspark.conf import SparkConf
index ba7ff03..0644999 100644 (file)
@@ -88,7 +88,7 @@ class SparkRunnersProvider extends RunnersProvider with Logging {
     // TODO: get rid of hard-coded version
     config.mode match {
       case "yarn" =>
-        pypath = s"$$PYTHONPATH:$$SPARK_HOME/python:$$SPARK_HOME/python/build:${config.spark.home}/python:${config.spark.home}/python/pyspark:${config.spark.home}/python/pyspark/build:${config.spark.home}/python/pyspark/lib/py4j-0.10.4-src.zip:${new File(".").getAbsolutePath}"
+        pypath = s"$$PYTHONPATH:$$SPARK_HOME/python:$$SPARK_HOME/python/build:${new File(".").getAbsolutePath}"
       case "mesos" =>
         pypath = s"${new File(".").getAbsolutePath}/miniconda/pkgs:${new File(".").getAbsolutePath}"
     }
index ce9c66b..f2c2afa 100644 (file)
@@ -152,7 +152,7 @@ object SparkRunnerHelper extends Logging {
           .set("spark.history.kerberos.principal", "none")
 
           .set("spark.master", master)
-          .set("spark.executor.instances", "1") // TODO: change this
+          .set("spark.executor.instances", config.spark.opts.getOrElse("executor.instances", "1"))
           .set("spark.yarn.jars", s"spark/jars/*")
           .set("spark.executor.memory", config.spark.opts.getOrElse("executor.memory", "1g"))
           .set("spark.dynamicAllocation.enabled", "false")
index a427e92..fd8dc0e 100755 (executable)
@@ -31,6 +31,7 @@ zip = zipfile.ZipFile('py4j-0.10.4-src.zip', 'r')
 zip.extractall()
 sys.path.insert(1, os.getcwd() + '/executor/src/test/resources/pyspark')
 sys.path.insert(1, os.getcwd() + '/executor/src/test/resources/py4j')
+sys.path.append(os.getcwd())
 
 # py4j_path = 'spark-2.2.1-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip'
 # py4j_importer = zipimport.zipimporter(py4j_path)