apparently we need the previous config 20/head
authorNadav Har Tzvi <nadavha@apache.org>
Sun, 27 May 2018 06:43:13 +0000 (09:43 +0300)
committerNadav Har Tzvi <nadavha@apache.org>
Sun, 27 May 2018 06:43:13 +0000 (09:43 +0300)
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRunnersProvider.scala

index 0644999..ba7ff03 100644 (file)
@@ -88,7 +88,7 @@ class SparkRunnersProvider extends RunnersProvider with Logging {
     // TODO: get rid of hard-coded version
     config.mode match {
       case "yarn" =>
-        pypath = s"$$PYTHONPATH:$$SPARK_HOME/python:$$SPARK_HOME/python/build:${new File(".").getAbsolutePath}"
+        pypath = s"$$PYTHONPATH:$$SPARK_HOME/python:$$SPARK_HOME/python/build:${config.spark.home}/python:${config.spark.home}/python/pyspark:${config.spark.home}/python/pyspark/build:${config.spark.home}/python/pyspark/lib/py4j-0.10.4-src.zip:${new File(".").getAbsolutePath}"
       case "mesos" =>
         pypath = s"${new File(".").getAbsolutePath}/miniconda/pkgs:${new File(".").getAbsolutePath}"
     }