increased mem for yarn jobs 31/head
authorYaniv Rodenski <roadan@gmail.com>
Mon, 23 Jul 2018 10:00:32 +0000 (20:00 +1000)
committerYaniv Rodenski <roadan@gmail.com>
Mon, 23 Jul 2018 10:00:32 +0000 (20:00 +1000)
leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
leader/src/main/scripts/amaterasu.properties

index 0dd9973..5e86188 100644 (file)
@@ -113,7 +113,7 @@ public class Client {
                         " env HADOOP_USER_NAME=" + UserGroupInformation.getCurrentUser().getUserName() +
                         " $JAVA_HOME/bin/java" +
                         " -Dscala.usejavacp=false" +
-                        " -Xmx1G" +
+                        " -Xmx2G" +
                         " org.apache.amaterasu.leader.yarn.ApplicationMaster " +
                         joinStrings(args) +
                         newId +
index c94a3ba..1f1aa25 100644 (file)
@@ -252,7 +252,7 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging {
           "/bin/bash ./miniconda.sh -b -p $PWD/miniconda && ",
           s"/bin/bash spark/bin/load-spark-env.sh && ",
           s"java -cp spark/jars/*:executor.jar:spark-runner.jar:spark-runtime.jar:spark/conf/:${config.YARN.hadoopHomeDir}/conf/ " +
-            "-Xmx1G " +
+            "-Xmx2G " +
             "-Dscala.usejavacp=true " +
             "-Dhdp.version=2.6.1.0-129 " +
             "org.apache.amaterasu.executor.yarn.executors.ActionsExecutorLauncher " +
index 0bd7afd..5c55d79 100755 (executable)
@@ -22,8 +22,10 @@ webserver.root=dist
 spark.version=2.2.1-bin-hadoop2.7
 yarn.queue=default
 yarn.jarspath=hdfs:///apps/amaterasu
-spark.home=/usr/hdp/current/spark2-client
+spark.home=/usr/lib/spark
 #spark.home=/opt/cloudera/parcels/SPARK2-2.1.0.cloudera2-1.cdh5.7.0.p0.171658/lib/spark2
 yarn.hadoop.home.dir=/etc/hadoop
 spark.opts.spark.yarn.am.extraJavaOptions="-Dhdp.version=2.6.1.0-129"
 spark.opts.spark.driver.extraJavaOptions="-Dhdp.version=2.6.1.0-129"
+yarn.master.memoryMB=2048
+yarn.worker.memoryMB=2048
\ No newline at end of file