fixed haddop classpath
authorYaniv Rodenski <roadan@gmail.com>
Sun, 22 Jul 2018 07:56:27 +0000 (17:56 +1000)
committerYaniv Rodenski <roadan@gmail.com>
Sun, 22 Jul 2018 07:56:27 +0000 (17:56 +1000)
leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
leader/src/main/scripts/ama-start-yarn.sh

index 8f16ee7..0dd9973 100644 (file)
@@ -26,21 +26,10 @@ import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.framework.recipes.barriers.DistributedBarrier;
 import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.*;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.*;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -52,19 +41,11 @@ import org.apache.log4j.LogManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.jms.Connection;
-import javax.jms.JMSException;
-import javax.jms.MessageConsumer;
-import javax.jms.Session;
-import javax.jms.Topic;
+import javax.jms.*;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
 
 import static java.lang.System.exit;
 
@@ -148,6 +129,7 @@ public class Client {
         // Setup local ama folder on hdfs.
         try {
 
+            System.out.println("===> " + jarPathQualified);
             if (!fs.exists(jarPathQualified)) {
                 File home = new File(opts.home);
                 fs.mkdirs(jarPathQualified);
@@ -157,6 +139,7 @@ public class Client {
                 }
 
                 // setup frameworks
+                System.out.println("===> setting up frameworks");
                 FrameworkProvidersFactory frameworkFactory = FrameworkProvidersFactory.apply(opts.env, config);
                 for (String group : frameworkFactory.groups()) {
                     System.out.println("===> setting up " + group);
@@ -208,7 +191,15 @@ public class Client {
 
         // set local resource on master container
         Map<String, LocalResource> localResources = new HashMap<>();
-        localResources.put("leader.jar", leaderJar);
+        //localResources.put("leader.jar", leaderJar);
+        // making the bin folder's content available to the appMaster
+        RemoteIterator<LocatedFileStatus> bin = fs.listFiles(Path.mergePaths(jarPath, new Path("/bin")), true);
+
+        while (bin.hasNext()){
+            LocatedFileStatus binFile = bin.next();
+            localResources.put(binFile.getPath().getName(), setLocalResourceFromPath(binFile.getPath()));
+        }
+
         localResources.put("amaterasu.properties", propFile);
         localResources.put("log4j.properties", log4jPropFile);
         amContainer.setLocalResources(localResources);
index f6af18f..fe14f6e 100755 (executable)
@@ -102,6 +102,8 @@ done
 echo "repo: ${REPO} "
 echo "force-bin: ${FORCE_BIN}"
 export HADOOP_USER_CLASSPATH_FIRST=true
+export YARN_USER_CLASSPATH=${YARN_USER_CLASSPATH}:bin/*
+
 CMD="yarn jar ${BASEDIR}/bin/leader-0.2.0-incubating-rc4-all.jar org.apache.amaterasu.leader.yarn.Client --home ${BASEDIR}"
 
 if [ -n "$REPO" ]; then