[VXQUERY-180] REST Server implementation 172/head
authorerandiganepola <erandiganepola@gmail.com>
Sun, 4 Jun 2017 09:02:29 +0000 (14:32 +0530)
committerErandi Ganepola <erandiganepola@gmail.com>
Fri, 25 Aug 2017 01:25:23 +0000 (06:55 +0530)
- Implemented REST API
- Altered CLI to use REST API
- Migrated XTests to use REST API

details:
- Implemented the REST Server to start through the cluster controller
  application.
- CLI now calls the REST API (remote if given, local one else) to
  execute queries.
- Migrated XTests to use the REST API to run queries related to tests

73 files changed:
pom.xml
src/site/apt/user_query.apt
vxquery-cli/pom.xml
vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
vxquery-cli/src/site/markdown/index.md [new file with mode: 0644]
vxquery-cli/src/site/site.xml
vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
vxquery-rest/pom.xml [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java [new file with mode: 0644]
vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java [new file with mode: 0644]
vxquery-rest/src/site/markdown/index.md [new file with mode: 0644]
vxquery-rest/src/site/markdown/specification.md [new file with mode: 0644]
vxquery-rest/src/site/site.xml [new file with mode: 0644]
vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java [new file with mode: 0644]
vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java [new file with mode: 0644]
vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java [new file with mode: 0644]
vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java [new file with mode: 0644]
vxquery-rest/src/test/resources/vxquery.properties [new file with mode: 0644]
vxquery-server/pom.xml
vxquery-xtest/pom.xml
vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java
vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml

diff --git a/pom.xml b/pom.xml
index 5f03a49..7e298c5 100644 (file)
--- a/pom.xml
+++ b/pom.xml
 
             <dependency>
                 <groupId>org.apache.hyracks</groupId>
+                <artifactId>hyracks-http</artifactId>
+                <version>${hyracks.fullstack.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty-all</artifactId>
+                <version>4.1.6.Final</version>
+            </dependency>
+            
+            <dependency>
+                <groupId>org.apache.hyracks</groupId>
                 <artifactId>algebricks-compiler</artifactId>
                 <version>${hyracks.version}</version>
             </dependency>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <hyracks.version>0.2.17-incubating</hyracks.version>
+        <hyracks.fullstack.version>0.3.1</hyracks.fullstack.version>
         <hyracks.version>0.3.0</hyracks.version>
         <apache-rat-plugin.version>0.11</apache-rat-plugin.version>
     </properties>
         <module>vxquery-cli</module>
         <module>vxquery-xtest</module>
         <module>vxquery-benchmark</module>
+        <module>vxquery-rest</module>
     </modules>
 </project>
index c5132c3..f4d0dd5 100644 (file)
@@ -35,21 +35,26 @@ vxq "path-to"\test.xq
   Command line options for all systems.
 
 ----------------------------------------
--O N                       : Optimization Level. Default: Full Optimization
--available-processors N    : Number of available processors. (default java's available processors)
--client-net-ip-address VAL : IP Address of the ClusterController
--client-net-port N         : Port of the ClusterController (default 1098)
--compileonly               : Compile the query and stop
--frame-size N              : Frame size in bytes. (default 65536)
--local-node-controllers N  : Number of local node controllers (default 1)
--repeatexec N              : Number of times to repeat execution
--showast                   : Show abstract syntax tree
--showoet                   : Show optimized expression tree
--showquery                 : Show query string
--showrp                    : Show Runtime plan
--showtet                   : Show translated expression tree
--timing                    : Produce timing information
--hdfs-conf VAL             : The folder containing the HDFS configuration files
+ -O N                      : Optimization Level. (default: Full Optimization)
+ -available-processors N   : Number of available processors. (default: java's available processors)
+ -buffer-size N            : Disk read buffer size in bytes.
+ -compileonly              : Compile the query and stop.
+ -frame-size N             : Frame size in bytes. (default: 65,536)
+ -hdfs-conf VAL            : Directory path to Hadoop configuration files
+ -join-hash-size N         : Join hash size in bytes. (default: 67,108,864)
+ -local-node-controllers N : Number of local node controllers. (default: 1)
+ -maximum-data-size N      : Maximum possible data size in bytes. (default: 150,323,855,000)
+ -repeatexec N             : Number of times to repeat execution.
+ -rest-ip-address VAL      : IP Address of the REST Server.
+ -rest-port N              : Port of REST Server.
+ -result-file VAL          : File path to save the query result.
+ -showast                  : Show abstract syntax tree.
+ -showoet                  : Show optimized expression tree.
+ -showquery                : Show query string.
+ -showrp                   : Show Runtime plan.
+ -showtet                  : Show translated expression tree.
+ -timing                   : Produce timing information.
+ -timing-ignore-queries N  : Ignore the first X number of quereies.
 ----------------------------------------
 
 * Java Options
index 52c32ad..e7ad16a 100644 (file)
@@ -14,7 +14,8 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
                     </execution>
                 </executions>
             </plugin>
-      <!--
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-site-plugin</artifactId>
-      </plugin>
-      -->
             <plugin>
                 <artifactId>maven-antrun-plugin</artifactId>
                 <executions>
     <dependencies>
         <dependency>
             <groupId>org.apache.vxquery</groupId>
-            <artifactId>apache-vxquery-core</artifactId>
+            <artifactId>apache-vxquery-rest</artifactId>
             <version>0.7-SNAPSHOT</version>
-            <scope>compile</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hyracks</groupId>
-            <artifactId>hyracks-api</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hyracks</groupId>
-            <artifactId>hyracks-client</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hyracks</groupId>
-            <artifactId>hyracks-control-cc</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hyracks</groupId>
-            <artifactId>hyracks-control-nc</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hyracks</groupId>
-            <artifactId>algebricks-common</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hyracks</groupId>
-            <artifactId>algebricks-core</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hyracks</groupId>
-            <artifactId>hyracks-control-common</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hyracks</groupId>
-            <artifactId>hyracks-dataflow-std</artifactId>
-        </dependency>
-
-        <dependency>
-                <groupId>org.apache.hyracks</groupId>
-                <artifactId>hyracks-hdfs-core</artifactId>
-        </dependency>
-
-        <dependency>
-                <groupId>org.apache.hyracks</groupId>
-                <artifactId>hyracks-hdfs-2.x</artifactId>
         </dependency>
     </dependencies>
 
-
-
     <reporting>
         <plugins>
             <plugin>
index 25ff9c4..23d5eaa 100644 (file)
  */
 package org.apache.vxquery.cli;
 
-import java.io.ByteArrayOutputStream;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.io.StringReader;
-import java.net.InetAddress;
-import java.nio.file.Files;
+import java.io.PrintStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
+import java.util.logging.LogManager;
+
+import javax.xml.bind.JAXBException;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.client.HyracksConnection;
-import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.client.NodeControllerInfo;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.job.JobFlag;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.client.dataset.HyracksDataset;
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.common.controllers.CCConfig;
-import org.apache.hyracks.control.common.controllers.NCConfig;
-import org.apache.hyracks.control.nc.NodeControllerService;
-import org.apache.hyracks.control.nc.resources.memory.FrameManager;
-import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
-import org.apache.vxquery.compiler.CompilerControlBlock;
-import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory;
-import org.apache.vxquery.context.DynamicContext;
-import org.apache.vxquery.context.DynamicContextImpl;
-import org.apache.vxquery.context.RootStaticContextImpl;
-import org.apache.vxquery.context.StaticContextImpl;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.result.ResultUtils;
-import org.apache.vxquery.xmlquery.query.Module;
-import org.apache.vxquery.xmlquery.query.VXQueryCompilationListener;
-import org.apache.vxquery.xmlquery.query.XMLQueryCompiler;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.vxquery.app.util.LocalClusterUtil;
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.response.AsyncQueryResponse;
+import org.apache.vxquery.rest.response.Error;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.apache.vxquery.rest.response.Metrics;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.rest.service.VXQueryConfig;
 import org.kohsuke.args4j.Argument;
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
+/**
+ * CLI for VXQuery. This class is using the REST API to execute statements given by the user.
+ *
+ * @author Erandi Ganepola
+ */
 public class VXQuery {
+
     private final CmdLineOptions opts;
-    private final CmdLineOptions indexOpts;
-
-    private ClusterControllerService cc;
-    private NodeControllerService[] ncs;
-    private IHyracksClientConnection hcc;
-    private IHyracksDataset hds;
-    private List<String> collectionList;
-    private ResultSetId resultSetId;
-    private static List<String> timingMessages = new ArrayList<>();
-    private static long sumTiming;
-    private static long sumSquaredTiming;
-    private static long minTiming = Long.MAX_VALUE;
-    private static long maxTiming = Long.MIN_VALUE;
+
+    private static LocalClusterUtil localClusterUtil;
+    private String restIpAddress;
+    private int restPort;
+
+    private static List<Metrics> metricsList = new ArrayList<>();
+    private int executionIteration;
 
     /**
      * Constructor to use command line options passed.
@@ -90,26 +74,17 @@ public class VXQuery {
      */
     public VXQuery(CmdLineOptions opts) {
         this.opts = opts;
-        // The index query returns only the result, without any other information.
-        this.indexOpts = opts;
-        indexOpts.showAST = false;
-        indexOpts.showOET = false;
-        indexOpts.showQuery = false;
-        indexOpts.showRP = false;
-        indexOpts.showTET = false;
-        indexOpts.timing = false;
-        indexOpts.compileOnly = false;
-        this.collectionList = new ArrayList<String>();
     }
 
     /**
      * Main method to get command line options and execute query process.
      *
      * @param args
-     * @throws Exception
+     *            command line arguments
      */
-    public static void main(String[] args) throws Exception {
-        Date start = new Date();
+    public static void main(String[] args) {
+        LogManager.getLogManager().reset();
+
         final CmdLineOptions opts = new CmdLineOptions();
         CmdLineParser parser = new CmdLineParser(opts);
 
@@ -120,243 +95,249 @@ public class VXQuery {
             parser.printUsage(System.err);
             return;
         }
-        if (opts.arguments.isEmpty()) {
+
+        if (opts.xqFiles.isEmpty()) {
             parser.printUsage(System.err);
             return;
         }
-        VXQuery vxq = new VXQuery(opts);
-        vxq.execute();
-        // if -timing argument passed, show the starting and ending times
-        if (opts.timing) {
-            Date end = new Date();
-            timingMessage("Execution time: " + (end.getTime() - start.getTime()) + " ms");
-            if (opts.repeatExec > opts.timingIgnoreQueries) {
-                Double mean = (double) (sumTiming) / (opts.repeatExec - opts.timingIgnoreQueries);
-                double sd = Math.sqrt(sumSquaredTiming / (opts.repeatExec - opts.timingIgnoreQueries) - mean * mean);
-                timingMessage("Average execution time: " + mean + " ms");
-                timingMessage("Standard deviation: " + String.format("%.4f", sd));
-                timingMessage("Coefficient of variation: " + String.format("%.4f", sd / mean));
-                timingMessage("Minimum execution time: " + minTiming + " ms");
-                timingMessage("Maximum execution time: " + maxTiming + " ms");
-            }
-            System.out.println("Timing Summary:");
-            for (String time : timingMessages) {
-                System.out.println("  " + time);
-            }
-        }
 
+        VXQuery vxq = new VXQuery(opts);
+        vxq.execute(opts.xqFiles);
     }
 
-    /**
-     * Creates a new Hyracks connection with: the client IP address and port provided, if IP address is provided in command line. Otherwise create a new virtual
-     * cluster with Hyracks nodes. Queries passed are run either way. After running queries, if a virtual cluster has been created, it is shut down.
-     *
-     * @throws Exception
-     */
-    private void execute() throws Exception {
-        System.setProperty("vxquery.buffer_size", Integer.toString(opts.bufferSize));
+    private void execute(List<String> xqFiles) {
+        if (opts.restIpAddress == null) {
+            System.out.println("No REST Ip address given. Creating a local hyracks cluster");
 
-        if (opts.clientNetIpAddress != null) {
-            hcc = new HyracksConnection(opts.clientNetIpAddress, opts.clientNetPort);
-            runQueries();
-        } else {
-            if (!opts.compileOnly) {
-                startLocalHyracks();
+            VXQueryConfig vxqConfig = new VXQueryConfig();
+            vxqConfig.setAvailableProcessors(opts.availableProcessors);
+            vxqConfig.setFrameSize(opts.frameSize);
+            vxqConfig.setHdfsConf(opts.hdfsConf);
+            vxqConfig.setJoinHashSize(opts.joinHashSize);
+            vxqConfig.setMaximumDataSize(opts.maximumDataSize);
+
+            localClusterUtil = new LocalClusterUtil();
+            try {
+                localClusterUtil.init(vxqConfig);
+                restIpAddress = localClusterUtil.getIpAddress();
+                restPort = localClusterUtil.getRestPort();
+            } catch (Exception e) {
+                System.err.println("Unable to start local hyracks cluster due to: " + e.getMessage());
+                e.printStackTrace();
+                return;
             }
+        } else {
+            restIpAddress = opts.restIpAddress;
+            restPort = opts.restPort;
+        }
+
+        System.out.println("Running queries given in: " + Arrays.toString(xqFiles.toArray()));
+        runQueries(xqFiles);
+
+        if (localClusterUtil != null) {
             try {
-                runQueries();
-            } finally {
-                if (!opts.compileOnly) {
-                    stopLocalHyracks();
-                }
+                localClusterUtil.deinit();
+            } catch (Exception e) {
+                System.err.println("Error occurred when stopping local hyracks: " + e.getMessage());
             }
         }
     }
 
-    /**
-     * Reads the contents of the files passed in the list of arguments to a string. If -showquery argument is passed, output the query as string. Run the query
-     * for the string.
-     *
-     * @throws IOException
-     * @throws SystemException
-     * @throws Exception
-     */
+    public void runQueries(List<String> xqFiles) {
+        for (String xqFile : xqFiles) {
+            String query;
+            try {
+                query = slurp(xqFile);
+            } catch (IOException e) {
+                System.err.println(String.format("Error occurred when reading XQuery file %s with message: %s", xqFile,
+                        e.getMessage()));
+                continue;
+            }
 
-    private void runQueries() throws Exception {
-        List<String> queries = opts.arguments;
-        // Run the showIndexes query before executing any target query, to store the index metadata
-        List<String> queriesIndex = new ArrayList<String>();
-        queriesIndex.add("vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndexes.xq");
-        OutputStream resultStream = new ByteArrayOutputStream();
-        executeQuery(queriesIndex.get(0), 1, resultStream, indexOpts);
-        ByteArrayOutputStream bos = (ByteArrayOutputStream) resultStream;
-        String result = new String(bos.toByteArray());
-        String[] collections = result.split("\n");
-        this.collectionList = Arrays.asList(collections);
-        executeQueries(queries);
-    }
+            System.out.println();
+            System.out.println("====================================================");
+            System.out.println("\tQuery - '" + xqFile + "'");
+            System.out.println("====================================================");
 
-    public void executeQueries(List<String> queries) throws Exception {
-        for (String query : queries) {
-            OutputStream resultStream = System.out;
-            if (opts.resultFile != null) {
-                resultStream = new FileOutputStream(new File(opts.resultFile));
+            QueryRequest request = createQueryRequest(opts, query);
+            metricsList.clear();
+
+            for (int i = 0; i < opts.repeatExec; i++) {
+                System.out.println("**** Repetition : " + (i + 1) + " ****");
+
+                executionIteration = i;
+                sendQueryRequest(xqFile, request, this);
+            }
+
+            if (opts.repeatExec > 1) {
+                showTimingSummary();
             }
-            executeQuery(query, opts.repeatExec, resultStream, opts);
         }
     }
 
-    public void executeQuery(String query, int repeatedExecution, OutputStream resultStream, CmdLineOptions options)
-            throws Exception {
-        PrintWriter writer = new PrintWriter(resultStream, true);
-        String qStr = slurp(query);
-        if (opts.showQuery) {
-            writer.println(qStr);
+    private void onSuccess(String xqFile, QueryRequest request, SyncQueryResponse response) {
+        if (response == null) {
+            System.err.println(String.format("Unable to execute query %s", request.getStatement()));
+            return;
         }
-        VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, opts.showOET,
-                opts.showRP);
 
-        Date start = opts.timing ? new Date() : null;
+        if (opts.showQuery) {
+            printField("Query", response.getStatement());
+        }
 
-        Map<String, NodeControllerInfo> nodeControllerInfos = null;
-        if (hcc != null) {
-            nodeControllerInfos = hcc.getNodeControllerInfos();
+        if (request.isShowMetrics()) {
+            String metrics = String.format("Compile Time:\t%d\nElapsed Time:\t%d",
+                    response.getMetrics().getCompileTime(), response.getMetrics().getElapsedTime());
+            printField("Metrics", metrics);
         }
-        XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
-                opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf);
-        resultSetId = createResultSetId();
-        CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
-                resultSetId, null);
-        compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel, this.collectionList);
-        // if -timing argument passed, show the starting and ending times
-        Date end = opts.timing ? new Date() : null;
-        if (opts.timing) {
-            timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms");
+
+        if (request.isShowAbstractSyntaxTree()) {
+            printField("Abstract Syntax Tree", response.getAbstractSyntaxTree());
         }
-        if (opts.compileOnly) {
-            return;
+
+        if (request.isShowTranslatedExpressionTree()) {
+            printField("Translated Expression Tree", response.getTranslatedExpressionTree());
         }
 
-        Module module = compiler.getModule();
-        JobSpecification js = module.getHyracksJobSpecification();
-
-        DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
-        js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
-
-        // Repeat execution for number of times provided in -repeatexec argument
-        for (int i = 0; i < repeatedExecution; ++i) {
-            start = opts.timing ? new Date() : null;
-            runJob(js, writer);
-            // if -timing argument passed, show the starting and ending times
-            if (opts.timing) {
-                end = new Date();
-                long currentRun = end.getTime() - start.getTime();
-                if ((i + 1) > opts.timingIgnoreQueries) {
-                    sumTiming += currentRun;
-                    sumSquaredTiming += currentRun * currentRun;
-                    if (currentRun < minTiming) {
-                        minTiming = currentRun;
-                    }
-                    if (maxTiming < currentRun) {
-                        maxTiming = currentRun;
-                    }
-                }
-                timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms");
-            }
+        if (request.isShowOptimizedExpressionTree()) {
+            printField("Optimized Expression Tree", response.getOptimizedExpressionTree());
         }
-    }
 
-    /**
-     * Creates a Hyracks dataset, if not already existing with the job frame size, and 1 reader. Allocates a new buffer of size specified in the frame of Hyracks
-     * node. Creates new dataset reader with the current job ID and result set ID. Outputs the string in buffer for each frame.
-     *
-     * @param spec
-     *            JobSpecification object, containing frame size. Current specified job.
-     * @param writer
-     *            Writer for output of job.
-     * @throws Exception
-     */
-    private void runJob(JobSpecification spec, PrintWriter writer) throws Exception {
-        int nReaders = 1;
-        if (hds == null) {
-            hds = new HyracksDataset(hcc, spec.getFrameSize(), nReaders);
+        if (request.isShowRuntimePlan()) {
+            printField("Runtime Plan", response.getRuntimePlan());
         }
 
-        JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
+        printField("Results", response.getResults());
 
-        FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize());
-        IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
-        IHyracksDatasetReader reader = hds.createReader(jobId, resultSetId);
-        IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
+        if (executionIteration >= opts.timingIgnoreQueries) {
+            metricsList.add(response.getMetrics());
+        }
+    }
 
-        while (reader.read(frame) > 0) {
-            writer.print(ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor));
-            writer.flush();
-            frame.getBuffer().clear();
+    private void onFailure(String xqFile, ErrorResponse response) {
+        if (response == null) {
+            System.err.println(String.format("Unable to execute query in %s", xqFile));
+            return;
         }
 
-        hcc.waitForCompletion(jobId);
-    }
+        System.err.println();
+        System.err.println("------------------------ Errors ---------------------");
 
-    /**
-     * Create a unique result set id to get the correct query back from the cluster.
-     *
-     * @return Result Set id generated with current system time.
-     */
-    protected ResultSetId createResultSetId() {
-        return new ResultSetId(System.nanoTime());
+        Error error = response.getError();
+        String errorMsg = String.format("Code:\t %d\nMessage:\t %s", error.getCode(), error.getMessage());
+        printField(System.err, String.format("Errors for '%s'", xqFile), errorMsg);
     }
 
     /**
-     * Start local virtual cluster with cluster controller node and node controller nodes. IP address provided for node controller is localhost. Unassigned ports
-     * 39000 and 39001 are used for client and cluster port respectively. Creates a new Hyracks connection with the IP address and client ports.
+     * Submits a query to be executed by the REST API. Will call {@link #onFailure(String, ErrorResponse)} if any error
+     * occurs when submitting the query. Else will call {@link #onSuccess(String, QueryRequest, SyncQueryResponse)} with
+     * the {@link AsyncQueryResponse}
      *
-     * @throws Exception
+     * @param xqFile
+     *            .xq file with the query to be executed
+     * @param request
+     *            {@link QueryRequest} instance to be submitted to REST API
+     * @param cli
+     *            cli class instance
      */
-    public void startLocalHyracks() throws Exception {
-        String localAddress = InetAddress.getLocalHost().getHostAddress();
-        CCConfig ccConfig = new CCConfig();
-        ccConfig.clientNetIpAddress = localAddress;
-        ccConfig.clientNetPort = 39000;
-        ccConfig.clusterNetIpAddress = localAddress;
-        ccConfig.clusterNetPort = 39001;
-        ccConfig.httpPort = 39002;
-        ccConfig.profileDumpPeriod = 10000;
-        cc = new ClusterControllerService(ccConfig);
-        cc.start();
-
-        ncs = new NodeControllerService[opts.localNodeControllers];
-        for (int i = 0; i < ncs.length; i++) {
-            NCConfig ncConfig = new NCConfig();
-            ncConfig.ccHost = "localhost";
-            ncConfig.ccPort = 39001;
-            ncConfig.clusterNetIPAddress = localAddress;
-            ncConfig.dataIPAddress = localAddress;
-            ncConfig.resultIPAddress = localAddress;
-            ncConfig.nodeId = "nc" + (i + 1);
-            //TODO: enable index folder as a cli option for on-the-fly indexing queries
-            ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString();
-            ncs[i] = new NodeControllerService(ncConfig);
-            ncs[i].start();
+    private static void sendQueryRequest(String xqFile, QueryRequest request, VXQuery cli) {
+        URI uri = null;
+        try {
+            uri = RestUtils.buildQueryURI(request, cli.restIpAddress, cli.restPort);
+        } catch (URISyntaxException e) {
+            System.err.println(
+                    String.format("Unable to build URI to call REST API for query: %s", request.getStatement()));
+            cli.onFailure(xqFile, null);
         }
 
-        hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
+        CloseableHttpClient httpClient = HttpClients.custom().build();
+        try {
+            HttpGet httpGet = new HttpGet(uri);
+            httpGet.setHeader(HttpHeaders.ACCEPT, CONTENT_TYPE_JSON);
+
+            try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) {
+                HttpEntity entity = httpResponse.getEntity();
+
+                String response = RestUtils.readEntity(entity);
+                if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                    cli.onSuccess(xqFile, request,
+                            RestUtils.mapEntity(response, SyncQueryResponse.class, CONTENT_TYPE_JSON));
+                } else {
+                    cli.onFailure(xqFile, RestUtils.mapEntity(response, ErrorResponse.class, CONTENT_TYPE_JSON));
+                }
+            } catch (IOException e) {
+                System.err.println("Error occurred when reading entity: " + e.getMessage());
+                cli.onFailure(xqFile, null);
+            } catch (JAXBException e) {
+                System.err.println("Error occurred when mapping query response: " + e.getMessage());
+                cli.onFailure(xqFile, null);
+            }
+        } finally {
+            HttpClientUtils.closeQuietly(httpClient);
+        }
     }
 
     /**
-     * Shuts down the virtual cluster, along with all nodes and node execution, network and queue managers.
-     *
-     * @throws Exception
+     * Once the query in a given .xq file has been executed (with repeated executions as well), this method calculates
+     * mean, standard deviation, minimum and maximum execution times.
      */
-    public void stopLocalHyracks() throws Exception {
-        for (int i = 0; i < ncs.length; i++) {
-            ncs[i].stop();
+    private void showTimingSummary() {
+        double sumTime = 0;
+        double sumSquaredTime = 0;
+        long minTime = Long.MAX_VALUE;
+        long maxTime = Long.MIN_VALUE;
+
+        for (int i = 0; i < metricsList.size(); i++) {
+            Metrics metrics = metricsList.get(i);
+            long totalTime = metrics.getCompileTime() + metrics.getElapsedTime();
+
+            sumTime += totalTime;
+            sumSquaredTime += totalTime * totalTime;
+
+            if (totalTime < minTime) {
+                minTime = totalTime;
+            }
+
+            if (totalTime > maxTime) {
+                maxTime = totalTime;
+            }
         }
-        cc.stop();
+
+        double mean = sumTime / (opts.repeatExec - opts.timingIgnoreQueries);
+        double sd = Math.sqrt(sumSquaredTime / (opts.repeatExec - opts.timingIgnoreQueries) - mean * mean);
+
+        System.out.println();
+        System.out.println("\t**** Timing Summary ****");
+        System.out.println("----------------------------------------------------");
+        System.out.println(String.format("Repetitions:\t%d, Timing Ignored Iterations:\t%d", opts.repeatExec,
+                opts.timingIgnoreQueries));
+        System.out.println("Average execution time:\t" + mean + " ms");
+        System.out.println("Standard deviation:\t" + String.format("%.4f", sd));
+        System.out.println("Coefficient of variation:\t" + String.format("%.4f", sd / mean));
+        System.out.println("Minimum execution time:\t" + minTime + " ms");
+        System.out.println("Maximum execution time:\t" + maxTime + " ms");
+        System.out.println();
+    }
+
+    private static QueryRequest createQueryRequest(CmdLineOptions opts, String query) {
+        QueryRequest request = new QueryRequest(query);
+        request.setCompileOnly(opts.compileOnly);
+        request.setOptimization(opts.optimizationLevel);
+        request.setFrameSize(opts.frameSize);
+        request.setRepeatExecutions(opts.repeatExec);
+        request.setShowMetrics(opts.timing);
+        request.setShowAbstractSyntaxTree(opts.showAST);
+        request.setShowTranslatedExpressionTree(opts.showTET);
+        request.setShowOptimizedExpressionTree(opts.showOET);
+        request.setShowRuntimePlan(opts.showRP);
+        request.setAsync(false);
+
+        return request;
     }
 
     /**
-     * Reads the contents of file given in query into a String. The file is always closed. For XML files UTF-8 encoding is used.
+     * Reads the contents of file given in query into a String. The file is always closed. For XML files UTF-8 encoding
+     * is used.
      *
      * @param query
      *            The query with filename to be processed
@@ -367,46 +348,49 @@ public class VXQuery {
         return FileUtils.readFileToString(new File(query), "UTF-8");
     }
 
-    /**
-     * Save and print out the timing message.
-     *
-     * @param message
-     */
-    private static void timingMessage(String message) {
-        System.out.println(message);
-        timingMessages.add(message);
+    private static void printField(PrintStream out, String field, String value) {
+        out.println();
+        field = field + ":";
+        out.print(field);
+
+        String[] lines = value.split("\n");
+        for (int i = 0; i < lines.length; i++) {
+            int margin = 4;
+            if (i != 0) {
+                margin += field.length();
+            }
+            System.out.print(String.format("%1$" + margin + "s%2$s\n", "", lines[i]));
+        }
+    }
+
+    private static void printField(String field, String value) {
+        printField(System.out, field, value);
     }
 
     /**
      * Helper class with fields and methods to handle all command line options
      */
     private static class CmdLineOptions {
-        @Option(name = "-available-processors", usage = "Number of available processors. (default: java's available processors)")
-        private int availableProcessors = -1;
+        @Option(name = "-rest-ip-address", usage = "IP Address of the REST Server")
+        private String restIpAddress = null;
 
-        @Option(name = "-client-net-ip-address", usage = "IP Address of the ClusterController.")
-        private String clientNetIpAddress = null;
+        @Option(name = "-rest-port", usage = "Port of REST Server")
+        private int restPort = 8085;
 
-        @Option(name = "-client-net-port", usage = "Port of the ClusterController. (default: 1098)")
-        private int clientNetPort = 1098;
+        @Option(name = "-compileonly", usage = "Compile the query and stop.")
+        private boolean compileOnly;
 
-        @Option(name = "-local-node-controllers", usage = "Number of local node controllers. (default: 1)")
-        private int localNodeControllers = 1;
+        @Option(name = "-O", usage = "Optimization Level. (default: Full Optimization)")
+        private int optimizationLevel = Integer.MAX_VALUE;
 
         @Option(name = "-frame-size", usage = "Frame size in bytes. (default: 65,536)")
         private int frameSize = 65536;
 
-        @Option(name = "-join-hash-size", usage = "Join hash size in bytes. (default: 67,108,864)")
-        private long joinHashSize = -1;
-
-        @Option(name = "-maximum-data-size", usage = "Maximum possible data size in bytes. (default: 150,323,855,000)")
-        private long maximumDataSize = -1;
-
-        @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.")
-        private int bufferSize = -1;
+        @Option(name = "-repeatexec", usage = "Number of times to repeat execution.")
+        private int repeatExec = 1;
 
-        @Option(name = "-O", usage = "Optimization Level. (default: Full Optimization)")
-        private int optimizationLevel = Integer.MAX_VALUE;
+        @Option(name = "-timing", usage = "Produce timing information.")
+        private boolean timing;
 
         @Option(name = "-showquery", usage = "Show query string.")
         private boolean showQuery;
@@ -423,29 +407,33 @@ public class VXQuery {
         @Option(name = "-showrp", usage = "Show Runtime plan.")
         private boolean showRP;
 
-        @Option(name = "-compileonly", usage = "Compile the query and stop.")
-        private boolean compileOnly;
+        // Optional (Not supported by REST API) parameters. Only used for creating a
+        // local hyracks cluster
+        @Option(name = "-join-hash-size", usage = "Join hash size in bytes. (default: 67,108,864)")
+        private long joinHashSize = -1;
 
-        @Option(name = "-repeatexec", usage = "Number of times to repeat execution.")
-        private int repeatExec = 1;
+        @Option(name = "-maximum-data-size", usage = "Maximum possible data size in bytes. (default: 150,323,855,000)")
+        private long maximumDataSize = -1;
+
+        @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.")
+        private int bufferSize = -1;
 
         @Option(name = "-result-file", usage = "File path to save the query result.")
         private String resultFile = null;
 
-        @Option(name = "-timing", usage = "Produce timing information.")
-        private boolean timing;
-
         @Option(name = "-timing-ignore-queries", usage = "Ignore the first X number of quereies.")
-        private int timingIgnoreQueries = 2;
-
-        @Option(name = "-x", usage = "Bind an external variable")
-        private Map<String, String> bindings = new HashMap<>();
+        private int timingIgnoreQueries = 0;
 
         @Option(name = "-hdfs-conf", usage = "Directory path to Hadoop configuration files")
         private String hdfsConf = null;
 
+        @Option(name = "-available-processors", usage = "Number of available processors. (default: java's available processors)")
+        private int availableProcessors = -1;
+
+        @Option(name = "-local-node-controllers", usage = "Number of local node controllers. (default: 1)")
+        private int localNodeControllers = 1;
+
         @Argument
-        private List<String> arguments = new ArrayList<>();
+        private List<String> xqFiles = new ArrayList<>();
     }
-
-}
+}
\ No newline at end of file
diff --git a/vxquery-cli/src/site/markdown/index.md b/vxquery-cli/src/site/markdown/index.md
new file mode 100644 (file)
index 0000000..758c947
--- /dev/null
@@ -0,0 +1,116 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# VXQuery CLI
+
+VXQuery CLI is the command line utility which can be used to execute XQueries 
+with ease. No pre-configuration needs to be done in order to execute an XQuery.
+
+---
+
+## Quick Start
+
+***
+
+### Requirements
+
+- Apache VXQuery™ source archive (apache-vxquery-X.Y-source-release.zip)
+- JDK >= 1.8
+- Apache Maven >= 3.2
+
+***
+
+### Installing
+
+VXQuery CLI comes bundled with the VXQuery source distribution 
+(apache-vxquery-X.Y-source-release.zip).
+
+First, run `mvn package`.
+
+```
+$ unzip apache-vxquery-X.Y-source-release.zip
+$ cd apache-vxquery-X.Y
+$ mvn package -DskipTests
+```
+
+**vxquery-cli** binaries are located at `vxquery-cli/target/appassembler/bin`. 
+There are 2 files in this directory, **vxq** which is the bash executable for unix
+based systems and **vxq.bat** for windows systems. Depending on the platform,
+suitable executable needs to be selected.
+
+***
+
+### Executing a Query
+
+#### Put the query into a file
+
+VXQuery CLI takes a file location as the argument where this file includes the 
+query(statement) to be executed. Suppose the following query needs to be executed.
+
+```
+for $x in doc("books.xml")/bookstore/book
+where $x/price>30
+order by $x/title
+return $x/title
+```
+This statement is querying for the book titles in **books.xml** where price of
+the book is greater than 30. Also this query asks for the results to be ordered by
+*title* as well. Now, create a file (say **test.xq**) and put the above query as
+the content.
+
+**NOTE:** You can replace **books.xml** with any XML file that you have and want 
+to run a query on.
+
+#### Execute the query
+
+We need to invoke the matching executable according to your platform (unix/windows) 
+inside `vxquery-cli/target/appassembler/bin` directory. To execute the query, run:
+
+```
+sh ./apache-vxquery-X.Y/vxquery-cli/target/appassembler/bin/vxq path/to/test.xq
+```
+
+***
+
+## Command Line Options
+
+```
+ -O N                      : Optimization Level. (default: Full Optimization)
+ -available-processors N   : Number of available processors. (default: java's available processors)
+ -buffer-size N            : Disk read buffer size in bytes.
+ -compileonly              : Compile the query and stop.
+ -frame-size N             : Frame size in bytes. (default: 65,536)
+ -hdfs-conf VAL            : Directory path to Hadoop configuration files
+ -join-hash-size N         : Join hash size in bytes. (default: 67,108,864)
+ -local-node-controllers N : Number of local node controllers. (default: 1)
+ -maximum-data-size N      : Maximum possible data size in bytes. (default: 150,323,855,000)
+ -repeatexec N             : Number of times to repeat execution.
+ -rest-ip-address VAL      : IP Address of the REST Server.
+ -rest-port N              : Port of REST Server.
+ -result-file VAL          : File path to save the query result.
+ -showast                  : Show abstract syntax tree.
+ -showoet                  : Show optimized expression tree.
+ -showquery                : Show query string.
+ -showrp                   : Show Runtime plan.
+ -showtet                  : Show translated expression tree.
+ -timing                   : Produce timing information.
+ -timing-ignore-queries N  : Ignore the first X number of quereies.
+```
+
+**NOTE:** Normally, CLI starts a local VXQuery Server to execute the query. But,
+if you already have a VXQuery Server running, you can send the query to the 
+inbuilt *REST Server* running in that server by specifying the **port** and **ip address** 
+of the REST Server through options `-rest-ip-address` and `-rest-port`.
index 4d35a0f..8b3198d 100644 (file)
@@ -15,34 +15,39 @@ See the License for the specific language governing permissions and
 limitations under the License.
 -->
 <project name="VXQuery">
-  <bannerLeft>
-    <name>VXQuery</name>
-    <src>../images/VXQuery.png</src>
-    <href>../index.html</href>
-  </bannerLeft>
+    <bannerLeft>
+        <name>VXQuery</name>
+        <src>../images/VXQuery.png</src>
+        <href>../index.html</href>
+    </bannerLeft>
 
-  <bannerRight>
-    <name>Apache Software Foundation</name>
-    <src>../images/asf_logo_wide.png</src>
-    <href>http://www.apache.org/</href>
-  </bannerRight>
+    <bannerRight>
+        <name>Apache Software Foundation</name>
+        <src>../images/asf_logo_wide.png</src>
+        <href>http://www.apache.org/</href>
+    </bannerRight>
 
-  <skin>
-    <groupId>org.apache.maven.skins</groupId>
-    <artifactId>maven-fluido-skin</artifactId>
-    <version>1.5</version>
-  </skin>
+    <skin>
+        <groupId>org.apache.maven.skins</groupId>
+        <artifactId>maven-fluido-skin</artifactId>
+        <version>1.5</version>
+    </skin>
 
-  <body>
-    <menu ref="reports"/>
-    <footer><![CDATA[
-      <div class="row-fluid">Apache VXQuery, VXQuery, Apache, the Apache
-        feather logo, and the Apache VXQuery project logo are either
-        registered trademarks or trademarks of The Apache Software
-        Foundation in the United States and other countries.
-        All other marks mentioned may be trademarks or registered
-        trademarks of their respective owners.</div>
-    ]]></footer>
-  </body>
+    <body>
+        <menu name="VXQuery CLI">
+            <item name="Overview" href="index.html"/>
+        </menu>
+
+        <menu ref="reports"/>
+        <footer><![CDATA[
+            <div class="row-fluid">Apache VXQuery, VXQuery, Apache, the Apache
+                feather logo, and the Apache VXQuery project logo are either
+                registered trademarks or trademarks of The Apache Software
+                Foundation in the United States and other countries.
+                All other marks mentioned may be trademarks or registered
+                trademarks of their respective owners.
+            </div>]]>
+        </footer>
+    </body>
 </project>
 
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java
new file mode 100644 (file)
index 0000000..7f748b7
--- /dev/null
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.exceptions;
+
+/**
+ * A runtime exception to be thrown by the VXQuery and related classes of the rest server
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryRuntimeException extends RuntimeException {
+
+    public VXQueryRuntimeException(String message) {
+        super(message);
+    }
+
+    public VXQueryRuntimeException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java
new file mode 100644 (file)
index 0000000..f05cd6f
--- /dev/null
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.exceptions;
+
+/**
+ * A runtime exception class to be used to be thrown when runtime errors occur within servlets.
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryServletRuntimeException extends VXQueryRuntimeException {
+
+    public VXQueryServletRuntimeException(String message) {
+        super(message);
+    }
+
+    public VXQueryServletRuntimeException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/vxquery-rest/pom.xml b/vxquery-rest/pom.xml
new file mode 100644 (file)
index 0000000..052132f
--- /dev/null
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements. See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License. You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>apache-vxquery</artifactId>
+        <groupId>org.apache.vxquery</groupId>
+        <version>0.7-SNAPSHOT</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>jar</packaging>
+    <name>VXQuery REST Server</name>
+    <description>Apache VXQuery REST Server</description>
+
+    <artifactId>apache-vxquery-rest</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.vxquery</groupId>
+            <artifactId>apache-vxquery-core</artifactId>
+            <version>0.7-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hyracks</groupId>
+            <artifactId>hyracks-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hyracks</groupId>
+            <artifactId>hyracks-http</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java
new file mode 100644 (file)
index 0000000..f5e0165
--- /dev/null
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.app;
+
+import static org.apache.vxquery.rest.Constants.Properties.AVAILABLE_PROCESSORS;
+import static org.apache.vxquery.rest.Constants.Properties.HDFS_CONFIG;
+import static org.apache.vxquery.rest.Constants.Properties.JOIN_HASH_SIZE;
+import static org.apache.vxquery.rest.Constants.Properties.MAXIMUM_DATA_SIZE;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.hyracks.api.application.ICCApplicationContext;
+import org.apache.hyracks.api.application.ICCApplicationEntryPoint;
+import org.apache.hyracks.api.client.ClusterControllerInfo;
+import org.apache.vxquery.exceptions.VXQueryRuntimeException;
+import org.apache.vxquery.rest.RestServer;
+import org.apache.vxquery.rest.service.VXQueryConfig;
+import org.apache.vxquery.rest.service.VXQueryService;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+/**
+ * Main class responsible for starting the {@link RestServer} and
+ * {@link VXQueryService} classes.
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryApplication implements ICCApplicationEntryPoint {
+
+    private static final Logger LOGGER = Logger.getLogger(VXQueryApplication.class.getName());
+
+    private VXQueryService vxQueryService;
+    private RestServer restServer;
+
+    @Override
+    public void start(ICCApplicationContext ccAppCtx, String[] args) throws Exception {
+        AppArgs appArgs = new AppArgs();
+        if (args != null) {
+            CmdLineParser parser = new CmdLineParser(appArgs);
+            try {
+                parser.parseArgument(args);
+            } catch (Exception e) {
+                parser.printUsage(System.err);
+                throw new VXQueryRuntimeException("Unable to parse app arguments", e);
+            }
+        }
+
+        VXQueryConfig config =
+                loadConfiguration(ccAppCtx.getCCContext().getClusterControllerInfo(), appArgs.getVxqueryConfig());
+        vxQueryService = new VXQueryService(config);
+        restServer = new RestServer(vxQueryService, appArgs.getRestPort());
+    }
+
+    public synchronized void stop() {
+        try {
+            LOGGER.log(Level.INFO, "Stopping REST server");
+            restServer.stop();
+
+            LOGGER.log(Level.INFO, "Stopping VXQueryService");
+            vxQueryService.stop();
+        } catch (Exception e) {
+            LOGGER.log(Level.SEVERE, "Error occurred when stopping the application", e);
+        }
+    }
+
+    @Override
+    public void startupCompleted() throws Exception {
+        try {
+            LOGGER.log(Level.INFO, "Starting VXQueryService");
+            vxQueryService.start();
+            LOGGER.log(Level.INFO, "VXQueryService started successfully");
+
+            LOGGER.log(Level.INFO, "Starting REST server");
+            restServer.start();
+        } catch (Exception e) {
+            LOGGER.log(Level.SEVERE, "Error occurred when starting application", e);
+            stop();
+            throw new VXQueryRuntimeException("Error occurred when starting application", e);
+        }
+    }
+
+    /**
+     * Loads properties from
+     * 
+     * <pre>
+     * -appConfig foo/bar.properties
+     * </pre>
+     * 
+     * file if specified in the app arguments.
+     *
+     * @param clusterControllerInfo
+     *            cluster controller information
+     * @param propertiesFile
+     *            vxquery configuration properties file, given by
+     * 
+     *            <pre>
+     *            -appConfig
+     *            </pre>
+     * 
+     *            option in app argument
+     * @return A new {@link VXQueryConfig} instance with either default properties
+     *         or properties loaded from the properties file given.
+     */
+    private VXQueryConfig loadConfiguration(ClusterControllerInfo clusterControllerInfo, String propertiesFile) {
+        VXQueryConfig vxqConfig = new VXQueryConfig();
+        if (propertiesFile != null) {
+            try (InputStream in = new FileInputStream(propertiesFile)) {
+                System.getProperties().load(in);
+            } catch (IOException e) {
+                LOGGER.log(Level.SEVERE,
+                        String.format("Error occurred when loading properties file %s", propertiesFile), e);
+            }
+        }
+
+        vxqConfig.setAvailableProcessors(Integer.getInteger(AVAILABLE_PROCESSORS, 1));
+        vxqConfig.setJoinHashSize(Long.getLong(JOIN_HASH_SIZE, -1));
+        vxqConfig.setHdfsConf(System.getProperty(HDFS_CONFIG));
+        vxqConfig.setMaximumDataSize(Long.getLong(MAXIMUM_DATA_SIZE, -1));
+
+        vxqConfig.setHyracksClientIp(clusterControllerInfo.getClientNetAddress());
+        vxqConfig.setHyracksClientPort(clusterControllerInfo.getClientNetPort());
+
+        return vxqConfig;
+    }
+
+    public VXQueryService getVxQueryService() {
+        return vxQueryService;
+    }
+
+    public RestServer getRestServer() {
+        return restServer;
+    }
+
+    /**
+     * Application Arguments bean class
+     */
+    private class AppArgs {
+        @Option(name = "-restPort", usage = "The port on which REST server starts")
+        private int restPort = 8080;
+
+        @Option(name = "-appConfig", usage = "Properties file location which includes VXQueryService Application additional configuration")
+        private String vxqueryConfig = null;
+
+        public String getVxqueryConfig() {
+            return vxqueryConfig;
+        }
+
+        public void setVxqueryConfig(String vxqueryConfig) {
+            this.vxqueryConfig = vxqueryConfig;
+        }
+
+        public int getRestPort() {
+            return restPort;
+        }
+
+        public void setRestPort(int restPort) {
+            this.restPort = restPort;
+        }
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java
new file mode 100644 (file)
index 0000000..9b18745
--- /dev/null
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.vxquery.app.util;
+
+import static org.apache.vxquery.rest.Constants.Properties.AVAILABLE_PROCESSORS;
+import static org.apache.vxquery.rest.Constants.Properties.HDFS_CONFIG;
+import static org.apache.vxquery.rest.Constants.Properties.JOIN_HASH_SIZE;
+import static org.apache.vxquery.rest.Constants.Properties.MAXIMUM_DATA_SIZE;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.util.Arrays;
+
+import org.apache.hyracks.api.client.HyracksConnection;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.control.cc.ClusterControllerService;
+import org.apache.hyracks.control.common.controllers.CCConfig;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.vxquery.app.VXQueryApplication;
+import org.apache.vxquery.rest.service.VXQueryConfig;
+import org.apache.vxquery.rest.service.VXQueryService;
+
+/**
+ * A utility class to start the a local hyracks cluster.
+ *
+ * @author Preston Carman
+ */
+public class LocalClusterUtil {
+    /*
+     * Start local virtual cluster with cluster controller node and node controller
+     * nodes. IP address provided for node controller is localhost. Unassigned ports
+     * 39000 and 39001 are used for client and cluster port respectively.
+     */
+    public static final int DEFAULT_HYRACKS_CC_CLIENT_PORT = 39000;
+    public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 39001;
+    public static final int DEFAULT_HYRACKS_CC_HTTP_PORT = 39002;
+    public static final int DEFAULT_VXQUERY_REST_PORT = 39003;
+
+    // TODO review variable scope after XTest is updated to use the REST service.
+    public ClusterControllerService clusterControllerService;
+    public NodeControllerService nodeControllerSerivce;
+    public IHyracksClientConnection hcc;
+    public IHyracksDataset hds;
+    public VXQueryService vxQueryService;
+
+    public void init(VXQueryConfig config) throws Exception {
+        // Following properties are needed by the app to setup
+        System.setProperty(AVAILABLE_PROCESSORS, String.valueOf(config.getAvailableProcessors()));
+        System.setProperty(JOIN_HASH_SIZE, String.valueOf(config.getJoinHashSize()));
+        System.setProperty(MAXIMUM_DATA_SIZE, String.valueOf(config.getMaximumDataSize()));
+        if (config.getHdfsConf() != null) {
+            System.setProperty(HDFS_CONFIG, config.getHdfsConf());
+        }
+
+        // Cluster controller
+        CCConfig ccConfig = createCCConfig();
+        clusterControllerService = new ClusterControllerService(ccConfig);
+        clusterControllerService.start();
+
+        hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
+        hds = new HyracksDataset(hcc, config.getFrameSize(), config.getAvailableProcessors());
+
+        // Node controller
+        NCConfig ncConfig = createNCConfig();
+        nodeControllerSerivce = new NodeControllerService(ncConfig);
+        nodeControllerSerivce.start();
+
+        hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
+
+        // REST controller
+        config.setHyracksClientIp(ccConfig.clientNetIpAddress);
+        config.setHyracksClientPort(ccConfig.clientNetPort);
+        vxQueryService = new VXQueryService(config);
+        vxQueryService.start();
+    }
+
+    protected CCConfig createCCConfig() throws IOException {
+        String localAddress = getIpAddress();
+        CCConfig ccConfig = new CCConfig();
+        ccConfig.clientNetIpAddress = localAddress;
+        ccConfig.clientNetPort = DEFAULT_HYRACKS_CC_CLIENT_PORT;
+        ccConfig.clusterNetIpAddress = localAddress;
+        ccConfig.clusterNetPort = DEFAULT_HYRACKS_CC_CLUSTER_PORT;
+        ccConfig.httpPort = DEFAULT_HYRACKS_CC_HTTP_PORT;
+        ccConfig.profileDumpPeriod = 10000;
+        ccConfig.appCCMainClass = VXQueryApplication.class.getName();
+        ccConfig.appArgs = Arrays.asList("-restPort", String.valueOf(DEFAULT_VXQUERY_REST_PORT));
+
+        return ccConfig;
+    }
+
+    protected NCConfig createNCConfig() throws IOException {
+        String localAddress = getIpAddress();
+        NCConfig ncConfig = new NCConfig();
+        ncConfig.ccHost = "localhost";
+        ncConfig.ccPort = DEFAULT_HYRACKS_CC_CLUSTER_PORT;
+        ncConfig.clusterNetIPAddress = localAddress;
+        ncConfig.dataIPAddress = localAddress;
+        ncConfig.resultIPAddress = localAddress;
+        ncConfig.nodeId = "test_node";
+        ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString();
+        return ncConfig;
+    }
+
+    public IHyracksClientConnection getHyracksClientConnection() {
+        return hcc;
+    }
+
+    public VXQueryService getVxQueryService() {
+        return vxQueryService;
+    }
+
+    public void deinit() throws Exception {
+        vxQueryService.stop();
+        nodeControllerSerivce.stop();
+        clusterControllerService.stop();
+    }
+
+    public static void main(String[] args) {
+        LocalClusterUtil localClusterUtil = new LocalClusterUtil();
+        VXQueryConfig config = new VXQueryConfig();
+        run(localClusterUtil, config);
+    }
+
+    protected static void run(final LocalClusterUtil localClusterUtil, VXQueryConfig config) {
+        Runtime.getRuntime().addShutdownHook(new Thread() {
+            @Override
+            public void run() {
+                try {
+                    localClusterUtil.deinit();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        try {
+            localClusterUtil.init(config);
+            while (true) {
+                Thread.sleep(10000);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    public String getIpAddress() throws UnknownHostException {
+        return InetAddress.getLocalHost().getHostAddress();
+    }
+
+    public int getRestPort() {
+        return DEFAULT_VXQUERY_REST_PORT;
+    }
+
+    @Deprecated
+    public IHyracksClientConnection getConnection() {
+        return hcc;
+    }
+
+    @Deprecated
+    public IHyracksDataset getDataset() {
+        return hds;
+    }
+
+}
\ No newline at end of file
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java
new file mode 100644 (file)
index 0000000..fe91836
--- /dev/null
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.app.util;
+
+import static org.apache.vxquery.rest.Constants.MODE_ASYNC;
+import static org.apache.vxquery.rest.Constants.MODE_SYNC;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+import static org.apache.vxquery.rest.Constants.Parameters.COMPILE_ONLY;
+import static org.apache.vxquery.rest.Constants.Parameters.FRAME_SIZE;
+import static org.apache.vxquery.rest.Constants.Parameters.METRICS;
+import static org.apache.vxquery.rest.Constants.Parameters.MODE;
+import static org.apache.vxquery.rest.Constants.Parameters.OPTIMIZATION;
+import static org.apache.vxquery.rest.Constants.Parameters.REPEAT_EXECUTIONS;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_AST;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_OET;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_RP;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_TET;
+import static org.apache.vxquery.rest.Constants.Parameters.STATEMENT;
+import static org.apache.vxquery.rest.Constants.URLs.QUERY_ENDPOINT;
+import static org.apache.vxquery.rest.Constants.URLs.QUERY_RESULT_ENDPOINT;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.StringReader;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+
+/**
+ * A set of utility methods used by the REST related tasks
+ *
+ * @author Erandi Ganepola
+ */
+public class RestUtils {
+
+    private RestUtils() {
+    }
+
+    /**
+     * Builds the {@link URI} once the {@link QueryRequest} is given. Only the
+     * parameters given (different from the default values) are put in the
+     * {@link URI}
+     * 
+     * @param request
+     *            {@link QueryRequest} to be converted to a {@link URI}
+     * @param restIpAddress
+     *            Ip address of the REST server
+     * @param restPort
+     *            port of the REST server
+     * @return generated {@link URI}
+     * @throws URISyntaxException
+     */
+    public static URI buildQueryURI(QueryRequest request, String restIpAddress, int restPort)
+            throws URISyntaxException {
+        URIBuilder builder =
+                new URIBuilder().setScheme("http").setHost(restIpAddress).setPort(restPort).setPath(QUERY_ENDPOINT);
+
+        if (request.getStatement() != null) {
+            builder.addParameter(STATEMENT, request.getStatement());
+        }
+        if (request.isCompileOnly()) {
+            builder.addParameter(COMPILE_ONLY, String.valueOf(request.isCompileOnly()));
+        }
+        if (request.getOptimization() != QueryRequest.DEFAULT_OPTIMIZATION) {
+            builder.addParameter(OPTIMIZATION, String.valueOf(request.getOptimization()));
+        }
+        if (request.getFrameSize() != QueryRequest.DEFAULT_FRAMESIZE) {
+            builder.addParameter(FRAME_SIZE, String.valueOf(request.getFrameSize()));
+        }
+        if (request.getRepeatExecutions() != 1) {
+            builder.addParameter(REPEAT_EXECUTIONS, String.valueOf(request.getRepeatExecutions()));
+        }
+        if (request.isShowMetrics()) {
+            builder.addParameter(METRICS, String.valueOf(request.isShowMetrics()));
+        }
+        if (request.isShowAbstractSyntaxTree()) {
+            builder.addParameter(SHOW_AST, String.valueOf(request.isShowAbstractSyntaxTree()));
+        }
+        if (request.isShowTranslatedExpressionTree()) {
+            builder.addParameter(SHOW_TET, String.valueOf(request.isShowTranslatedExpressionTree()));
+        }
+        if (request.isShowOptimizedExpressionTree()) {
+            builder.addParameter(SHOW_OET, String.valueOf(request.isShowOptimizedExpressionTree()));
+        }
+        if (request.isShowRuntimePlan()) {
+            builder.addParameter(SHOW_RP, String.valueOf(request.isShowRuntimePlan()));
+        }
+        if (!request.isAsync()) {
+            builder.addParameter(MODE, request.isAsync() ? MODE_ASYNC : MODE_SYNC);
+        }
+
+        return builder.build();
+    }
+
+    /**
+     * Builds the query result {@link URI} given the {@link QueryResultRequest}
+     * 
+     * @param resultRequest
+     *            result request
+     * @param restIpAddress
+     *            rest server's ip
+     * @param restPort
+     *            port of the rest server
+     * @return generated {@link URI}
+     * @throws URISyntaxException
+     */
+    public static URI buildQueryResultURI(QueryResultRequest resultRequest, String restIpAddress, int restPort)
+            throws URISyntaxException {
+        URIBuilder builder = new URIBuilder().setScheme("http").setHost(restIpAddress).setPort(restPort)
+                .setPath(QUERY_RESULT_ENDPOINT.replace("*", String.valueOf(resultRequest.getResultId())));
+
+        if (resultRequest.isShowMetrics()) {
+            builder.setParameter(METRICS, String.valueOf(resultRequest.isShowMetrics()));
+        }
+
+        return builder.build();
+    }
+
+    /**
+     * Reads the entity from an {@link HttpEntity}
+     * 
+     * @param entity
+     *            entity instance to be read
+     * @return entity read by this method as a string
+     * @throws IOException
+     */
+    public static String readEntity(HttpEntity entity) throws IOException {
+        StringBuilder responseBody = new StringBuilder();
+
+        try (InputStream in = entity.getContent()) {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+            String line;
+            while ((line = reader.readLine()) != null) {
+                responseBody.append(line);
+            }
+        }
+        return responseBody.toString();
+    }
+
+    /**
+     * Maps the object in the string representation to a java object. To map json
+     * entities, this method use {@link ObjectMapper}. For XML this method use
+     * {@link Unmarshaller}.
+     * 
+     * @param entity
+     *            string representation of the object
+     * @param type
+     *            the class to which the string needs to be mapped to
+     * @param contentType
+     *            json or XML
+     * @param <T>
+     *            content's class type
+     * @return mapped object
+     * @throws IOException
+     * @throws JAXBException
+     */
+    public static <T> T mapEntity(String entity, Class<T> type, String contentType) throws IOException, JAXBException {
+        if (contentType == null) {
+            contentType = CONTENT_TYPE_JSON;
+        }
+
+        switch (contentType) {
+            case CONTENT_TYPE_XML:
+                JAXBContext jaxbContext = JAXBContext.newInstance(type);
+                Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
+                return type.cast(unmarshaller.unmarshal(new StringReader(entity)));
+            case CONTENT_TYPE_JSON:
+            default:
+                ObjectMapper jsonMapper = new ObjectMapper();
+                return jsonMapper.readValue(entity, type);
+        }
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java
new file mode 100644 (file)
index 0000000..4ba79ec
--- /dev/null
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+public class Constants {
+
+    private Constants() {
+    }
+
+    public class Parameters {
+        public static final String STATEMENT = "statement";
+        public static final String RESULT_ID = "resultId ";
+        public static final String COMPILE_ONLY = "compileOnly";
+        public static final String OPTIMIZATION = "optimization";
+        public static final String FRAME_SIZE = "frameSize";
+        public static final String REPEAT_EXECUTIONS = "repeatExecutions";
+        public static final String METRICS = "metrics";
+        public static final String SHOW_AST = "showAbstractSyntaxTree";
+        public static final String SHOW_TET = "showTranslatedExpressionTree";
+        public static final String SHOW_OET = "showOptimizedExpressionTree";
+        public static final String SHOW_RP = "showRuntimePlan";
+        public static final String MODE = "mode";
+    }
+
+    public class URLs {
+        public static final String BASE_PATH = "/vxquery";
+
+        public static final String QUERY_ENDPOINT = BASE_PATH + "/query";
+        public static final String QUERY_RESULT_ENDPOINT = BASE_PATH + "/query/result/*";
+    }
+
+    public class Properties {
+        public static final String AVAILABLE_PROCESSORS = "org.apache.vxquery.available_processors";
+        public static final String LOCAL_NODE_CONTROLLERS = "org.apache.vxquery.local_nc";
+        public static final String JOIN_HASH_SIZE = "org.apache.vxquery.join_hash";
+        public static final String MAXIMUM_DATA_SIZE = "org.apache.vxquery.data_size";
+        public static final String HDFS_CONFIG = "org.apache.vxquery.hdfs_config";
+    }
+
+    public class HttpHeaderValues {
+        public static final String CONTENT_TYPE_JSON = "application/json";
+        public static final String CONTENT_TYPE_XML = "application/xml";
+    }
+
+    public class ErrorCodes {
+        public static final int PROBLEM_WITH_QUERY = 400;
+        public static final int UNFORSEEN_PROBLEM = 500;
+        public static final int INVALID_INPUT = 405;
+        public static final int NOT_FOUND = 404;
+    }
+
+    public static final String RESULT_URL_PREFIX = "/vxquery/query/result/";
+
+    public static final String MODE_ASYNC = "async";
+    public static final String MODE_SYNC = "sync";
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java
new file mode 100644 (file)
index 0000000..5b59c0c
--- /dev/null
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import static org.apache.vxquery.rest.Constants.URLs.QUERY_ENDPOINT;
+import static org.apache.vxquery.rest.Constants.URLs.QUERY_RESULT_ENDPOINT;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.hyracks.http.server.HttpServer;
+import org.apache.hyracks.http.server.WebManager;
+import org.apache.vxquery.exceptions.VXQueryRuntimeException;
+import org.apache.vxquery.rest.service.VXQueryService;
+import org.apache.vxquery.rest.servlet.QueryAPIServlet;
+import org.apache.vxquery.rest.servlet.QueryResultAPIServlet;
+
+/**
+ * REST Server class responsible for starting a new server on a given port.
+ *
+ * @author Erandi Ganepola
+ */
+public class RestServer {
+
+    public static final Logger LOGGER = Logger.getLogger(RestServer.class.getName());
+
+    private WebManager webManager;
+    private int port;
+
+    public RestServer(VXQueryService vxQueryService, int port) {
+        if (port == 0) {
+            throw new IllegalArgumentException("REST Server port cannot be 0");
+        }
+
+        this.port = port;
+
+        webManager = new WebManager();
+        HttpServer restServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), this.port);
+        restServer.addServlet(new QueryAPIServlet(vxQueryService, restServer.ctx(), QUERY_ENDPOINT));
+        restServer.addServlet(new QueryResultAPIServlet(vxQueryService, restServer.ctx(), QUERY_RESULT_ENDPOINT));
+        webManager.add(restServer);
+    }
+
+    public void start() {
+        try {
+            LOGGER.log(Level.FINE, "Starting rest server on port: " + port);
+            webManager.start();
+        } catch (Exception e) {
+            LOGGER.log(Level.SEVERE, "Error occurred when starting rest server", e);
+            throw new VXQueryRuntimeException("Unable to start REST server", e);
+        }
+        LOGGER.log(Level.INFO, "Rest server started on port: " + port);
+    }
+
+    public void stop() {
+        try {
+            LOGGER.log(Level.FINE, "Stopping rest server");
+            webManager.stop();
+        } catch (Exception e) {
+            LOGGER.log(Level.SEVERE, "Error occurred when stopping VXQueryService", e);
+            throw new VXQueryRuntimeException("Error occurred when stopping rest server", e);
+        }
+        LOGGER.log(Level.INFO, "Rest server stopped");
+    }
+
+    public int getPort() {
+        return port;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java
new file mode 100644 (file)
index 0000000..a88ae1c
--- /dev/null
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.rest.request;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.vxquery.rest.RestServer;
+
+/**
+ * Request to represent a query request coming to the {@link RestServer}
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryRequest {
+
+    public static final int DEFAULT_FRAMESIZE = 65536;
+    public static final int DEFAULT_OPTIMIZATION = 0;
+
+    private String statement;
+    private boolean async = true;
+    private boolean compileOnly;
+    private int optimization = DEFAULT_OPTIMIZATION;
+    /** Frame size in bytes. (default: 65,536) */
+    private int frameSize = DEFAULT_FRAMESIZE;
+    private int repeatExecutions = 1;
+    private boolean showMetrics = false;
+    private boolean showAbstractSyntaxTree = false;
+    private boolean showTranslatedExpressionTree = false;
+    private boolean showOptimizedExpressionTree = false;
+    private boolean showRuntimePlan = false;
+    /** A unique UUID to uniquely identify a given request */
+    private String requestId;
+
+    /** An optional map of source files. Required for XTests */
+    private Map<String, File> sourceFileMap = new HashMap<>();
+
+    public QueryRequest(String statement) {
+        this(null, statement);
+    }
+
+    public QueryRequest(String requestId, String statement) {
+        if (statement == null) {
+            throw new IllegalArgumentException("Statement cannot be null");
+        }
+
+        this.statement = statement;
+        this.requestId = requestId;
+    }
+
+    public String getStatement() {
+        return statement;
+    }
+
+    public boolean isCompileOnly() {
+        return compileOnly;
+    }
+
+    public void setCompileOnly(boolean compileOnly) {
+        this.compileOnly = compileOnly;
+    }
+
+    public int getOptimization() {
+        return optimization;
+    }
+
+    public void setOptimization(int optimization) {
+        this.optimization = optimization;
+    }
+
+    public int getFrameSize() {
+        return frameSize;
+    }
+
+    public void setFrameSize(int frameSize) {
+        this.frameSize = frameSize;
+    }
+
+    public int getRepeatExecutions() {
+        return repeatExecutions;
+    }
+
+    public void setRepeatExecutions(int repeatExecutions) {
+        this.repeatExecutions = repeatExecutions;
+    }
+
+    public boolean isShowAbstractSyntaxTree() {
+        return showAbstractSyntaxTree;
+    }
+
+    public void setShowAbstractSyntaxTree(boolean showAbstractSyntaxTree) {
+        this.showAbstractSyntaxTree = showAbstractSyntaxTree;
+    }
+
+    public boolean isShowTranslatedExpressionTree() {
+        return showTranslatedExpressionTree;
+    }
+
+    public void setShowTranslatedExpressionTree(boolean showTranslatedExpressionTree) {
+        this.showTranslatedExpressionTree = showTranslatedExpressionTree;
+    }
+
+    public boolean isShowOptimizedExpressionTree() {
+        return showOptimizedExpressionTree;
+    }
+
+    public void setShowOptimizedExpressionTree(boolean showOptimizedExpressionTree) {
+        this.showOptimizedExpressionTree = showOptimizedExpressionTree;
+    }
+
+    public boolean isShowRuntimePlan() {
+        return showRuntimePlan;
+    }
+
+    public void setShowRuntimePlan(boolean showRuntimePlan) {
+        this.showRuntimePlan = showRuntimePlan;
+    }
+
+    public boolean isShowMetrics() {
+        return showMetrics;
+    }
+
+    public void setShowMetrics(boolean showMetrics) {
+        this.showMetrics = showMetrics;
+    }
+
+    public String toString() {
+        return String.format("{ statement : %s }", statement);
+    }
+
+    public String getRequestId() {
+        return requestId;
+    }
+
+    public boolean isAsync() {
+        return async;
+    }
+
+    public void setAsync(boolean async) {
+        this.async = async;
+    }
+
+    public Map<String, File> getSourceFileMap() {
+        return sourceFileMap;
+    }
+
+    public void setSourceFileMap(Map<String, File> sourceFileMap) {
+        this.sourceFileMap = sourceFileMap;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java
new file mode 100644 (file)
index 0000000..5e43181
--- /dev/null
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.request;
+
+import org.apache.vxquery.rest.RestServer;
+
+/**
+ * Request to represent a query request coming to the {@link RestServer}
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryResultRequest {
+
+    private long resultId;
+    private boolean showMetrics = false;
+    private String requestId;
+
+    public QueryResultRequest(long resultId) {
+        this(resultId, null);
+    }
+
+    public QueryResultRequest(long resultId, String requestId) {
+        this.resultId = resultId;
+        this.requestId = requestId;
+    }
+
+    public long getResultId() {
+        return resultId;
+    }
+
+    public boolean isShowMetrics() {
+        return showMetrics;
+    }
+
+    public void setShowMetrics(boolean showMetrics) {
+        this.showMetrics = showMetrics;
+    }
+
+    public String getRequestId() {
+        return requestId;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java
new file mode 100644 (file)
index 0000000..57b67b7
--- /dev/null
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import static org.apache.vxquery.rest.Constants.RESULT_URL_PREFIX;
+
+import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.service.Status;
+
+/**
+ * Base class for any type of response which can be sent by the REST server.
+ * These responses can be query responses, error responses or query result
+ * responses.
+ *
+ * @author Erandi Ganepola
+ */
+public class APIResponse {
+
+    private String status;
+    private String requestId;
+
+    public APIResponse() {
+        status = Status.SUCCESS.toString();
+    }
+
+    public APIResponse(String status) {
+        this.status = status;
+    }
+
+    public String getStatus() {
+        return status;
+    }
+
+    public String getRequestId() {
+        return requestId;
+    }
+
+    public void setRequestId(String requestId) {
+        this.requestId = requestId;
+    }
+
+    public static ErrorResponse newErrorResponse(String requestId, Error error) {
+        ErrorResponse response = new ErrorResponse();
+        response.setRequestId(requestId);
+        response.setError(error);
+        return response;
+    }
+
+    public static QueryResponse newQueryResponse(QueryRequest request, ResultSetId resultSetId) {
+        QueryResponse response;
+        if (request.isAsync()) {
+            AsyncQueryResponse asyncQueryResponse = new AsyncQueryResponse();
+            if (!request.isCompileOnly()) {
+                asyncQueryResponse.setResultId(resultSetId.getId());
+                asyncQueryResponse.setResultUrl(RESULT_URL_PREFIX + resultSetId.getId());
+            }
+            response = asyncQueryResponse;
+        } else {
+            response = new SyncQueryResponse();
+        }
+
+        response.setRequestId(request.getRequestId());
+        return response;
+    }
+
+    public static QueryResultResponse newQueryResultResponse(String requestId) {
+        QueryResultResponse response = new QueryResultResponse();
+        response.setRequestId(requestId);
+        return response;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java
new file mode 100644 (file)
index 0000000..3216dce
--- /dev/null
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Resource class to represent a response to a given user query
+ *
+ * @author Erandi Ganepola
+ */
+@XmlRootElement
+public class AsyncQueryResponse extends QueryResponse {
+
+    private long resultId;
+    private String resultUrl;
+
+    public long getResultId() {
+        return resultId;
+    }
+
+    public void setResultId(long resultId) {
+        this.resultId = resultId;
+    }
+
+    public String getResultUrl() {
+        return resultUrl;
+    }
+
+    public void setResultUrl(String resultUrl) {
+        this.resultUrl = resultUrl;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java
new file mode 100644 (file)
index 0000000..96a709d
--- /dev/null
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Represents the
+ * 
+ * <pre>
+ * error
+ * </pre>
+ * 
+ * part of an {@link ErrorResponse}.
+ * 
+ * <pre>
+ *     {@code
+ *     <error>
+ *         <code>405</code>
+ *         <message>Invalid Input</message>
+ *     </error>}
+ * </pre>
+ *
+ * @author Erandi Ganepola
+ */
+@XmlRootElement
+public class Error {
+
+    private int code;
+    private String message;
+
+    public Error() {
+    }
+
+    public Error(int code, String message) {
+        this.code = code;
+        this.message = message;
+    }
+
+    public int getCode() {
+        return code;
+    }
+
+    public void setCode(int code) {
+        this.code = code;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private int code = -1;
+        private String message = null;
+
+        public Builder withCode(int code) {
+            this.code = code;
+            return this;
+        }
+
+        public Builder withMessage(String message) {
+            this.message = message;
+            return this;
+        }
+
+        public Error build() {
+            if (code == -1) {
+                code = 500;
+            }
+
+            if (message == null) {
+                message = "unexpected Error";
+            }
+
+            return new Error(code, message);
+        }
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java
new file mode 100644 (file)
index 0000000..2de2b25
--- /dev/null
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.vxquery.rest.service.Status;
+
+/**
+ * <pre>
+ *     {@code
+ *     <errorResponse>
+ *         <status>FATAL</status>
+ *         <requestId>jabsa-jkk77j-hbah45-jknasj-jjlas</requestId>
+ *         <error>
+ *             <code>405</code>
+ *             <message>Invalid Input</message>
+ *         </error>
+ *     </errorResponse>
+ *     }
+ * </pre>
+ *
+ * @author Erandi Ganepola
+ */
+@XmlRootElement
+public class ErrorResponse extends APIResponse {
+
+    private Error error;
+
+    public ErrorResponse() {
+        super(Status.FATAL.toString());
+    }
+
+    public Error getError() {
+        return error;
+    }
+
+    public void setError(Error error) {
+        this.error = error;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java
new file mode 100644 (file)
index 0000000..e34e1c6
--- /dev/null
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+public class Metrics {
+    private long compileTime;
+    private long elapsedTime;
+
+    public long getCompileTime() {
+        return compileTime;
+    }
+
+    public void setCompileTime(long compileTime) {
+        this.compileTime = compileTime;
+    }
+
+    public long getElapsedTime() {
+        return elapsedTime;
+    }
+
+    public void setElapsedTime(long elapsedTime) {
+        this.elapsedTime = elapsedTime;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java
new file mode 100644 (file)
index 0000000..4916ffe
--- /dev/null
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import org.apache.vxquery.rest.service.Status;
+
+/**
+ * The base class of the query response (the response returned when a query is
+ * sent for execution)
+ * 
+ * @author Erandi Ganepola
+ */
+public class QueryResponse extends APIResponse {
+
+    private String statement;
+    private String abstractSyntaxTree;
+    private String translatedExpressionTree;
+    private String optimizedExpressionTree;
+    private String runtimePlan;
+    private Metrics metrics = new Metrics();
+
+    public QueryResponse() {
+        super(Status.SUCCESS.toString());
+    }
+
+    public String getStatement() {
+        return statement;
+    }
+
+    public void setStatement(String statement) {
+        this.statement = statement;
+    }
+
+    public String getAbstractSyntaxTree() {
+        return abstractSyntaxTree;
+    }
+
+    public void setAbstractSyntaxTree(String abstractSyntaxTree) {
+        this.abstractSyntaxTree = abstractSyntaxTree;
+    }
+
+    public String getTranslatedExpressionTree() {
+        return translatedExpressionTree;
+    }
+
+    public void setTranslatedExpressionTree(String translatedExpressionTree) {
+        this.translatedExpressionTree = translatedExpressionTree;
+    }
+
+    public String getOptimizedExpressionTree() {
+        return optimizedExpressionTree;
+    }
+
+    public void setOptimizedExpressionTree(String optimizedExpressionTree) {
+        this.optimizedExpressionTree = optimizedExpressionTree;
+    }
+
+    public String getRuntimePlan() {
+        return runtimePlan;
+    }
+
+    public void setRuntimePlan(String runtimePlan) {
+        this.runtimePlan = runtimePlan;
+    }
+
+    public Metrics getMetrics() {
+        return metrics;
+    }
+
+    public void setMetrics(Metrics metrics) {
+        this.metrics = metrics;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java
new file mode 100644 (file)
index 0000000..2f74865
--- /dev/null
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.vxquery.rest.service.Status;
+
+@XmlRootElement
+public class QueryResultResponse extends APIResponse {
+
+    private String results;
+    private Metrics metrics = new Metrics();
+
+    public QueryResultResponse() {
+        super(Status.SUCCESS.toString());
+    }
+
+    public String getResults() {
+        return results;
+    }
+
+    public void setResults(String results) {
+        this.results = results;
+    }
+
+    public Metrics getMetrics() {
+        return metrics;
+    }
+
+    public void setMetrics(Metrics metrics) {
+        this.metrics = metrics;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java
new file mode 100644 (file)
index 0000000..b42a912
--- /dev/null
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+public class SyncQueryResponse extends QueryResponse {
+
+    private String results;
+
+    public String getResults() {
+        return results;
+    }
+
+    public void setResults(String results) {
+        this.results = results;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java
new file mode 100644 (file)
index 0000000..9dc967f
--- /dev/null
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.service;
+
+import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.job.JobId;
+
+/**
+ * A class to map {@link ResultSetId} with {@link JobId} when a job is submitted
+ * to hyracks. This mapping will later be used to determine the {@link JobId}
+ * instance of the corresponding {@link ResultSetId}
+ *
+ * @author Erandi Ganepola
+ */
+public class HyracksJobContext {
+
+    private JobId jobId;
+    private int frameSize;
+    private ResultSetId resultSetId;
+
+    public HyracksJobContext(JobId jobId, int frameSize, ResultSetId resultSetId) {
+        this.jobId = jobId;
+        this.frameSize = frameSize;
+        this.resultSetId = resultSetId;
+    }
+
+    public JobId getJobId() {
+        return jobId;
+    }
+
+    public int getFrameSize() {
+        return frameSize;
+    }
+
+    public ResultSetId getResultSetId() {
+        return resultSetId;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java
new file mode 100644 (file)
index 0000000..d6b4b3c
--- /dev/null
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.service;
+
+/**
+ * An enum to represent states of {@link VXQueryService} class
+ *
+ * @author Erandi Ganepola
+ */
+public enum State {
+    STARTING,
+    STARTED,
+    STOPPING,
+    STOPPED
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java
new file mode 100644 (file)
index 0000000..5757ae7
--- /dev/null
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.service;
+
+/**
+ * There can only 2 states for a response from the
+ * {@link org.apache.vxquery.rest.RestServer}. They are,
+ * 
+ * <pre>
+ * SUCCESS
+ * </pre>
+ * 
+ * and
+ * 
+ * <pre>
+ * FATAL
+ * </pre>
+ * 
+ * . This enum represents those two types.
+ *
+ * @author Erandi Ganepola
+ */
+public enum Status {
+    SUCCESS("success"),
+    FATAL("fatal");
+
+    private final String name;
+
+    Status(String name) {
+        this.name = name;
+    }
+
+    public String toString() {
+        return name;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java
new file mode 100644 (file)
index 0000000..4f20c6b
--- /dev/null
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.service;
+
+/**
+ * A class to store default/user specified configurations required at runtime by
+ * the {@link VXQueryService} class. These configuration will be loaded through
+ * a properties file.
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryConfig {
+
+    /** Number of available processors. (default: java's available processors) */
+    private int availableProcessors = Runtime.getRuntime().availableProcessors();
+    /** Setting frame size. (default: 65,536) */
+    private int frameSize = 65536;
+    /** Join hash size in bytes. (default: 67,108,864) */
+    private long joinHashSize = -1;
+    /** Maximum possible data size in bytes. (default: 150,323,855,000) */
+    private long maximumDataSize = -1;
+    /** Directory path to Hadoop configuration files */
+    private String hdfsConf = null;
+
+    private String hyracksClientIp;
+    private int hyracksClientPort;
+
+    public int getAvailableProcessors() {
+        return availableProcessors;
+    }
+
+    public void setAvailableProcessors(int availableProcessors) {
+        if (availableProcessors > 0) {
+            this.availableProcessors = availableProcessors;
+        }
+    }
+
+    public long getJoinHashSize() {
+        return joinHashSize;
+    }
+
+    public void setJoinHashSize(long joinHashSize) {
+        this.joinHashSize = joinHashSize;
+    }
+
+    public long getMaximumDataSize() {
+        return maximumDataSize;
+    }
+
+    public void setMaximumDataSize(long maximumDataSize) {
+        this.maximumDataSize = maximumDataSize;
+    }
+
+    public String getHdfsConf() {
+        return hdfsConf;
+    }
+
+    public void setHdfsConf(String hdfsConf) {
+        this.hdfsConf = hdfsConf;
+    }
+
+    public int getHyracksClientPort() {
+        return hyracksClientPort;
+    }
+
+    public void setHyracksClientPort(int hyracksClientPort) {
+        this.hyracksClientPort = hyracksClientPort;
+    }
+
+    public String getHyracksClientIp() {
+        return hyracksClientIp;
+    }
+
+    public void setHyracksClientIp(String hyracksClientIp) {
+        this.hyracksClientIp = hyracksClientIp;
+    }
+
+    public int getFrameSize() {
+        return frameSize;
+    }
+
+    public void setFrameSize(int frameSize) {
+        this.frameSize = frameSize;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java
new file mode 100644 (file)
index 0000000..1d51b6a
--- /dev/null
@@ -0,0 +1,482 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.rest.service;
+
+import static java.util.logging.Level.SEVERE;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.NOT_FOUND;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.PROBLEM_WITH_QUERY;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.UNFORSEEN_PROBLEM;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
+import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import org.apache.hyracks.api.client.HyracksConnection;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.client.NodeControllerInfo;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameTupleAccessor;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.dataset.DatasetJobRecord;
+import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
+import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.job.JobFlag;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.control.nc.resources.memory.FrameManager;
+import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
+import org.apache.vxquery.compiler.CompilerControlBlock;
+import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory;
+import org.apache.vxquery.compiler.algebricks.prettyprint.VXQueryLogicalExpressionPrettyPrintVisitor;
+import org.apache.vxquery.context.DynamicContext;
+import org.apache.vxquery.context.DynamicContextImpl;
+import org.apache.vxquery.context.RootStaticContextImpl;
+import org.apache.vxquery.context.StaticContextImpl;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.exceptions.VXQueryRuntimeException;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.Error;
+import org.apache.vxquery.rest.response.QueryResponse;
+import org.apache.vxquery.rest.response.QueryResultResponse;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.result.ResultUtils;
+import org.apache.vxquery.xmlquery.ast.ModuleNode;
+import org.apache.vxquery.xmlquery.query.Module;
+import org.apache.vxquery.xmlquery.query.XMLQueryCompiler;
+import org.apache.vxquery.xmlquery.query.XQueryCompilationListener;
+
+import com.thoughtworks.xstream.XStream;
+import com.thoughtworks.xstream.io.xml.DomDriver;
+
+/**
+ * Main class responsible for handling query requests. This class will first
+ * compile, then submit query to hyracks and finally fetch results for a given
+ * query.
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryService {
+
+    private static final Logger LOGGER = Logger.getLogger(VXQueryService.class.getName());
+
+    private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})");
+
+    private volatile State state = State.STOPPED;
+    private VXQueryConfig vxQueryConfig;
+    private AtomicLong atomicLong = new AtomicLong(0);
+    private Map<Long, HyracksJobContext> jobContexts = new ConcurrentHashMap<>();
+    private IHyracksClientConnection hyracksClientConnection;
+    private HyracksDataset hyracksDataset;
+
+    public VXQueryService(VXQueryConfig config) {
+        vxQueryConfig = config;
+    }
+
+    /**
+     * Starts VXQueryService class by creating a {@link IHyracksClientConnection}
+     * which will later be used to submit and retrieve queries and results to/from
+     * hyracks.
+     */
+    public synchronized void start() {
+        if (!State.STOPPED.equals(state)) {
+            throw new IllegalStateException("VXQueryService is at state : " + state);
+        }
+
+        if (vxQueryConfig.getHyracksClientIp() == null) {
+            throw new IllegalArgumentException("hyracksClientIp is required to connect to hyracks");
+        }
+
+        setState(State.STARTING);
+
+        try {
+            hyracksClientConnection =
+                    new HyracksConnection(vxQueryConfig.getHyracksClientIp(), vxQueryConfig.getHyracksClientPort());
+        } catch (Exception e) {
+            LOGGER.log(SEVERE, String.format("Unable to create a hyracks client connection to %s:%d",
+                    vxQueryConfig.getHyracksClientIp(), vxQueryConfig.getHyracksClientPort()));
+            throw new VXQueryRuntimeException("Unable to create a hyracks client connection", e);
+        }
+
+        LOGGER.log(Level.FINE, String.format("Using hyracks connection to %s:%d", vxQueryConfig.getHyracksClientIp(),
+                vxQueryConfig.getHyracksClientPort()));
+
+        setState(State.STARTED);
+        LOGGER.log(Level.INFO, "VXQueryService started successfully");
+    }
+
+    private synchronized void setState(State newState) {
+        state = newState;
+    }
+
+    /**
+     * Submits a query to hyracks to be run after compiling. Required intermediate
+     * results and metrics are also calculated according to the
+     * {@link QueryRequest}. Checks if this class has started before moving further.
+     *
+     * @param request
+     *            {@link QueryRequest} containing information about the query to be
+     *            executed and the merics required along with the results
+     * @return AsyncQueryResponse if no error occurs | ErrorResponse else
+     */
+    public APIResponse execute(final QueryRequest request) {
+        QueryRequest indexingRequest = new QueryRequest("show-indexes()");
+        indexingRequest.setAsync(false);
+        SyncQueryResponse indexingResponse = (SyncQueryResponse) execute(indexingRequest, new ArrayList<>());
+        LOGGER.log(Level.FINE, String.format("Found indexes: %s", indexingResponse.getResults()));
+
+        List<String> collections = Arrays.asList(indexingResponse.getResults().split("\n"));
+        return execute(request, collections);
+    }
+
+    private APIResponse execute(final QueryRequest request, List<String> collections) {
+        if (!State.STARTED.equals(state)) {
+            throw new IllegalStateException("VXQueryService is at state : " + state);
+        }
+
+        String query = request.getStatement();
+        final ResultSetId resultSetId = createResultSetId();
+
+        QueryResponse response = APIResponse.newQueryResponse(request, resultSetId);
+        response.setStatement(query);
+
+        // Obtaining the node controller information from hyracks client connection
+        Map<String, NodeControllerInfo> nodeControllerInfos = null;
+        try {
+            nodeControllerInfos = hyracksClientConnection.getNodeControllerInfos();
+        } catch (HyracksException e) {
+            LOGGER.log(Level.SEVERE, String.format("Error occurred when obtaining NC info: '%s'", e.getMessage()));
+            return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(UNFORSEEN_PROBLEM)
+                    .withMessage("Hyracks connection problem: " + e.getMessage()).build());
+        }
+
+        // Adding a query compilation listener
+        VXQueryCompilationListener listener = new VXQueryCompilationListener(response,
+                request.isShowAbstractSyntaxTree(), request.isShowTranslatedExpressionTree(),
+                request.isShowOptimizedExpressionTree(), request.isShowRuntimePlan());
+
+        Date start = new Date();
+        // Compiling the XQuery given
+        final XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, request.getFrameSize(),
+                vxQueryConfig.getAvailableProcessors(), vxQueryConfig.getJoinHashSize(),
+                vxQueryConfig.getMaximumDataSize(), vxQueryConfig.getHdfsConf());
+        CompilerControlBlock compilerControlBlock = new CompilerControlBlock(
+                new StaticContextImpl(RootStaticContextImpl.INSTANCE), resultSetId, request.getSourceFileMap());
+        try {
+            compiler.compile(null, new StringReader(query), compilerControlBlock, request.getOptimization(),
+                    collections);
+        } catch (AlgebricksException e) {
+            LOGGER.log(Level.SEVERE, String.format("Error occurred when compiling query: '%s' with message: '%s'",
+                    query, e.getMessage()));
+            return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(PROBLEM_WITH_QUERY)
+                    .withMessage("Query compilation failure: " + e.getMessage()).build());
+        } catch (SystemException e) {
+            LOGGER.log(Level.SEVERE, String.format("Error occurred when compiling query: '%s' with message: '%s'",
+                    query, e.getMessage()));
+            return APIResponse.newErrorResponse(request.getRequestId(),
+                    new Error(PROBLEM_WITH_QUERY, "Query compilation failure: " + e.getCode()));
+        }
+
+        if (request.isShowMetrics()) {
+            response.getMetrics().setCompileTime(new Date().getTime() - start.getTime());
+        }
+
+        if (request.isCompileOnly()) {
+            return response;
+        }
+
+        Module module = compiler.getModule();
+        JobSpecification js = module.getHyracksJobSpecification();
+        DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
+        js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
+
+        HyracksJobContext hyracksJobContext;
+        start = new Date();
+        if (!request.isAsync()) {
+            for (int i = 0; i < request.getRepeatExecutions(); i++) {
+                try {
+                    hyracksJobContext = executeJob(js, resultSetId, request);
+
+                } catch (Exception e) {
+                    LOGGER.log(SEVERE, "Error occurred when submitting job to hyracks for query: " + query, e);
+                    return APIResponse.newErrorResponse(request.getRequestId(),
+                            Error.builder().withCode(UNFORSEEN_PROBLEM)
+                                    .withMessage("Error occurred when starting hyracks job").build());
+                }
+                try {
+                    String results = readResults(hyracksJobContext);
+                    ((SyncQueryResponse) response).setResults(results);
+                } catch (HyracksException e) {
+                    LOGGER.log(Level.SEVERE, "Error occurred when reading results", e);
+                    SystemException se = getSystemException(e);
+                    return APIResponse.newErrorResponse(request.getRequestId(), new Error(UNFORSEEN_PROBLEM,
+                            String.format("Error occurred when reading results: %s", se != null ? se.getCode() : "")));
+                } catch (Exception e) {
+                    LOGGER.log(Level.SEVERE, "Error occurred when reading results", e);
+                    return APIResponse.newErrorResponse(request.getRequestId(),
+                            new Error(UNFORSEEN_PROBLEM, "Error occurred when reading results: " + e.getMessage()));
+                }
+            }
+        } else {
+            try {
+                hyracksJobContext = executeJob(js, resultSetId, request);
+            } catch (Exception e) {
+                LOGGER.log(SEVERE, "Error occurred when submitting job to hyracks for query: " + query, e);
+                return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(UNFORSEEN_PROBLEM)
+                        .withMessage("Error occurred when starting hyracks job").build());
+            }
+            jobContexts.put(resultSetId.getId(), hyracksJobContext);
+        }
+
+        if (request.isShowMetrics()) {
+            response.getMetrics().setElapsedTime(new Date().getTime() - start.getTime());
+        }
+
+        return response;
+    }
+
+    private HyracksJobContext executeJob(JobSpecification js, ResultSetId resultSetId, QueryRequest request)
+            throws Exception {
+        HyracksJobContext hyracksJobContext;
+        JobId jobId = hyracksClientConnection.startJob(js, EnumSet.of(JobFlag.PROFILE_RUNTIME));
+        hyracksJobContext = new HyracksJobContext(jobId, js.getFrameSize(), resultSetId);
+
+        return hyracksJobContext;
+    }
+
+    private static SystemException getSystemException(HyracksException e) {
+        Throwable t = e;
+        Throwable candidate = t instanceof SystemException ? t : null;
+        while (t.getCause() != null) {
+            t = t.getCause();
+            if (t instanceof SystemException) {
+                candidate = t;
+            }
+        }
+
+        t = candidate == null ? t : candidate;
+        final String message = t.getMessage();
+        if (message != null) {
+            Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(message);
+            if (m.find()) {
+                String eCode = m.group(1);
+                return new SystemException(ErrorCode.valueOf(eCode), e);
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Returns the query results for a given result set id.
+     *
+     * @param request
+     *            {@link QueryResultRequest} with result ID required
+     * @return Either a {@link QueryResultResponse} if no error occurred |
+     *         {@link org.apache.vxquery.rest.response.ErrorResponse} else.
+     */
+    public APIResponse getResult(QueryResultRequest request) {
+        if (jobContexts.containsKey(request.getResultId())) {
+            QueryResultResponse resultResponse = APIResponse.newQueryResultResponse(request.getRequestId());
+            Date start = new Date();
+            try {
+                String results = readResults(jobContexts.get(request.getResultId()));
+                resultResponse.setResults(results);
+            } catch (Exception e) {
+                LOGGER.log(Level.SEVERE, "Error occurred when reading results for id : " + request.getResultId());
+                return APIResponse.newErrorResponse(request.getRequestId(), new Error(UNFORSEEN_PROBLEM,
+                        "Error occurred when reading results for: " + request.getResultId()));
+            }
+
+            if (request.isShowMetrics()) {
+                resultResponse.getMetrics().setElapsedTime(new Date().getTime() - start.getTime());
+            }
+
+            return resultResponse;
+        } else {
+            return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(NOT_FOUND)
+                    .withMessage("No query found for result ID : " + request.getResultId()).build());
+        }
+    }
+
+    /**
+     * Reads results from hyracks given the {@link HyracksJobContext} containing
+     * {@link ResultSetId} and {@link JobId} mapping.
+     *
+     * @param jobContext
+     *            mapoing between the {@link ResultSetId} and corresponding hyracks
+     *            {@link JobId}
+     * @return Results of the given query
+     * @throws Exception
+     *             IOErrors and etc
+     */
+    private String readResults(HyracksJobContext jobContext) throws Exception {
+        int nReaders = 1;
+
+        if (hyracksDataset == null) {
+            hyracksDataset = new HyracksDataset(hyracksClientConnection, jobContext.getFrameSize(), nReaders);
+        }
+
+        FrameManager resultDisplayFrameMgr = new FrameManager(jobContext.getFrameSize());
+        IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
+        IHyracksDatasetReader reader = hyracksDataset.createReader(jobContext.getJobId(), jobContext.getResultSetId());
+        OutputStream resultStream = new ByteArrayOutputStream();
+
+        // This loop is required for XTests to reliably identify the error code of
+        // SystemException.
+        while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) {
+            Thread.sleep(100);
+        }
+
+        IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
+        try (PrintWriter writer = new PrintWriter(resultStream, true)) {
+            while (reader.read(frame) > 0) {
+                writer.print(ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor));
+                writer.flush();
+                frame.getBuffer().clear();
+            }
+        }
+
+        hyracksClientConnection.waitForCompletion(jobContext.getJobId());
+        LOGGER.log(Level.FINE, String.format("Result for resultId %d completed", jobContext.getResultSetId().getId()));
+        return resultStream.toString();
+    }
+
+    /**
+     * Create a unique result set id to get the correct query back from the cluster.
+     *
+     * @return Result Set id generated with current system time.
+     */
+    protected ResultSetId createResultSetId() {
+        long resultSetId = atomicLong.incrementAndGet();
+        LOGGER.log(Level.FINE, String.format("Creating result set with ID : %d", resultSetId));
+        return new ResultSetId(resultSetId);
+    }
+
+    public synchronized void stop() {
+        if (!State.STOPPED.equals(state)) {
+            setState(State.STOPPING);
+            LOGGER.log(Level.FINE, "Stooping VXQueryService");
+            setState(State.STOPPED);
+            LOGGER.log(Level.INFO, "VXQueryService stopped successfully");
+        } else {
+            LOGGER.log(Level.INFO, "VXQueryService is already in state : " + state);
+        }
+    }
+
+    public State getState() {
+        return state;
+    }
+
+    /**
+     * A {@link XQueryCompilationListener} implementation to be used to add
+     * AbstractSyntaxTree, RuntimePlan and etc to the {@link QueryResponse} if
+     * requested by the user.
+     */
+    private class VXQueryCompilationListener implements XQueryCompilationListener {
+        private QueryResponse response;
+        private boolean showAbstractSyntaxTree;
+        private boolean showTranslatedExpressionTree;
+        private boolean showOptimizedExpressionTree;
+        private boolean showRuntimePlan;
+
+        public VXQueryCompilationListener(QueryResponse response, boolean showAbstractSyntaxTree,
+                boolean showTranslatedExpressionTree, boolean showOptimizedExpressionTree, boolean showRuntimePlan) {
+            this.response = response;
+            this.showAbstractSyntaxTree = showAbstractSyntaxTree;
+            this.showTranslatedExpressionTree = showTranslatedExpressionTree;
+            this.showOptimizedExpressionTree = showOptimizedExpressionTree;
+            this.showRuntimePlan = showRuntimePlan;
+        }
+
+        @Override
+        public void notifyParseResult(ModuleNode moduleNode) {
+            if (showAbstractSyntaxTree) {
+                response.setAbstractSyntaxTree(new XStream(new DomDriver()).toXML(moduleNode));
+            }
+        }
+
+        @Override
+        public void notifyTranslationResult(Module module) {
+            if (showTranslatedExpressionTree) {
+                response.setTranslatedExpressionTree(appendPrettyPlan(new StringBuilder(), module).toString());
+            }
+        }
+
+        @Override
+        public void notifyTypecheckResult(Module module) {
+        }
+
+        @Override
+        public void notifyCodegenResult(Module module) {
+            if (showRuntimePlan) {
+                JobSpecification jobSpec = module.getHyracksJobSpecification();
+                try {
+                    response.setRuntimePlan(jobSpec.toJSON().toString());
+                } catch (IOException e) {
+                    LOGGER.log(SEVERE,
+                            "Error occurred when obtaining runtime plan from job specification : " + jobSpec.toString(),
+                            e);
+                }
+            }
+        }
+
+        @Override
+        public void notifyOptimizedResult(Module module) {
+            if (showOptimizedExpressionTree) {
+                response.setOptimizedExpressionTree(appendPrettyPlan(new StringBuilder(), module).toString());
+            }
+        }
+
+        @SuppressWarnings("Duplicates")
+        private StringBuilder appendPrettyPlan(StringBuilder sb, Module module) {
+            try {
+                ILogicalExpressionVisitor<String, Integer> ev =
+                        new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext());
+                AlgebricksAppendable buffer = new AlgebricksAppendable();
+                LogicalOperatorPrettyPrintVisitor v = new LogicalOperatorPrettyPrintVisitor(buffer, ev);
+                PlanPrettyPrinter.printPlan(module.getBody(), v, 0);
+                sb.append(buffer.toString());
+            } catch (AlgebricksException e) {
+                LOGGER.log(SEVERE, "Error occurred when pretty printing expression : " + e.getMessage());
+            }
+            return sb;
+        }
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java
new file mode 100644 (file)
index 0000000..45ef910
--- /dev/null
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.rest.servlet;
+
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
+import static org.apache.vxquery.rest.Constants.MODE_ASYNC;
+import static org.apache.vxquery.rest.Constants.MODE_SYNC;
+import static org.apache.vxquery.rest.Constants.Parameters.COMPILE_ONLY;
+import static org.apache.vxquery.rest.Constants.Parameters.FRAME_SIZE;
+import static org.apache.vxquery.rest.Constants.Parameters.METRICS;
+import static org.apache.vxquery.rest.Constants.Parameters.MODE;
+import static org.apache.vxquery.rest.Constants.Parameters.OPTIMIZATION;
+import static org.apache.vxquery.rest.Constants.Parameters.REPEAT_EXECUTIONS;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_AST;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_OET;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_RP;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_TET;
+import static org.apache.vxquery.rest.Constants.Parameters.STATEMENT;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.AbstractMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.Constants;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.Error;
+import org.apache.vxquery.rest.service.VXQueryService;
+
+/**
+ * Servlet to handle query requests.
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryAPIServlet extends RestAPIServlet {
+
+    private VXQueryService vxQueryService;
+
+    public QueryAPIServlet(VXQueryService vxQueryService, ConcurrentMap<String, Object> ctx, String... paths) {
+        super(ctx, paths);
+        this.vxQueryService = vxQueryService;
+    }
+
+    @Override
+    protected APIResponse doHandle(IServletRequest request) {
+        LOGGER.log(Level.INFO,
+                String.format("Received a query request with query : %s", request.getParameter("statement")));
+
+        QueryRequest queryRequest;
+        try {
+            queryRequest = getQueryRequest(request);
+        } catch (Exception e) {
+            return APIResponse.newErrorResponse(null,
+                    Error.builder().withCode(Constants.ErrorCodes.INVALID_INPUT).withMessage("Invalid input").build());
+        }
+
+        try {
+            return vxQueryService.execute(queryRequest);
+        } catch (Exception e) {
+            LOGGER.log(Level.SEVERE, "Error occurred when trying to execute query : " + queryRequest.getStatement(), e);
+            return APIResponse.newErrorResponse(queryRequest.getRequestId(), Error.builder()
+                    .withCode(Constants.ErrorCodes.UNFORSEEN_PROBLEM).withMessage(e.getMessage()).build());
+        }
+    }
+
+    private QueryRequest getQueryRequest(IServletRequest request) throws IOException, JAXBException {
+        if (request.getParameter(STATEMENT) == null || request.getParameter(STATEMENT).trim().isEmpty()) {
+            throw new IllegalArgumentException("Parameter 'statement' is required to handle the request");
+        }
+
+        QueryRequest queryRequest = new QueryRequest(UUID.randomUUID().toString(), request.getParameter(STATEMENT));
+        queryRequest.setCompileOnly(Boolean.parseBoolean(request.getParameter(COMPILE_ONLY)));
+        queryRequest.setShowMetrics(Boolean.parseBoolean(request.getParameter(METRICS)));
+
+        queryRequest.setShowAbstractSyntaxTree(Boolean.parseBoolean(request.getParameter(SHOW_AST)));
+        queryRequest.setShowTranslatedExpressionTree(Boolean.parseBoolean(request.getParameter(SHOW_TET)));
+        queryRequest.setShowOptimizedExpressionTree(Boolean.parseBoolean(request.getParameter(SHOW_OET)));
+        queryRequest.setShowRuntimePlan(Boolean.parseBoolean(request.getParameter(SHOW_RP)));
+
+        if (request.getParameter(OPTIMIZATION) != null) {
+            queryRequest.setOptimization(Integer.parseInt(request.getParameter(OPTIMIZATION)));
+        }
+        if (request.getParameter(FRAME_SIZE) != null) {
+            queryRequest.setFrameSize(Integer.parseInt(request.getParameter(FRAME_SIZE)));
+        }
+        if (request.getParameter(REPEAT_EXECUTIONS) != null) {
+            queryRequest.setRepeatExecutions(Integer.parseInt(request.getParameter(REPEAT_EXECUTIONS)));
+        }
+
+        String sourceFileMap = request.getHttpRequest().content().toString(StandardCharsets.UTF_8);
+        if (sourceFileMap != null && !sourceFileMap.isEmpty()) {
+            Map<String, String> map = (Map<String, String>) RestUtils.mapEntity(sourceFileMap, Map.class,
+                    request.getHeader(CONTENT_TYPE));
+            LOGGER.log(Level.FINE, "Found source file map");
+            Map<String, File> fileMap = map.entrySet().stream()
+                    .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), new File(entry.getValue())))
+                    .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
+            queryRequest.setSourceFileMap(fileMap);
+        }
+
+        if (request.getParameter(MODE) != null) {
+            switch (request.getParameter(MODE)) {
+                case MODE_SYNC:
+                    queryRequest.setAsync(false);
+                    break;
+                case MODE_ASYNC:
+                default:
+                    queryRequest.setAsync(true);
+                    break;
+            }
+        }
+
+        return queryRequest;
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java
new file mode 100644 (file)
index 0000000..84b0187
--- /dev/null
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.servlet;
+
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.logging.Level;
+
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.vxquery.rest.Constants;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.Error;
+import org.apache.vxquery.rest.service.VXQueryService;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * Servlet to handle query results requests.
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryResultAPIServlet extends RestAPIServlet {
+
+    private VXQueryService vxQueryService;
+
+    public QueryResultAPIServlet(VXQueryService vxQueryService, ConcurrentMap<String, Object> ctx, String... paths) {
+        super(ctx, paths);
+        this.vxQueryService = vxQueryService;
+    }
+
+    @Override
+    protected APIResponse doHandle(IServletRequest request) {
+        String uri = request.getHttpRequest().uri();
+        long resultId;
+        try {
+            String pathParam = uri.substring(uri.lastIndexOf("/") + 1);
+            pathParam = pathParam.contains("?") ? pathParam.split("\\?")[0] : pathParam;
+            resultId = Long.parseLong(pathParam);
+        } catch (NumberFormatException e) {
+            LOGGER.log(Level.SEVERE, "Result ID could not be retrieved from URL");
+            return APIResponse.newErrorResponse(null, Error.builder().withCode(HttpResponseStatus.BAD_REQUEST.code())
+                    .withMessage("Result ID couldn't be retrieved from URL").build());
+        }
+
+        QueryResultRequest resultRequest = new QueryResultRequest(resultId, UUID.randomUUID().toString());
+        resultRequest.setShowMetrics(Boolean.parseBoolean(request.getParameter(Constants.Parameters.METRICS)));
+        LOGGER.log(Level.INFO,
+                String.format("Received a result request with resultId : %d", resultRequest.getResultId()));
+        return vxQueryService.getResult(resultRequest);
+    }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java
new file mode 100644 (file)
index 0000000..bc93dfc
--- /dev/null
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.servlet;
+
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.concurrent.ConcurrentMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+
+import org.apache.htrace.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.api.IServletResponse;
+import org.apache.hyracks.http.server.AbstractServlet;
+import org.apache.hyracks.http.server.utils.HttpUtil;
+import org.apache.vxquery.exceptions.VXQueryRuntimeException;
+import org.apache.vxquery.exceptions.VXQueryServletRuntimeException;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.AsyncQueryResponse;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.apache.vxquery.rest.response.QueryResultResponse;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.rest.service.Status;
+
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * Abstract servlet to handle REST API requests.
+ *
+ * @author Erandi Ganepola
+ */
+public abstract class RestAPIServlet extends AbstractServlet {
+
+    protected final Logger LOGGER;
+
+    private JAXBContext jaxbContext;
+
+    public RestAPIServlet(ConcurrentMap<String, Object> ctx, String... paths) {
+        super(ctx, paths);
+        LOGGER = Logger.getLogger(this.getClass().getName());
+        try {
+            jaxbContext = JAXBContext.newInstance(QueryResultResponse.class, AsyncQueryResponse.class,
+                    SyncQueryResponse.class, ErrorResponse.class);
+        } catch (JAXBException e) {
+            LOGGER.log(Level.SEVERE, "Error occurred when creating JAXB context", e);
+            throw new VXQueryRuntimeException("Unable to load JAXBContext", e);
+        }
+    }
+
+    @Override
+    protected final void post(IServletRequest request, IServletResponse response) {
+        getOrPost(request, response);
+    }
+
+    @Override
+    protected final void get(IServletRequest request, IServletResponse response) {
+        getOrPost(request, response);
+    }
+
+    private void getOrPost(IServletRequest request, IServletResponse response) {
+        try {
+            initResponse(request, response);
+            APIResponse entity = doHandle(request);
+            if (entity == null) {
+                LOGGER.log(Level.WARNING, "No entity found for request : " + request);
+                response.setStatus(HttpResponseStatus.BAD_REQUEST);
+            } else {
+                // Important to set Status OK before setting the entity because the response
+                // (chunked) checks it before
+                // writing the response to channel.
+                setResponseStatus(response, entity);
+                setEntity(request, response, entity);
+            }
+        } catch (IOException e) {
+            response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+            LOGGER.log(Level.SEVERE, "Error occurred when setting content type", e);
+        }
+    }
+
+    private void initResponse(IServletRequest request, IServletResponse response) throws IOException {
+        // enable cross-origin resource sharing
+        response.setHeader("Access-Control-Allow-Origin", "*");
+        response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+
+        HttpUtil.setContentType(response, "text/plain");
+    }
+
+    private void setEntity(IServletRequest request, IServletResponse response, APIResponse entity) throws IOException {
+        String accept = request.getHeader(HttpHeaderNames.ACCEPT, "");
+        String entityString;
+        switch (accept) {
+            case CONTENT_TYPE_XML:
+                try {
+                    HttpUtil.setContentType(response, CONTENT_TYPE_XML);
+
+                    Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+                    jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+                    StringWriter sw = new StringWriter();
+                    jaxbMarshaller.marshal(entity, sw);
+                    entityString = sw.toString();
+                } catch (JAXBException e) {
+                    LOGGER.log(Level.SEVERE, "Error occurred when mapping java object into xml", e);
+                    throw new VXQueryServletRuntimeException("Error occurred when marshalling entity", e);
+                }
+                break;
+            case CONTENT_TYPE_JSON:
+            default:
+                try {
+                    HttpUtil.setContentType(response, CONTENT_TYPE_JSON);
+                    ObjectMapper jsonMapper = new ObjectMapper();
+                    entityString = jsonMapper.writeValueAsString(entity);
+                } catch (JsonProcessingException e) {
+                    LOGGER.log(Level.SEVERE, "Error occurred when mapping java object into JSON", e);
+                    throw new VXQueryServletRuntimeException("Error occurred when mapping entity", e);
+                }
+                break;
+        }
+
+        response.writer().print(entityString);
+    }
+
+    private void setResponseStatus(IServletResponse response, APIResponse entity) {
+        if (Status.SUCCESS.toString().equals(entity.getStatus())) {
+            response.setStatus(HttpResponseStatus.OK);
+        } else if (Status.FATAL.toString().equals(entity.getStatus())) {
+            HttpResponseStatus status = HttpResponseStatus.INTERNAL_SERVER_ERROR;
+            if (entity instanceof ErrorResponse) {
+                status = HttpResponseStatus.valueOf(((ErrorResponse) entity).getError().getCode());
+            }
+            response.setStatus(status);
+        }
+    }
+
+    /**
+     * This abstract method is supposed to return an object which will be the entity
+     * of the response being sent to the client. Implementing classes doesn't have
+     * to worry about the content type of the request.
+     *
+     * @param request
+     *            {@link IServletRequest} received
+     * @return Object to be set as the entity of the response
+     */
+    protected abstract APIResponse doHandle(IServletRequest request);
+}
diff --git a/vxquery-rest/src/site/markdown/index.md b/vxquery-rest/src/site/markdown/index.md
new file mode 100644 (file)
index 0000000..703df65
--- /dev/null
@@ -0,0 +1,249 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# VXQuery REST Server
+
+VXQuery REST Server allows users to submit queries and receive results either synchronously or
+asynchronously through the exposed REST API. Along with the statement to be executed, few other parameters can be given as
+well. Complete REST API specification can be found at [REST API Specification](specification.html).
+
+## Installation
+
+No additional steps needed to be taken to get the REST Server up and running. That is, setting up a VXQuery cluster will
+automatically start the REST Server on port `8080`. Please see [VXQuery Cluster Setup](../user_cluster_installation.html)
+to understand how a VXQuery cluster is setup.
+
+## Getting Started
+
+Suppose we want to execute a very simple XQuery like:
+
+```
+for $x in (1, 2.0, 3) return $x
+```
+
+### Async (Default Mode) Example
+
+If we want to send this, following will be the plain HTTP request.
+
+```
+GET http://127.0.1.1:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x HTTP/1.1
+```
+
+Note the query parameter `statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x` in which the above mentioned statement
+has been encoded. If we send this request using **cURL**, it will look like follows.
+
+#### Accept: application/json
+
+```
+curl -i -H "Accept: application/json" -X GET "http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/json
+content-length: 320
+
+{
+  "status": "success",
+  "requestId": "b0cbe06f-3454-4422-ba23-59150e1c1400",
+  "statement": "for $x in (1, 2.0, 3) return $x",
+  "abstractSyntaxTree": null,
+  "translatedExpressionTree": null,
+  "optimizedExpressionTree": null,
+  "runtimePlan": null,
+  "metrics": {
+    "compileTime": 0,
+    "elapsedTime": 0
+  },
+  "resultId": 6,
+  "resultUrl": "/vxquery/query/result/6"
+}
+```
+
+#### Accept: application/xml
+
+```
+curl -i -H "Accept: application/xml" -X GET "http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/xml
+content-length: 403
+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<asyncQueryResponse>
+    <requestId>d0c2c0ef-2e46-4153-9d4b-1ef4593184e7</requestId>
+    <metrics>
+        <compileTime>0</compileTime>
+        <elapsedTime>0</elapsedTime>
+    </metrics>
+    <statement>for $x in (1, 2.0, 3) return $x</statement>
+    <resultId>8</resultId>
+    <resultUrl>/vxquery/query/result/8</resultUrl>
+</asyncQueryResponse>
+```
+
+#### Result Fetching
+
+Since we have used the default mode (**async**), we only got the **resultId**. Now we have to send another request asking
+for the actual results. Send a cURL request to `/vxquery/query/result/8` to fetch results for result ID 8.
+
+##### Accept: application/json
+
+```
+curl -i -H "Accept: application/json" -X GET "http://localhost:39003/vxquery/query/result/8"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/json
+content-length: 137
+
+{
+  "status": "success",
+  "requestId": "d0c2c0ef-2e46-4153-9d4b-1ef4593184e7",
+  "results": "1\n2\n3\n",
+  "metrics": {
+    "compileTime": 0,
+    "elapsedTime": 0
+  }
+}
+```
+
+Note the *results* in the JSON content in the response.
+
+##### Accept: application/xml
+
+```
+curl -i -H "Accept: application/xml" -X GET "http://localhost:39003/vxquery/query/result/8"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/xml
+content-length: 298
+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<queryResultResponse>
+    <requestId>d0c2c0ef-2e46-4153-9d4b-1ef4593184e7</requestId>
+    <metrics>
+        <compileTime>0</compileTime>
+        <elapsedTime>0</elapsedTime>
+    </metrics>
+    <results>1
+2
+3
+</results>
+</queryResultResponse>
+```
+
+Note the *<results></results>* in the XML content in the response.
+
+### Sync (Synchronous Mode) Example
+
+Similarly to what we did under async requests, we can send the query requests here as well, but with the added query parameter
+`mode=sync` which is to indicate that the response should be a synchronous one. That is, we wait for the query to be 
+executed and the response to arrive.
+
+```
+curl -i -H "Accept: application/xml" -X GET \
+"http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x&mode=sync"
+```
+
+and the response now contains **results** instead of the **resultId** we received previously.
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/xml
+content-length: 353
+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<syncQueryResponse>
+    <requestId>93b67f50-4f14-4304-a9b2-f75b4a736df3</requestId>
+    <metrics>
+        <compileTime>0</compileTime>
+        <elapsedTime>0</elapsedTime>
+    </metrics>
+    <statement>for $x in (1, 2.0, 3) return $x</statement>
+    <results>1
+2
+3
+</results>
+</syncQueryResponse>
+```
+
+Similarly with `accept:application/json`,
+
+```
+curl -i -H "Accept: application/json" -X GET \
+"http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x&mode=sync"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/json
+content-length: 291
+
+{
+  "status": "success",
+  "requestId": "8010a699-a6f2-423c-91e1-8ac17cd5c5cd",
+  "statement": "for $x in (1, 2.0, 3) return $x",
+  "abstractSyntaxTree": null,
+  "translatedExpressionTree": null,
+  "optimizedExpressionTree": null,
+  "runtimePlan": null,
+  "metrics": {
+    "compileTime": 0,
+    "elapsedTime": 0
+  },
+  "results": "1\n2\n3\n"
+}
+```
diff --git a/vxquery-rest/src/site/markdown/specification.md b/vxquery-rest/src/site/markdown/specification.md
new file mode 100644 (file)
index 0000000..3977439
--- /dev/null
@@ -0,0 +1,102 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# REST API Specification
+
+Swagger configuration of the REST API can be found 
+[here](https://cwiki.apache.org/confluence/display/VXQUERY/SwaggerIO+Configuration). 
+
+**NOTE:** This REST API supports both **content types**, `application/json` and `application/xml`. Based on the `accept` header 
+of your query request, REST API will return the results wither in *json* form or *XML* form. Returned content type 
+defaults to `application/json` if no `accept` header is present.
+
+Base Path **${host}/vxquery**
+
+## Query Request
+
+Request of this type should be submitted for a given *query* to be executed. Depending on the value of the parameter
+`mode`, a **synchronous** response or an **asynchronous** response will be returned
+
+`*` required
+
+| Path | Method |Parameters | Type | Description |
+| ------ | ------ | ------ | ----- |----- |
+| /query | GET  |statement*     | string    | Statement to be executed |
+|       |       |mode           | string    | `sync` or `async`. **async** will return an asynchronous response **(default: async)** |
+|       |       |compileOnly    | boolean   | If `true`, statement will be compiled, but won't be executed (default: false) |
+|       |       |optimization   | integer   | Optimization level (0 - 2,147,483,647). (Default: Full optimization) |
+|       |       |frameSize      | integer   | Frame size in bytes (default: 65536) |
+|       |       |repeatExecutions|integer   | Number of times to repeat execution (default: 1) |
+|       |       |metrics        | boolean   | If `true`, returns metrics (compile and execution time) with the response (default: false) |
+|       |       |showAbstractSyntaxTree         | boolean | Shows abstract syntax tree if `true` (default: false) |
+|       |       |showTranslatedExpressionTree   | boolean | Shows translated expression tree if `true` (default: false) |
+|       |       |showOptimizedExpressionTree    | boolean | Shows optimized expression tree if `true` (default: false) |
+|       |       |showRuntimePlan| boolean   | Shows runtime plan if set to `true` (default: false) |
+
+### Synchronous Query Response
+
+Received only when `mode` is set to `sync` in the query request above. 
+
+| Attribute | Type | Description |
+| ------ | ------ | ------ |
+|statement* |string         | Statement submitted to be executed |
+|status*       |string         | `success` to indicate that the query execution was successful | 
+|requestId*    |string         | A unique ID assigned to the request sent earlier |
+|abstractSyntaxTree            |string | Abstract Syntax Tree if requested in the query request. Else `null` |
+|translatedExpressionTree    |string | Translated Expression Tree if requested in the query request. Else `null` |
+|optimizedExpressionTree           |string | Optimized Expression Tree if requested in the query request. Else `null` |
+|runtimePlan    |string     | Runtime plan if requested in the query request. Else `null` |
+|metrics           |metrics    | Metrics (`compileTime` and `elapsedTime`) if requested in the query request |
+|results*       |string     | Results of the query/statement submitted for execution |
+
+### Asynchronous Query Response
+
+Received only when `mode` is set to `async` (which is the default value) in the query request above. 
+
+| Attribute | Type | Description |
+| ------ | ------ | ------ |
+|statement* |string         | Statement submitted to be executed |
+|status*       |string         | `success` to indicate that the query execution was successful | 
+|requestId*    |string         | A unique ID assigned to the request sent earlier |
+|abstractSyntaxTree            |string | Abstract Syntax Tree if requested in the query request. Else `null` |
+|translatedExpressionTree    |string | Translated Expression Tree if requested in the query request. Else `null` |
+|optimizedExpressionTree           |string | Optimized Expression Tree if requested in the query request. Else `null` |
+|runtimePlan    |string     | Runtime plan if requested in the query request. Else `null` |
+|metrics           |metrics    | Metrics (`compileTime` and `elapsedTime`) if requested in the query request |
+|resultId*  |string     | Result ID of the query submitted for execution. This ID is required later for result fetching |
+|resultUrl* |string     | URL from which the results of the submitted query can be retrieved |
+
+### Result fetching (After an Asynchronous Query Response)
+
+The **resultId** received in the asynchronous query response needs to be submitted as a 
+**path parameter** (`/query/result/${resultId}`) to the REST API in order to retrieve the corresponding results.
+
+| Path | Method |Parameters | Type | Description |
+| ------ | ------ | ------ | ----- |----- |
+| /query/result/${resultId}    | GET | metrics | boolean   | If `true`, returns metrics (compile and execution time) with the response (default: false) |
+
+***
+
+## Error Response
+
+In any of the above scenarios, if an error occurred while processing, REST API will return an *Error Response* as 
+specified below.
+
+| Attribute | Type | Description |
+| ------ | ------ | ------ |
+|status*       |string         | `fatal` to indicate that the query execution was failed at some point | 
+|requestId*    |string         | A unique ID assigned to the request sent |
+|error*     |Error          | An error object which include an error code with an error message. {code: xxx, message: "error message"} |
diff --git a/vxquery-rest/src/site/site.xml b/vxquery-rest/src/site/site.xml
new file mode 100644 (file)
index 0000000..bd9b9ab
--- /dev/null
@@ -0,0 +1,53 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements. See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License. You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project name="VXQuery">
+    <bannerLeft>
+        <name>VXQuery</name>
+        <src>../images/VXQuery.png</src>
+        <href>../index.html</href>
+    </bannerLeft>
+
+    <bannerRight>
+        <name>Apache Software Foundation</name>
+        <src>../images/asf_logo_wide.png</src>
+        <href>http://www.apache.org/</href>
+    </bannerRight>
+
+    <skin>
+        <groupId>org.apache.maven.skins</groupId>
+        <artifactId>maven-fluido-skin</artifactId>
+        <version>1.5</version>
+    </skin>
+
+    <body>
+        <menu name="VXQuery REST API">
+            <item name="Overview" href="index.html"/>
+            <item name="Specification" href="specification.html"/>
+        </menu>
+
+        <menu ref="reports"/>
+        <footer><![CDATA[
+            <div class="row-fluid">Apache VXQuery, VXQuery, Apache, the Apache
+                feather logo, and the Apache VXQuery project logo are either
+                registered trademarks or trademarks of The Apache Software
+                Foundation in the United States and other countries.
+                All other marks mentioned may be trademarks or registered
+                trademarks of their respective owners.
+            </div>]]>
+        </footer>
+    </body>
+</project>
\ No newline at end of file
diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java
new file mode 100644 (file)
index 0000000..3d4b124
--- /dev/null
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import java.net.URI;
+import java.util.concurrent.TimeUnit;
+
+import javax.ws.rs.HttpMethod;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.vxquery.app.VXQueryApplication;
+import org.apache.vxquery.app.util.LocalClusterUtil;
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.AsyncQueryResponse;
+import org.apache.vxquery.rest.response.QueryResponse;
+import org.apache.vxquery.rest.response.QueryResultResponse;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.rest.service.VXQueryConfig;
+import org.apache.vxquery.rest.service.VXQueryService;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * Abstract test class to be used for {@link VXQueryApplication} related tests.
+ * These tests are expected to use the REST API for querying and fetching
+ * results
+ *
+ * @author Erandi Ganepola
+ */
+public class AbstractRestServerTest {
+
+    protected static LocalClusterUtil vxqueryLocalCluster = new LocalClusterUtil();
+    protected static String restIpAddress;
+    protected static int restPort;
+    protected static VXQueryService vxQueryService;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        vxqueryLocalCluster.init(new VXQueryConfig());
+        vxQueryService = vxqueryLocalCluster.getVxQueryService();
+        restIpAddress = vxqueryLocalCluster.getIpAddress();
+        restPort = vxqueryLocalCluster.getRestPort();
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        vxqueryLocalCluster.deinit();
+    }
+
+    protected static String normalize(String string) {
+        if (string == null) {
+            return null;
+        }
+
+        return string.replace("\r\n", "").replace("\n", "").replace("\r", "");
+    }
+
+    protected static void checkMetrics(QueryResponse response, boolean showMetrics) {
+        if (showMetrics) {
+            Assert.assertTrue(response.getMetrics().getCompileTime() > 0);
+            Assert.assertTrue(response.getMetrics().getElapsedTime() > 0);
+        } else {
+            Assert.assertTrue(response.getMetrics().getCompileTime() == 0);
+            Assert.assertTrue(response.getMetrics().getElapsedTime() == 0);
+        }
+    }
+
+    protected static void checkResults(AsyncQueryResponse response, boolean compileOnly) {
+        if (compileOnly) {
+            Assert.assertNull(response.getResultUrl());
+            Assert.assertEquals(0, response.getResultId());
+        } else {
+            Assert.assertTrue(response.getResultUrl().startsWith(Constants.RESULT_URL_PREFIX));
+            Assert.assertNotEquals(0, response.getResultId());
+        }
+    }
+
+    protected static void checkResults(SyncQueryResponse response, boolean compileOnly) {
+        if (compileOnly) {
+            Assert.assertNull(response.getResults());
+        } else {
+            Assert.assertNotNull(response.getResults());
+            Assert.assertFalse(response.getResults().isEmpty());
+        }
+    }
+
+    /**
+     * Submit a {@link QueryRequest} and fetth the resulting
+     * {@link AsyncQueryResponse}
+     *
+     * @param uri
+     *            uri of the GET request
+     * @param accepts
+     *            application/json | application/xml
+     * @param method
+     *            Http Method to be used to send the request
+     * @return Response received for the query request
+     * @throws Exception
+     */
+    protected static <T> T getQuerySuccessResponse(URI uri, String accepts, Class<T> type, String method)
+            throws Exception {
+        CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build();
+
+        try {
+            HttpUriRequest request = getRequest(uri, method);
+
+            if (accepts != null) {
+                request.setHeader(HttpHeaders.ACCEPT, accepts);
+            }
+
+            try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {
+                Assert.assertEquals(HttpResponseStatus.OK.code(), httpResponse.getStatusLine().getStatusCode());
+                if (accepts != null) {
+                    Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue());
+                }
+
+                HttpEntity entity = httpResponse.getEntity();
+                Assert.assertNotNull(entity);
+
+                String response = RestUtils.readEntity(entity);
+                return RestUtils.mapEntity(response, type, accepts);
+            }
+        } finally {
+            HttpClientUtils.closeQuietly(httpClient);
+        }
+    }
+
+    /**
+     * Fetch the {@link QueryResultResponse} from query result endpoint once the
+     * corresponding {@link QueryResultRequest} is given.
+     *
+     * @param resultRequest
+     *            {@link QueryResultRequest}
+     * @param accepts
+     *            expected
+     * 
+     *            <pre>
+     *            Accepts
+     *            </pre>
+     * 
+     *            header in responses
+     * @param method
+     *            Http Method to be used to send the request
+     * @return query result response received
+     * @throws Exception
+     */
+    protected static QueryResultResponse getQueryResultResponse(QueryResultRequest resultRequest, String accepts,
+            String method) throws Exception {
+        URI uri = RestUtils.buildQueryResultURI(resultRequest, restIpAddress, restPort);
+        CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build();
+        try {
+            HttpUriRequest request = getRequest(uri, method);
+
+            if (accepts != null) {
+                request.setHeader(HttpHeaders.ACCEPT, accepts);
+            }
+
+            try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {
+                if (accepts != null) {
+                    Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue());
+                }
+                Assert.assertEquals(httpResponse.getStatusLine().getStatusCode(), HttpResponseStatus.OK.code());
+
+                HttpEntity entity = httpResponse.getEntity();
+                Assert.assertNotNull(entity);
+
+                String response = RestUtils.readEntity(entity);
+                return RestUtils.mapEntity(response, QueryResultResponse.class, accepts);
+            }
+        } finally {
+            HttpClientUtils.closeQuietly(httpClient);
+        }
+    }
+
+    /**
+     * Creates a POST or GET request accordingly from the given {@link URI}
+     *
+     * @param uri
+     *            URI to which the request us to be sent
+     * @param method
+     *            Http method- GET or POST
+     * @return request
+     */
+    protected static HttpUriRequest getRequest(URI uri, String method) {
+        HttpUriRequest request;
+        switch (method) {
+            case HttpMethod.POST:
+                request = new HttpPost(uri);
+                break;
+            case HttpMethod.GET:
+            default:
+                request = new HttpGet(uri);
+        }
+
+        return request;
+    }
+}
diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java
new file mode 100644 (file)
index 0000000..12b61f3
--- /dev/null
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import static org.apache.vxquery.app.util.RestUtils.buildQueryResultURI;
+import static org.apache.vxquery.app.util.RestUtils.buildQueryURI;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.INVALID_INPUT;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.NOT_FOUND;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.PROBLEM_WITH_QUERY;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+
+import java.net.URI;
+import java.util.concurrent.TimeUnit;
+
+import javax.ws.rs.HttpMethod;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests error codes of the possible error responses that can be received for
+ * erroneous queries.
+ *
+ * @author Erandi Ganepola
+ */
+public class ErrorResponseTest extends AbstractRestServerTest {
+
+    @Test
+    public void testInvalidInput01() throws Exception {
+        QueryRequest request = new QueryRequest("   ");
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, INVALID_INPUT);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, INVALID_INPUT);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, INVALID_INPUT);
+    }
+
+    @Test
+    public void testInvalidInput02() throws Exception {
+        QueryRequest request = new QueryRequest("");
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, 405);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, 405);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, 405);
+    }
+
+    @Test
+    public void testInvalidQuery01() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1,2,3) return $y");
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+    }
+
+    @Test
+    public void testInvalidQuery02() throws Exception {
+        QueryRequest request = new QueryRequest("for x in (1,2,3) return $x");
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+    }
+
+    @Test
+    public void testInvalidQuery03() throws Exception {
+        QueryRequest request = new QueryRequest("insert nodes <book></book> into doc(\"abcd.xml\")/books");
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+    }
+
+    @Test
+    public void testInvalidQuery04() throws Exception {
+        QueryRequest request = new QueryRequest("delete nodes /a/b//node()");
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+    }
+
+    @Test
+    public void testInvalidResultId() throws Exception {
+        QueryResultRequest request = new QueryResultRequest(1000);
+        runTest(buildQueryResultURI(request, restIpAddress, restPort), null, NOT_FOUND);
+        runTest(buildQueryResultURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, NOT_FOUND);
+        runTest(buildQueryResultURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, NOT_FOUND);
+    }
+
+    @Test
+    public void testSyncInvalidInput01() throws Exception {
+        QueryRequest request = new QueryRequest("   ");
+        request.setAsync(false);
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, INVALID_INPUT);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, INVALID_INPUT);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, INVALID_INPUT);
+    }
+
+    @Test
+    public void testSyncInvalidInput02() throws Exception {
+        QueryRequest request = new QueryRequest("");
+        request.setAsync(false);
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, 405);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, 405);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, 405);
+    }
+
+    @Test
+    public void testSyncInvalidQuery01() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1,2,3) return $y");
+        request.setAsync(false);
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+    }
+
+    @Test
+    public void testSyncInvalidQuery02() throws Exception {
+        QueryRequest request = new QueryRequest("for x in (1,2,3) return $x");
+        request.setAsync(false);
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+    }
+
+    @Test
+    public void testSyncInvalidQuery03() throws Exception {
+        QueryRequest request = new QueryRequest("insert nodes <book></book> into doc(\"abcd.xml\")/books");
+        request.setAsync(false);
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+    }
+
+    @Test
+    public void testSyncInvalidQuery04() throws Exception {
+        QueryRequest request = new QueryRequest("delete nodes /a/b//node()");
+        request.setAsync(false);
+        runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+        runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+    }
+
+    private void runTest(URI uri, String accepts, int expectedStatusCode) throws Exception {
+        runTest(uri, accepts, expectedStatusCode, HttpMethod.GET);
+        runTest(uri, accepts, expectedStatusCode, HttpMethod.POST);
+    }
+
+    private void runTest(URI uri, String accepts, int expectedStatusCode, String httpMethod) throws Exception {
+        CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build();
+
+        ErrorResponse errorResponse;
+        try {
+            HttpUriRequest request = getRequest(uri, httpMethod);
+            if (accepts != null) {
+                request.setHeader(HttpHeaders.ACCEPT, accepts);
+            }
+
+            try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {
+                Assert.assertEquals(expectedStatusCode, httpResponse.getStatusLine().getStatusCode());
+                if (accepts != null) {
+                    Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue());
+                }
+
+                HttpEntity entity = httpResponse.getEntity();
+                Assert.assertNotNull(entity);
+
+                String response = RestUtils.readEntity(entity);
+                errorResponse = RestUtils.mapEntity(response, ErrorResponse.class, accepts);
+            }
+        } finally {
+            HttpClientUtils.closeQuietly(httpClient);
+        }
+
+        Assert.assertNotNull(errorResponse);
+        Assert.assertNotNull(errorResponse.getError().getMessage());
+        Assert.assertEquals(errorResponse.getError().getCode(), expectedStatusCode);
+    }
+}
diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java
new file mode 100644 (file)
index 0000000..d1385c8
--- /dev/null
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+
+import java.net.URI;
+
+import javax.ws.rs.HttpMethod;
+
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.AsyncQueryResponse;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.apache.vxquery.rest.response.QueryResultResponse;
+import org.apache.vxquery.rest.service.Status;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests the success responses received for XQueries submitted. i.e
+ * we are submitting correct queries which are expected to return a predictable
+ * result. All the parameters that are expected to be sent with query requests
+ * are subjected to test in this test class
+ *
+ * @author Erandi Ganepola
+ */
+public class SuccessAsyncResponseTest extends AbstractRestServerTest {
+
+    @Test
+    public void testSimpleQuery001() throws Exception {
+        QueryRequest request = new QueryRequest("1+1");
+        request.setShowAbstractSyntaxTree(true);
+        request.setShowOptimizedExpressionTree(true);
+        request.setShowRuntimePlan(true);
+        request.setShowTranslatedExpressionTree(true);
+        request.setShowMetrics(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSimpleQuery002() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setShowAbstractSyntaxTree(true);
+        request.setShowOptimizedExpressionTree(true);
+        request.setShowRuntimePlan(true);
+        request.setShowTranslatedExpressionTree(true);
+        request.setShowMetrics(true);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSimpleQuery003() throws Exception {
+        QueryRequest request = new QueryRequest("1+2+3");
+        request.setShowAbstractSyntaxTree(false);
+        request.setShowOptimizedExpressionTree(false);
+        request.setShowRuntimePlan(false);
+        request.setShowTranslatedExpressionTree(false);
+        request.setShowMetrics(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSimpleQuery004() throws Exception {
+        QueryRequest request = new QueryRequest("fn:true()");
+        request.setShowAbstractSyntaxTree(false);
+        request.setShowOptimizedExpressionTree(false);
+        request.setShowRuntimePlan(true);
+        request.setShowTranslatedExpressionTree(false);
+        request.setShowMetrics(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterNone() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterMetrics() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setShowMetrics(true);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterAST() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setShowAbstractSyntaxTree(true);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterOptimization() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setOptimization(10000);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterFrameSize() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setFrameSize((int) Math.pow(2, 12));
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterCompileOnly() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setCompileOnly(true);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterOET() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setShowOptimizedExpressionTree(true);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterTET() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setShowTranslatedExpressionTree(true);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterRP() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setShowRuntimePlan(true);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    private void runTest(String contentType, QueryRequest request) throws Exception {
+        runTest(contentType, request, HttpMethod.GET);
+        runTest(contentType, request, HttpMethod.POST);
+    }
+
+    private void runTest(String contentType, QueryRequest request, String httpMethod) throws Exception {
+        URI queryEndpointUri = RestUtils.buildQueryURI(request, restIpAddress, restPort);
+
+        /*
+         * ========== Query Response Testing ==========
+         */
+        // Testing the accuracy of VXQueryService class
+        AsyncQueryResponse expectedAsyncQueryResponse = (AsyncQueryResponse) vxQueryService.execute(request);
+
+        Assert.assertEquals(Status.SUCCESS.toString(), expectedAsyncQueryResponse.getStatus());
+        Assert.assertEquals(request.getStatement(), expectedAsyncQueryResponse.getStatement());
+        checkResults(expectedAsyncQueryResponse, request.isCompileOnly());
+        checkMetrics(expectedAsyncQueryResponse, request.isShowMetrics());
+        if (request.isShowMetrics()) {
+            Assert.assertTrue(expectedAsyncQueryResponse.getMetrics().getCompileTime() > 0);
+        } else {
+            Assert.assertTrue(expectedAsyncQueryResponse.getMetrics().getCompileTime() == 0);
+        }
+        if (request.isShowAbstractSyntaxTree()) {
+            Assert.assertNotNull(expectedAsyncQueryResponse.getAbstractSyntaxTree());
+        } else {
+            Assert.assertNull(expectedAsyncQueryResponse.getAbstractSyntaxTree());
+        }
+        if (request.isShowTranslatedExpressionTree()) {
+            Assert.assertNotNull(expectedAsyncQueryResponse.getTranslatedExpressionTree());
+        } else {
+            Assert.assertNull(expectedAsyncQueryResponse.getTranslatedExpressionTree());
+        }
+        if (request.isShowOptimizedExpressionTree()) {
+            Assert.assertNotNull(expectedAsyncQueryResponse.getOptimizedExpressionTree());
+        } else {
+            Assert.assertNull(expectedAsyncQueryResponse.getOptimizedExpressionTree());
+        }
+        if (request.isShowRuntimePlan()) {
+            Assert.assertNotNull(expectedAsyncQueryResponse.getRuntimePlan());
+        } else {
+            Assert.assertNull(expectedAsyncQueryResponse.getRuntimePlan());
+        }
+
+        // Testing the accuracy of REST server and servlets
+        AsyncQueryResponse actualAsyncQueryResponse =
+                getQuerySuccessResponse(queryEndpointUri, contentType, AsyncQueryResponse.class, httpMethod);
+
+        Assert.assertNotNull(actualAsyncQueryResponse.getRequestId());
+        Assert.assertEquals(request.getStatement(), actualAsyncQueryResponse.getStatement());
+        Assert.assertEquals(Status.SUCCESS.toString(), actualAsyncQueryResponse.getStatus());
+        checkResults(actualAsyncQueryResponse, request.isCompileOnly());
+        checkMetrics(actualAsyncQueryResponse, request.isShowMetrics());
+        // Cannot check this because Runtime plan include some object IDs which differ
+        // Assert.assertEquals(expectedAsyncQueryResponse.getRuntimePlan(),
+        // actualAsyncQueryResponse.getRuntimePlan());
+        if (request.isShowRuntimePlan()) {
+            Assert.assertNotNull(actualAsyncQueryResponse.getRuntimePlan());
+        } else {
+            Assert.assertNull(actualAsyncQueryResponse.getRuntimePlan());
+        }
+        Assert.assertEquals(normalize(expectedAsyncQueryResponse.getOptimizedExpressionTree()),
+                normalize(actualAsyncQueryResponse.getOptimizedExpressionTree()));
+        Assert.assertEquals(normalize(expectedAsyncQueryResponse.getTranslatedExpressionTree()),
+                normalize(actualAsyncQueryResponse.getTranslatedExpressionTree()));
+        Assert.assertEquals(normalize(expectedAsyncQueryResponse.getAbstractSyntaxTree()),
+                normalize(actualAsyncQueryResponse.getAbstractSyntaxTree()));
+
+        /*
+         * ========== Query Result Response Testing ========
+         */
+        QueryResultRequest resultRequest = new QueryResultRequest(actualAsyncQueryResponse.getResultId());
+        resultRequest.setShowMetrics(true);
+
+        if (request.isCompileOnly()) {
+            APIResponse resultResponse = vxQueryService.getResult(resultRequest);
+            Assert.assertTrue(resultResponse instanceof ErrorResponse);
+        } else {
+            QueryResultResponse expectedResultResponse = (QueryResultResponse) vxQueryService.getResult(resultRequest);
+            Assert.assertEquals(expectedResultResponse.getStatus(), Status.SUCCESS.toString());
+            Assert.assertNotNull(expectedResultResponse.getResults());
+
+            QueryResultResponse actualResultResponse = getQueryResultResponse(resultRequest, contentType, httpMethod);
+            Assert.assertEquals(actualResultResponse.getStatus(), Status.SUCCESS.toString());
+            Assert.assertNotNull(actualResultResponse.getResults());
+            Assert.assertNotNull(actualResultResponse.getRequestId());
+            Assert.assertEquals(normalize(expectedResultResponse.getResults()),
+                    normalize(actualResultResponse.getResults()));
+            if (resultRequest.isShowMetrics()) {
+                Assert.assertTrue(actualResultResponse.getMetrics().getElapsedTime() > 0);
+            } else {
+                Assert.assertTrue(actualResultResponse.getMetrics().getElapsedTime() == 0);
+            }
+
+        }
+    }
+}
diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java
new file mode 100644 (file)
index 0000000..31e0162
--- /dev/null
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+
+import java.net.URI;
+
+import javax.ws.rs.HttpMethod;
+
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.rest.service.Status;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests the success responses received for XQueries submitted. i.e
+ * we are submitting correct queries which are expected to return a predictable
+ * result. All the parameters that are expected to be sent with query requests
+ * are subjected to test in this test class
+ *
+ * @author Erandi Ganepola
+ */
+public class SuccessSyncResponseTest extends AbstractRestServerTest {
+
+    @Test
+    public void testSimpleQuery001() throws Exception {
+        QueryRequest request = new QueryRequest("1+1");
+        request.setShowAbstractSyntaxTree(true);
+        request.setShowOptimizedExpressionTree(true);
+        request.setShowRuntimePlan(true);
+        request.setShowTranslatedExpressionTree(true);
+        request.setShowMetrics(false);
+        request.setAsync(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSimpleQuery002() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setShowAbstractSyntaxTree(true);
+        request.setShowOptimizedExpressionTree(true);
+        request.setShowRuntimePlan(true);
+        request.setShowTranslatedExpressionTree(true);
+        request.setShowMetrics(true);
+        request.setAsync(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSimpleQuery003() throws Exception {
+        QueryRequest request = new QueryRequest("1+2+3");
+        request.setShowAbstractSyntaxTree(false);
+        request.setShowOptimizedExpressionTree(false);
+        request.setShowRuntimePlan(false);
+        request.setShowTranslatedExpressionTree(false);
+        request.setShowMetrics(false);
+        request.setAsync(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSimpleQuery004() throws Exception {
+        QueryRequest request = new QueryRequest("fn:true()");
+        request.setShowAbstractSyntaxTree(false);
+        request.setShowOptimizedExpressionTree(false);
+        request.setShowRuntimePlan(true);
+        request.setShowTranslatedExpressionTree(false);
+        request.setShowMetrics(false);
+        request.setAsync(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterNone() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setAsync(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterCompileOnly() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setCompileOnly(true);
+        request.setAsync(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    @Test
+    public void testSingleParameterRepeatExecutions() throws Exception {
+        QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+        request.setRepeatExecutions(5);
+        request.setAsync(false);
+
+        runTest(null, request);
+        runTest(CONTENT_TYPE_JSON, request);
+        runTest(CONTENT_TYPE_XML, request);
+    }
+
+    private void runTest(String contentType, QueryRequest request) throws Exception {
+        runTest(contentType, request, HttpMethod.GET);
+        runTest(contentType, request, HttpMethod.POST);
+    }
+
+    private void runTest(String contentType, QueryRequest request, String httpMethod) throws Exception {
+        URI queryEndpointUri = RestUtils.buildQueryURI(request, restIpAddress, restPort);
+
+        /*
+         * ========== Query Response Testing ==========
+         */
+        // Testing the accuracy of VXQueryService class
+        SyncQueryResponse expectedSyncQueryResponse = (SyncQueryResponse) vxQueryService.execute(request);
+
+        Assert.assertEquals(Status.SUCCESS.toString(), expectedSyncQueryResponse.getStatus());
+        Assert.assertEquals(request.getStatement(), expectedSyncQueryResponse.getStatement());
+        checkResults(expectedSyncQueryResponse, request.isCompileOnly());
+        checkMetrics(expectedSyncQueryResponse, request.isShowMetrics());
+        if (request.isShowMetrics()) {
+            Assert.assertTrue(expectedSyncQueryResponse.getMetrics().getCompileTime() > 0);
+        } else {
+            Assert.assertTrue(expectedSyncQueryResponse.getMetrics().getCompileTime() == 0);
+        }
+        if (request.isShowAbstractSyntaxTree()) {
+            Assert.assertNotNull(expectedSyncQueryResponse.getAbstractSyntaxTree());
+        } else {
+            Assert.assertNull(expectedSyncQueryResponse.getAbstractSyntaxTree());
+        }
+        if (request.isShowTranslatedExpressionTree()) {
+            Assert.assertNotNull(expectedSyncQueryResponse.getTranslatedExpressionTree());
+        } else {
+            Assert.assertNull(expectedSyncQueryResponse.getTranslatedExpressionTree());
+        }
+        if (request.isShowOptimizedExpressionTree()) {
+            Assert.assertNotNull(expectedSyncQueryResponse.getOptimizedExpressionTree());
+        } else {
+            Assert.assertNull(expectedSyncQueryResponse.getOptimizedExpressionTree());
+        }
+        if (request.isShowRuntimePlan()) {
+            Assert.assertNotNull(expectedSyncQueryResponse.getRuntimePlan());
+        } else {
+            Assert.assertNull(expectedSyncQueryResponse.getRuntimePlan());
+        }
+
+        // Testing the accuracy of REST server and servlets
+        SyncQueryResponse actualSyncQueryResponse =
+                getQuerySuccessResponse(queryEndpointUri, contentType, SyncQueryResponse.class, httpMethod);
+
+        Assert.assertNotNull(actualSyncQueryResponse.getRequestId());
+        Assert.assertEquals(request.getStatement(), actualSyncQueryResponse.getStatement());
+        Assert.assertEquals(Status.SUCCESS.toString(), actualSyncQueryResponse.getStatus());
+        checkMetrics(actualSyncQueryResponse, request.isShowMetrics());
+        checkResults(actualSyncQueryResponse, request.isCompileOnly());
+        // Cannot check this because Runtime plan include some object IDs which differ
+        // Assert.assertEquals(expectedSyncQueryResponse.getRuntimePlan(),
+        // actualSyncQueryResponse.getRuntimePlan());
+        if (request.isShowRuntimePlan()) {
+            Assert.assertNotNull(actualSyncQueryResponse.getRuntimePlan());
+        } else {
+            Assert.assertNull(actualSyncQueryResponse.getRuntimePlan());
+        }
+        Assert.assertEquals(normalize(expectedSyncQueryResponse.getOptimizedExpressionTree()),
+                normalize(actualSyncQueryResponse.getOptimizedExpressionTree()));
+        Assert.assertEquals(normalize(expectedSyncQueryResponse.getTranslatedExpressionTree()),
+                normalize(actualSyncQueryResponse.getTranslatedExpressionTree()));
+        Assert.assertEquals(normalize(expectedSyncQueryResponse.getAbstractSyntaxTree()),
+                normalize(actualSyncQueryResponse.getAbstractSyntaxTree()));
+
+        /*
+         * ========== Query Result Response Testing ========
+         */
+        String expectedResults = expectedSyncQueryResponse.getResults();
+        String actualResults = actualSyncQueryResponse.getResults();
+        if (!request.isCompileOnly()) {
+            Assert.assertNotNull(expectedResults);
+            Assert.assertNotNull(actualResults);
+        }
+        Assert.assertEquals(normalize(expectedResults), normalize(actualResults));
+    }
+}
diff --git a/vxquery-rest/src/test/resources/vxquery.properties b/vxquery-rest/src/test/resources/vxquery.properties
new file mode 100644 (file)
index 0000000..8870f79
--- /dev/null
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Number of processors to be used for query processing
+#org.apache.vxquery.available_processors=-1
+
+# Number of local node controllers to be created when creating a local hyracks cluster
+org.apache.vxquery.local_nc=1
+
+# Join hash size
+#org.apache.vxquery.join_hash=-1
+
+# Maximum Data Size
+#org.apache.vxquery.data_size=-1
+
+# hdfs config directory
+#org.apache.vxquery.hdfs_config=foo/bar
index e572c36..7331fde 100644 (file)
@@ -41,7 +41,7 @@
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>appassembler-maven-plugin</artifactId>
-                <version>1.1.1</version>
+                <version>2.0.0</version>
                 <executions>
                     <execution>
                         <configuration>
                                 <program>
                                     <mainClass>org.apache.hyracks.control.cc.CCDriver</mainClass>
                                     <name>vxquerycc</name>
+                                    <commandLineArguments>
+                                        <commandLineArgument>-app-cc-main-class</commandLineArgument>
+                                        <commandLineArgument>org.apache.vxquery.app.VXQueryApplication</commandLineArgument>
+                                    </commandLineArguments>
                                 </program>
                                 <program>
                                     <mainClass>org.apache.hyracks.control.nc.NCDriver</mainClass>
     <dependencies>
         <dependency>
             <groupId>org.apache.vxquery</groupId>
-            <artifactId>apache-vxquery-core</artifactId>
+            <artifactId>apache-vxquery-rest</artifactId>
             <version>0.7-SNAPSHOT</version>
         </dependency>
-
         <dependency>
             <groupId>org.apache.hyracks</groupId>
             <artifactId>hyracks-control-cc</artifactId>
         </dependency>
-
         <dependency>
             <groupId>org.apache.hyracks</groupId>
             <artifactId>hyracks-control-nc</artifactId>
index a00bec2..a32d913 100644 (file)
         </dependency>
 
         <dependency>
+            <groupId>org.apache.vxquery</groupId>
+            <artifactId>apache-vxquery-rest</artifactId>
+            <version>0.7-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
             <groupId>org.apache.hyracks</groupId>
             <artifactId>hyracks-api</artifactId>
         </dependency>
index 0e5b481..4d2ae8a 100644 (file)
@@ -19,81 +19,45 @@ package org.apache.vxquery.xtest;
 
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.client.dataset.HyracksDataset;
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.common.controllers.CCConfig;
-import org.apache.hyracks.control.common.controllers.NCConfig;
-import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.vxquery.app.util.LocalClusterUtil;
+import org.apache.vxquery.rest.service.VXQueryConfig;
 
-import java.io.File;
 import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
 
 public class TestClusterUtil {
 
-    private static final int CLIENT_NET_PORT = 39000;
-    private static final int CLUSTER_NET_PORT = 39001;
-    private static final int PROFILE_DUMP_PERIOD = 10000;
-    private static final String CC_HOST = "localhost";
-    private static final String NODE_ID = "nc1";
-    private static final String IO_DEVICES = "target/tmp/indexFolder";
-
     private static HyracksConnection hcc;
     private static HyracksDataset hds;
 
+    public static final LocalClusterUtil localClusterUtil = new LocalClusterUtil();
+
     private TestClusterUtil() {
     }
 
-    public static CCConfig createCCConfig() throws UnknownHostException {
-        String publicAddress = InetAddress.getLocalHost().getHostAddress();
-        CCConfig ccConfig = new CCConfig();
-        ccConfig.clientNetIpAddress = publicAddress;
-        ccConfig.clientNetPort = CLIENT_NET_PORT;
-        ccConfig.clusterNetIpAddress = publicAddress;
-        ccConfig.clusterNetPort = CLUSTER_NET_PORT;
-        ccConfig.profileDumpPeriod = PROFILE_DUMP_PERIOD;
-        return ccConfig;
-    }
+    private static VXQueryConfig loadConfiguration(XTestOptions opts) {
+        VXQueryConfig vxqConfig = new VXQueryConfig();
 
-    public static NCConfig createNCConfig() throws UnknownHostException {
-        String publicAddress = InetAddress.getLocalHost().getHostAddress();
-        NCConfig ncConfig1 = new NCConfig();
-        ncConfig1.ccHost = CC_HOST;
-        ncConfig1.ccPort = CLUSTER_NET_PORT;
-        ncConfig1.clusterNetIPAddress = publicAddress;
-        ncConfig1.dataIPAddress = publicAddress;
-        ncConfig1.resultIPAddress = publicAddress;
-        ncConfig1.nodeId = NODE_ID;
-        ncConfig1.ioDevices = IO_DEVICES;
-        return ncConfig1;
+        vxqConfig.setAvailableProcessors(opts.threads);
+        vxqConfig.setFrameSize(opts.frameSize);
+        vxqConfig.setHdfsConf(opts.hdfsConf);
+
+        return vxqConfig;
     }
 
-    public static ClusterControllerService startCC(XTestOptions opts) throws IOException {
-        CCConfig ccConfig = createCCConfig();
-        File outDir = new File("target/ClusterController");
-        outDir.mkdirs();
-        File ccRoot = File.createTempFile(TestRunner.class.getName(), ".data", outDir);
-        ccRoot.delete();
-        ccRoot.mkdir();
-        ccConfig.ccRoot = ccRoot.getAbsolutePath();
+    public static void startCluster(XTestOptions opts, LocalClusterUtil localClusterUtil) throws IOException {
         try {
-            ClusterControllerService cc = new ClusterControllerService(ccConfig);
-            cc.start();
-            hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
-            hds = new HyracksDataset(hcc, opts.frameSize, opts.threads);
-            return cc;
+            VXQueryConfig config = loadConfiguration(opts);
+            localClusterUtil.init(config);
+            hcc = (HyracksConnection) localClusterUtil.getConnection();
+            hds = (HyracksDataset) localClusterUtil.getDataset();
         } catch (Exception e) {
             throw new IOException(e);
         }
-
     }
 
-    public static NodeControllerService startNC() throws IOException {
-        NCConfig ncConfig = createNCConfig();
+    public static void stopCluster(LocalClusterUtil localClusterUtil) throws IOException {
         try {
-            NodeControllerService nc = new NodeControllerService(ncConfig);
-            nc.start();
-            return nc;
+            localClusterUtil.deinit();
         } catch (Exception e) {
             throw new IOException(e);
         }
@@ -107,13 +71,4 @@ public class TestClusterUtil {
         return hds;
     }
 
-    public static void stopCluster(ClusterControllerService cc, NodeControllerService nc) throws IOException {
-        try {
-            nc.stop();
-            cc.stop();
-        } catch (Exception e) {
-            throw new IOException(e);
-        }
-    }
-
 }
index fa0a900..8ef9426 100644 (file)
  */
 package org.apache.vxquery.xtest;
 
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.List;
+import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.client.NodeControllerInfo;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.dataset.DatasetJobRecord;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.job.JobFlag;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.control.nc.resources.memory.FrameManager;
-import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
-import org.apache.vxquery.compiler.CompilerControlBlock;
-import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory;
-import org.apache.vxquery.context.DynamicContext;
-import org.apache.vxquery.context.DynamicContextImpl;
-import org.apache.vxquery.context.RootStaticContextImpl;
-import org.apache.vxquery.context.StaticContextImpl;
+import javax.xml.bind.JAXBException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.vxquery.app.util.RestUtils;
 import org.apache.vxquery.exceptions.ErrorCode;
 import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.result.ResultUtils;
-import org.apache.vxquery.xmlquery.query.VXQueryCompilationListener;
-import org.apache.vxquery.xmlquery.query.XMLQueryCompiler;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.codehaus.jackson.map.ObjectMapper;
 
 public class TestRunner {
+
     private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})");
-    private List<String> collectionList;
+
     private XTestOptions opts;
-    private IHyracksClientConnection hcc;
-    private IHyracksDataset hds;
 
     public TestRunner(XTestOptions opts) throws UnknownHostException {
         this.opts = opts;
-        this.collectionList = new ArrayList<String>();
     }
 
     public void open() throws Exception {
-        hcc = TestClusterUtil.getConnection();
-        hds = TestClusterUtil.getDataset();
-    }
-
-    protected static TestConfiguration getIndexConfiguration(TestCase testCase) {
-        XTestOptions opts = new XTestOptions();
-        opts.verbose = false;
-        opts.threads = 1;
-        opts.showQuery = true;
-        opts.showResult = true;
-        opts.hdfsConf = "src/test/resources/hadoop/conf";
-        opts.catalog = StringUtils.join(new String[] { "src", "test", "resources", "VXQueryCatalog.xml" },
-                File.separator);
-        TestConfiguration indexConf = new TestConfiguration();
-        indexConf.options = opts;
-        String baseDir = new File(opts.catalog).getParent();
-        try {
-            String root = new File(baseDir).getCanonicalPath();
-            indexConf.testRoot = new File(root + "/./");
-            indexConf.resultOffsetPath = new File(root + "/./ExpectedResults/");
-            indexConf.sourceFileMap = testCase.getSourceFileMap();
-            indexConf.xqueryFileExtension = ".xq";
-            indexConf.xqueryxFileExtension = "xqx";
-            indexConf.xqueryQueryOffsetPath = new File(root + "/./Queries/XQuery/");
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        return indexConf;
-
     }
 
     public TestCaseResult run(final TestCase testCase) {
         TestCaseResult res = new TestCaseResult(testCase);
-        TestCase testCaseIndex = new TestCase(getIndexConfiguration(testCase));
-        testCaseIndex.setFolder("Indexing/Partition-1/");
-        testCaseIndex.setName("showIndexes");
-        runQuery(testCaseIndex, res);
-        String[] collections = res.result.split("\n");
-        this.collectionList = Arrays.asList(collections);
         runQueries(testCase, res);
         return res;
     }
@@ -121,83 +74,32 @@ public class TestRunner {
         long start = System.currentTimeMillis();
 
         try {
-            try {
-                if (opts.showQuery) {
+            String query = FileUtils.readFileToString(testCase.getXQueryFile(), "UTF-8");
 
-                    FileInputStream query = new FileInputStream(testCase.getXQueryFile());
-                    System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": ");
-                    System.err.println(IOUtils.toString(query, "UTF-8"));
-                    query.close();
-                }
-
-                VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET,
-                        opts.showOET, opts.showRP);
-
-                Map<String, NodeControllerInfo> nodeControllerInfos = null;
-                if (hcc != null) {
-                    nodeControllerInfos = hcc.getNodeControllerInfos();
-                }
+            if (opts.showQuery) {
+                System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": ");
+                System.err.println(query);
+            }
 
-                XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
-                        opts.hdfsConf);
-                Reader in = new InputStreamReader(new FileInputStream(testCase.getXQueryFile()), "UTF-8");
-                CompilerControlBlock ccb = new CompilerControlBlock(
-                        new StaticContextImpl(RootStaticContextImpl.INSTANCE),
-                        new ResultSetId(testCase.getXQueryDisplayName().hashCode()), testCase.getSourceFileMap());
-                compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel, collectionList);
-                JobSpecification spec = compiler.getModule().getHyracksJobSpecification();
-                in.close();
-
-                DynamicContext dCtx = new DynamicContextImpl(compiler.getModule().getModuleContext());
-                spec.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
-
-                spec.setMaxReattempts(0);
-                JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
-
-                FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize());
-                IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
-                IHyracksDatasetReader reader = hds.createReader(jobId, ccb.getResultSetId());
-                // TODO(tillw) remove this loop once the IHyracksDatasetReader reliably returns the correct exception
-                while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) {
-                    Thread.sleep(1);
-                }
-                IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
-                res.result = "";
-                while (reader.read(frame) > 0) {
-                    res.result += ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor);
-                    frame.getBuffer().clear();
-                }
-                res.result.trim();
-                hcc.waitForCompletion(jobId);
-            } catch (HyracksException e) {
-                Throwable t = e;
-                while (t.getCause() != null) {
-                    t = t.getCause();
-                }
-                final String message = t.getMessage();
-                if (message != null) {
-                    Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(message);
-                    if (m.find()) {
-                        String eCode = m.group(1);
-                        throw new SystemException(ErrorCode.valueOf(eCode), e);
-                    }
+            QueryRequest request = createQueryRequest(opts, query);
+            APIResponse response = sendQueryRequest(request, testCase.getSourceFileMap());
+            if (response instanceof SyncQueryResponse) {
+                res.result = ((SyncQueryResponse) response).getResults();
+            } else {
+                System.err.println("Error response: Failure when running the query");
+                ErrorResponse errorResponse = (ErrorResponse) response;
+                Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(errorResponse.getError().getMessage());
+
+                Exception e = new RuntimeException("Failed to run the query");
+                if (m.find()) {
+                    String eCode = m.group(1);
+                    throw new SystemException(ErrorCode.valueOf(eCode), e);
+                } else {
+                    throw e;
                 }
-                throw e;
             }
         } catch (Throwable e) {
-            // Check for nested SystemExceptions.
-            Throwable error = e;
-            while (error != null) {
-                if (error instanceof SystemException) {
-                    res.error = error;
-                    break;
-                }
-                error = error.getCause();
-            }
-            // Default
-            if (res.error == null) {
-                res.error = e;
-            }
+            res.error = e;
         } finally {
             try {
                 res.compare();
@@ -208,6 +110,7 @@ public class TestRunner {
             long end = System.currentTimeMillis();
             res.time = end - start;
         }
+
         if (opts.showResult) {
             if (res.result == null) {
                 System.err.println("***Error: ");
@@ -218,7 +121,55 @@ public class TestRunner {
                 System.err.println(res.result);
             }
         }
+    }
+
+    private static QueryRequest createQueryRequest(XTestOptions opts, String query) {
+        QueryRequest request = new QueryRequest(query);
+        request.setCompileOnly(opts.compileOnly);
+        request.setOptimization(opts.optimizationLevel);
+        request.setFrameSize(opts.frameSize);
+        request.setShowAbstractSyntaxTree(opts.showAST);
+        request.setShowTranslatedExpressionTree(opts.showTET);
+        request.setShowOptimizedExpressionTree(opts.showOET);
+        request.setShowRuntimePlan(opts.showRP);
+        request.setAsync(false);
+
+        return request;
+    }
+
+    private static APIResponse sendQueryRequest(QueryRequest request, Map<String, File> sourceFileMap)
+            throws IOException, URISyntaxException {
+
+        URI uri = RestUtils.buildQueryURI(request, TestClusterUtil.localClusterUtil.getIpAddress(),
+                TestClusterUtil.localClusterUtil.getRestPort());
+        CloseableHttpClient httpClient = HttpClients.custom().build();
+
+        try {
+            HttpPost httpRequest = new HttpPost(uri);
+            httpRequest.setHeader(HttpHeaders.ACCEPT, CONTENT_TYPE_JSON);
+
+            ObjectMapper mapper = new ObjectMapper();
+            String fileMap = mapper.writeValueAsString(sourceFileMap);
+            httpRequest.setEntity(new StringEntity(fileMap, StandardCharsets.UTF_8));
+
+            try (CloseableHttpResponse httpResponse = httpClient.execute(httpRequest)) {
+                HttpEntity entity = httpResponse.getEntity();
+                String response = RestUtils.readEntity(entity);
+                if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                    return RestUtils.mapEntity(response, SyncQueryResponse.class, CONTENT_TYPE_JSON);
+                } else {
+                    return RestUtils.mapEntity(response, ErrorResponse.class, CONTENT_TYPE_JSON);
+                }
+            } catch (IOException e) {
+                System.err.println("Error occurred when reading entity: " + e.getMessage());
+            } catch (JAXBException e) {
+                System.err.println("Error occurred when mapping query response: " + e.getMessage());
+            }
+        } finally {
+            HttpClientUtils.closeQuietly(httpClient);
+        }
 
+        return null;
     }
 
     public void runQueries(TestCase testCase, TestCaseResult res) {
index 5aae691..df7a71d 100644 (file)
  */
 package org.apache.vxquery.xtest;
 
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.nc.NodeControllerService;
-import org.mortbay.jetty.Server;
-
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -28,6 +24,8 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import org.mortbay.jetty.Server;
+
 public class XTest {
     private XTestOptions opts;
     private Server server;
@@ -36,8 +34,6 @@ public class XTest {
     private TestRunnerFactory trf;
     private int count;
     private int finishCount;
-    private static NodeControllerService nc;
-    private static ClusterControllerService cc;
 
     XTest(XTestOptions opts) {
         this.opts = opts;
@@ -81,8 +77,7 @@ public class XTest {
                 }
             }
         });
-        cc = TestClusterUtil.startCC(opts);
-        nc = TestClusterUtil.startNC();
+        TestClusterUtil.startCluster(opts, TestClusterUtil.localClusterUtil);
         trf = new TestRunnerFactory(opts);
         trf.registerReporters(reporters);
         TestCaseFactory tcf = new TestCaseFactory(trf, eSvc, opts);
@@ -104,7 +99,7 @@ public class XTest {
             r.close();
         }
         try {
-            TestClusterUtil.stopCluster(cc, nc);
+            TestClusterUtil.stopCluster(TestClusterUtil.localClusterUtil);
         } catch (IOException e) {
             e.printStackTrace();
         }
index 1e2dcf6..8f77de4 100644 (file)
@@ -22,8 +22,6 @@ import java.io.File;
 import java.io.IOException;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.nc.NodeControllerService;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -38,8 +36,6 @@ public abstract class AbstractXQueryTest {
     private TestRunner tr;
     private static MiniDFS dfs;
     private final static String TMP = "target/tmp";
-    private static NodeControllerService nc;
-    private static ClusterControllerService cc;
 
     protected abstract XTestOptions getTestOptions();
 
@@ -92,8 +88,7 @@ public abstract class AbstractXQueryTest {
 
     @BeforeClass
     public static void setup() throws IOException {
-        cc = TestClusterUtil.startCC(getDefaultTestOptions());
-        nc = TestClusterUtil.startNC();
+        TestClusterUtil.startCluster(getDefaultTestOptions(), TestClusterUtil.localClusterUtil);
         setupFS();
     }
 
@@ -116,7 +111,7 @@ public abstract class AbstractXQueryTest {
     @AfterClass
     public static void shutdown() throws IOException {
         removeFS();
-        TestClusterUtil.stopCluster(cc, nc);
+        TestClusterUtil.stopCluster(TestClusterUtil.localClusterUtil);
     }
 
     public static void removeFS() throws IOException {
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt
new file mode 100644 (file)
index 0000000..b1db973
--- /dev/null
@@ -0,0 +1,3 @@
+<value>33</value>
+<value>32</value>
+<value>31</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt
new file mode 100644 (file)
index 0000000..d93567e
--- /dev/null
@@ -0,0 +1,3 @@
+<value>33</value>
+<value>31</value>
+<value>32</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt
new file mode 100644 (file)
index 0000000..2ab8764
--- /dev/null
@@ -0,0 +1,3 @@
+<value>32</value>
+<value>33</value>
+<value>31</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt
new file mode 100644 (file)
index 0000000..d1d6bb7
--- /dev/null
@@ -0,0 +1,3 @@
+<value>32</value>
+<value>31</value>
+<value>33</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt
new file mode 100644 (file)
index 0000000..2044b18
--- /dev/null
@@ -0,0 +1,3 @@
+<value>31</value>
+<value>33</value>
+<value>32</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt
new file mode 100644 (file)
index 0000000..b1db973
--- /dev/null
@@ -0,0 +1,3 @@
+<value>33</value>
+<value>32</value>
+<value>31</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt
new file mode 100644 (file)
index 0000000..d93567e
--- /dev/null
@@ -0,0 +1,3 @@
+<value>33</value>
+<value>31</value>
+<value>32</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt
new file mode 100644 (file)
index 0000000..2ab8764
--- /dev/null
@@ -0,0 +1,3 @@
+<value>32</value>
+<value>33</value>
+<value>31</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt
new file mode 100644 (file)
index 0000000..d1d6bb7
--- /dev/null
@@ -0,0 +1,3 @@
+<value>32</value>
+<value>31</value>
+<value>33</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt
new file mode 100644 (file)
index 0000000..2044b18
--- /dev/null
@@ -0,0 +1,3 @@
+<value>31</value>
+<value>33</value>
+<value>32</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt
new file mode 100644 (file)
index 0000000..ea5a19e
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt
new file mode 100644 (file)
index 0000000..46985a9
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt
new file mode 100644 (file)
index 0000000..b97b850
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt
new file mode 100644 (file)
index 0000000..0bacd2b
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt
new file mode 100644 (file)
index 0000000..0e79fdc
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt
new file mode 100644 (file)
index 0000000..ea5a19e
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt
new file mode 100644 (file)
index 0000000..46985a9
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt
new file mode 100644 (file)
index 0000000..b97b850
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt
new file mode 100644 (file)
index 0000000..0bacd2b
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt
new file mode 100644 (file)
index 0000000..0e79fdc
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt
new file mode 100644 (file)
index 0000000..ea5a19e
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt
new file mode 100644 (file)
index 0000000..46985a9
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt
new file mode 100644 (file)
index 0000000..b97b850
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt
new file mode 100644 (file)
index 0000000..0bacd2b
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt
new file mode 100644 (file)
index 0000000..0e79fdc
--- /dev/null
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
index 1287225..9f07e2b 100644 (file)
       <description>Parsing a collection of json files.</description>
       <query name="q15_parser" date="2016-07-15"/>
      <output-file compare="Text">q15_parser.txt</output-file>
+     <output-file compare="Text">q15_parser-1.txt</output-file>
+     <output-file compare="Text">q15_parser-2.txt</output-file>
+     <output-file compare="Text">q15_parser-3.txt</output-file>
+     <output-file compare="Text">q15_parser-4.txt</output-file>
+     <output-file compare="Text">q15_parser-5.txt</output-file>
    </test-case>
    <test-case name="json-parser-q15-2" FilePath="Json/Parser/Partition-2" Creator="Christina Pavlopoulou">
       <description>Parsing a collection of json files.</description>
       <query name="q15_parser" date="2016-07-15"/>
-     <output-file compare="Text">q15_parser.txt</output-file>
+      <output-file compare="Text">q15_parser.txt</output-file>
+      <output-file compare="Text">q15_parser-1.txt</output-file>
+      <output-file compare="Text">q15_parser-2.txt</output-file>
+      <output-file compare="Text">q15_parser-3.txt</output-file>
+      <output-file compare="Text">q15_parser-4.txt</output-file>
+      <output-file compare="Text">q15_parser-5.txt</output-file>
    </test-case>
    <test-case name="json-parser-q15-4" FilePath="Json/Parser/Partition-4" Creator="Christina Pavlopoulou">
       <description>Parsing a collection of json files.</description>
       <query name="q15_parser" date="2016-07-15"/>
-     <output-file compare="Text">q15_parser.txt</output-file>
+      <output-file compare="Text">q15_parser.txt</output-file>
+      <output-file compare="Text">q15_parser-1.txt</output-file>
+      <output-file compare="Text">q15_parser-2.txt</output-file>
+      <output-file compare="Text">q15_parser-3.txt</output-file>
+      <output-file compare="Text">q15_parser-4.txt</output-file>
+      <output-file compare="Text">q15_parser-5.txt</output-file>
    </test-case>
    <test-case name="json-parser-q16" FilePath="Json/Parser/Partition-1" Creator="Christina Pavlopoulou">
       <description>Parsing a collection of json files.</description>