[VXQUERY-196][VXQUERY-204][VXQUERY-228] Cleaning up indexing query statements 169/head
authorChristina Pavlopoulou <cpavl001@ucr.edu>
Tue, 30 May 2017 18:54:31 +0000 (11:54 -0700)
committerChristina Pavlopoulou <cpavl001@ucr.edu>
Tue, 11 Jul 2017 17:20:22 +0000 (10:20 -0700)
1) Move XPath out of arguments for collection-from-index, leave only the collection path.
2) Hide the index usage from the user by creating a rewrite rule to figure out the existence of index.
3) Remove the unnecessary header tags that  collection-from-index creates.

63 files changed:
vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java
vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java with 91% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java with 78% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java with 89% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java with 89% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java with 80% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java with 87% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java with 85% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java with 95% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java with 94% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java with 97% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java
vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
vxquery-xtest/src/test/resources/VXQueryCatalog.xml
vxquery-xtest/src/test/resources/cat/IndexingQueries.xml

index e0e3843..25ff9c4 100644 (file)
@@ -14,6 +14,7 @@
  */
 package org.apache.vxquery.cli;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -23,6 +24,7 @@ import java.io.StringReader;
 import java.net.InetAddress;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -66,12 +68,13 @@ import org.kohsuke.args4j.Option;
 
 public class VXQuery {
     private final CmdLineOptions opts;
+    private final CmdLineOptions indexOpts;
 
     private ClusterControllerService cc;
     private NodeControllerService[] ncs;
     private IHyracksClientConnection hcc;
     private IHyracksDataset hds;
-
+    private List<String> collectionList;
     private ResultSetId resultSetId;
     private static List<String> timingMessages = new ArrayList<>();
     private static long sumTiming;
@@ -87,6 +90,16 @@ public class VXQuery {
      */
     public VXQuery(CmdLineOptions opts) {
         this.opts = opts;
+        // The index query returns only the result, without any other information.
+        this.indexOpts = opts;
+        indexOpts.showAST = false;
+        indexOpts.showOET = false;
+        indexOpts.showQuery = false;
+        indexOpts.showRP = false;
+        indexOpts.showTET = false;
+        indexOpts.timing = false;
+        indexOpts.compileOnly = false;
+        this.collectionList = new ArrayList<String>();
     }
 
     /**
@@ -168,71 +181,87 @@ public class VXQuery {
      * @throws SystemException
      * @throws Exception
      */
+
     private void runQueries() throws Exception {
-        Date start;
-        Date end;
-        for (String query : opts.arguments) {
-            String qStr = slurp(query);
-            if (opts.showQuery) {
-                System.err.println(qStr);
+        List<String> queries = opts.arguments;
+        // Run the showIndexes query before executing any target query, to store the index metadata
+        List<String> queriesIndex = new ArrayList<String>();
+        queriesIndex.add("vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndexes.xq");
+        OutputStream resultStream = new ByteArrayOutputStream();
+        executeQuery(queriesIndex.get(0), 1, resultStream, indexOpts);
+        ByteArrayOutputStream bos = (ByteArrayOutputStream) resultStream;
+        String result = new String(bos.toByteArray());
+        String[] collections = result.split("\n");
+        this.collectionList = Arrays.asList(collections);
+        executeQueries(queries);
+    }
+
+    public void executeQueries(List<String> queries) throws Exception {
+        for (String query : queries) {
+            OutputStream resultStream = System.out;
+            if (opts.resultFile != null) {
+                resultStream = new FileOutputStream(new File(opts.resultFile));
             }
+            executeQuery(query, opts.repeatExec, resultStream, opts);
+        }
+    }
 
-            VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET,
-                    opts.showOET, opts.showRP);
+    public void executeQuery(String query, int repeatedExecution, OutputStream resultStream, CmdLineOptions options)
+            throws Exception {
+        PrintWriter writer = new PrintWriter(resultStream, true);
+        String qStr = slurp(query);
+        if (opts.showQuery) {
+            writer.println(qStr);
+        }
+        VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, opts.showOET,
+                opts.showRP);
 
-            start = opts.timing ? new Date() : null;
+        Date start = opts.timing ? new Date() : null;
 
-            Map<String, NodeControllerInfo> nodeControllerInfos = null;
-            if (hcc != null) {
-                nodeControllerInfos = hcc.getNodeControllerInfos();
-            }
-            XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
-                    opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf);
-            resultSetId = createResultSetId();
-            CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
-                    resultSetId, null);
-            compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel);
-            // if -timing argument passed, show the starting and ending times
-            if (opts.timing) {
-                end = new Date();
-                timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms");
-            }
-            if (opts.compileOnly) {
-                continue;
-            }
-
-            Module module = compiler.getModule();
-            JobSpecification js = module.getHyracksJobSpecification();
+        Map<String, NodeControllerInfo> nodeControllerInfos = null;
+        if (hcc != null) {
+            nodeControllerInfos = hcc.getNodeControllerInfos();
+        }
+        XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
+                opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf);
+        resultSetId = createResultSetId();
+        CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
+                resultSetId, null);
+        compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel, this.collectionList);
+        // if -timing argument passed, show the starting and ending times
+        Date end = opts.timing ? new Date() : null;
+        if (opts.timing) {
+            timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms");
+        }
+        if (opts.compileOnly) {
+            return;
+        }
 
-            DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
-            js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
+        Module module = compiler.getModule();
+        JobSpecification js = module.getHyracksJobSpecification();
 
-            OutputStream resultStream = System.out;
-            if (opts.resultFile != null) {
-                resultStream = new FileOutputStream(new File(opts.resultFile));
-            }
+        DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
+        js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
 
-            PrintWriter writer = new PrintWriter(resultStream, true);
-            // Repeat execution for number of times provided in -repeatexec argument
-            for (int i = 0; i < opts.repeatExec; ++i) {
-                start = opts.timing ? new Date() : null;
-                runJob(js, writer);
-                // if -timing argument passed, show the starting and ending times
-                if (opts.timing) {
-                    end = new Date();
-                    long currentRun = end.getTime() - start.getTime();
-                    if ((i + 1) > opts.timingIgnoreQueries) {
-                        sumTiming += currentRun;
-                        sumSquaredTiming += currentRun * currentRun;
-                        if (currentRun < minTiming) {
-                            minTiming = currentRun;
-                        }
-                        if (maxTiming < currentRun) {
-                            maxTiming = currentRun;
-                        }
+        // Repeat execution for number of times provided in -repeatexec argument
+        for (int i = 0; i < repeatedExecution; ++i) {
+            start = opts.timing ? new Date() : null;
+            runJob(js, writer);
+            // if -timing argument passed, show the starting and ending times
+            if (opts.timing) {
+                end = new Date();
+                long currentRun = end.getTime() - start.getTime();
+                if ((i + 1) > opts.timingIgnoreQueries) {
+                    sumTiming += currentRun;
+                    sumSquaredTiming += currentRun * currentRun;
+                    if (currentRun < minTiming) {
+                        minTiming = currentRun;
+                    }
+                    if (maxTiming < currentRun) {
+                        maxTiming = currentRun;
                     }
-                    timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms");
                 }
+                timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms");
             }
         }
     }
index ceaf3c7..400fb15 100644 (file)
@@ -35,7 +35,8 @@ public class VXQueryCommons {
 
     static {
         indexingFunctions.add(BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier());
-        indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier());
+        indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier());
+        indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_1.getFunctionIdentifier());
         indexingFunctions.add(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier());
         indexingFunctions.add(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier());
     }
index 20283d8..11d3795 100644 (file)
  */
 package org.apache.vxquery.compiler.rewriter.rules;
 
+import java.util.ArrayList;
+
 import org.apache.commons.lang3.mutable.Mutable;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import org.apache.vxquery.common.VXQueryCommons;
 import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext;
 import org.apache.vxquery.metadata.VXQueryCollectionDataSource;
+import org.apache.vxquery.metadata.VXQueryIndexingDataSource;
+import org.apache.vxquery.metadata.VXQueryMetadataProvider;
 import org.apache.vxquery.types.AnyItemType;
 import org.apache.vxquery.types.Quantifier;
 import org.apache.vxquery.types.SequenceType;
@@ -61,11 +65,35 @@ public class IntroduceCollectionRule extends AbstractCollectionRule {
     public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
         String[] args = getFunctionalArguments(opRef, VXQueryCommons.collectionFunctions);
-
+        VXQueryMetadataProvider metadata = (VXQueryMetadataProvider) context.getMetadataProvider();
         if (args != null) {
             String collectionName = args[0];
             // Build the new operator and update the query plan.
             int collectionId = vxqueryContext.newCollectionId();
+            ArrayList<String> collectionTempName = new ArrayList<String>();
+            collectionTempName.add(collectionName);
+            if (collectionName.contains("|")) {
+                collectionTempName.remove(0);
+                int index = collectionName.indexOf("|");
+                int start = 0;
+                while (index >= 0) {
+                    collectionTempName.add(collectionName.substring(start, index));
+                    start = index + 1;
+                    index = collectionName.indexOf("|", index + 1);
+                    if (index == -1) {
+                        collectionTempName.add(collectionName.substring(start));
+                    }
+                }
+            }
+            if (metadata.hasIndex(collectionTempName)) {
+                VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collectionName,
+                        SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR),
+                        functionCall.getFunctionIdentifier().getName());
+                if (ids != null) {
+                    ids.setTotalDataSources(vxqueryContext.getTotalDataSources());
+                    return setDataSourceScan(ids, opRef);
+                }
+            }
             VXQueryCollectionDataSource ds = VXQueryCollectionDataSource.create(collectionId, collectionName,
                     SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR));
             if (ds != null) {
index 5b96131..6e60d75 100644 (file)
@@ -33,19 +33,19 @@ import org.apache.vxquery.types.SequenceType;
 public class IntroduceIndexingRule extends AbstractCollectionRule {
 
     @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
         VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
         String args[] = getFunctionalArguments(opRef, VXQueryCommons.indexingFunctions);
 
         if (args != null) {
 
             String collection = args[0];
-            String elementPath = args.length > 1?args[1]:null;
-
             // Build the new operator and update the query plan.
             int collectionId = vxqueryContext.newCollectionId();
-            VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection, elementPath,
-                    SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR), functionCall.getFunctionIdentifier().getName());
+            VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection,
+                    SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR),
+                    functionCall.getFunctionIdentifier().getName());
             if (ids != null) {
                 ids.setTotalDataSources(vxqueryContext.getTotalDataSources());
 
index dbcce54..2773154 100644 (file)
@@ -55,12 +55,9 @@ public class PushChildIntoDataScanRule extends AbstractPushExpressionIntoDatasca
 
     @Override
     boolean updateDataSource(IVXQueryDataSource datasource, Mutable<ILogicalExpression> expression) {
-        //TODO: indexing needs to be extended to support push child into datascan
-        if (datasource.usingIndex()) {
-            return false;
-        }
-        boolean added = false;
         List<Mutable<ILogicalExpression>> finds = new ArrayList<Mutable<ILogicalExpression>>();
+        boolean added = false;
+
         ExpressionToolbox.findAllFunctionExpressions(expression, BuiltinOperators.CHILD.getFunctionIdentifier(), finds);
         for (int i = finds.size(); i > 0; --i) {
             int typeId = ExpressionToolbox.getTypeExpressionTypeArgument(finds.get(i - 1));
index 1d8a55d..b901469 100644 (file)
@@ -59,6 +59,9 @@ public class PushValueIntoDatascanRule extends AbstractPushExpressionIntoDatasca
 
     @Override
     boolean updateDataSource(IVXQueryDataSource datasource, Mutable<ILogicalExpression> expression) {
+        if (datasource.usingIndex()) {
+            return false;
+        }
         VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) datasource;
         boolean added = false;
         List<Mutable<ILogicalExpression>> finds = new ArrayList<Mutable<ILogicalExpression>>();
index d64f423..4932a78 100644 (file)
     <!-- fn:collection-from-index($indexfolder  as xs:string?, $elementpath as xs:string?) as  node()* -->
     <function name="fn:collection-from-index">
         <param name="index-folder" type="xs:string?"/>
-        <param name="element-path" type="xs:string?"/>
         <return type="node()*"/>
         <property type="DocumentOrder" class="org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.InputPropertyPropagationPolicy">
             <argument value="0"/>
     <!-- fn:show-indexes as node()* -->
     <function name="fn:show-indexes">
         <return type="node()*"/>
-        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.ShowIndexScalarEvaluatorFactory"/>
+        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.ShowIndexesScalarEvaluatorFactory"/>
     </function>
 
     <!-- fn:collection-with-tag($arg1  as xs:string?, $arg2 as xs:string?) as  node()* -->
index 7524da4..1df31dd 100644 (file)
@@ -22,7 +22,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.data.std.api.IPointable;
 import org.apache.hyracks.data.std.primitive.BooleanPointable;
 import org.apache.hyracks.data.std.primitive.BytePointable;
 import org.apache.hyracks.data.std.primitive.DoublePointable;
@@ -56,12 +55,10 @@ import org.apache.vxquery.datamodel.accessors.nodes.NodeTreePointable;
 import org.apache.vxquery.datamodel.accessors.nodes.TextOrCommentNodePointable;
 import org.apache.vxquery.datamodel.values.ValueTag;
 import org.apache.vxquery.runtime.functions.cast.CastToStringOperation;
-import org.apache.vxquery.runtime.functions.index.updateIndex.Constants;
+import org.apache.vxquery.runtime.functions.index.update.Constants;
 import org.apache.vxquery.serializer.XMLSerializer;
 
 public class IndexDocumentBuilder extends XMLSerializer {
-    private final IPointable treePointable;
-
     private final PointablePool pp;
     private NodeTreePointable ntp;
 
@@ -88,16 +85,11 @@ public class IndexDocumentBuilder extends XMLSerializer {
     }
 
     //TODO: Handle Processing Instructions, PrefixedNames, and Namepsace entries
-    public IndexDocumentBuilder(IPointable tree, IndexWriter inWriter, String file) {
-        this.treePointable = tree;
+    public IndexDocumentBuilder(TaggedValuePointable tvp, IndexWriter inWriter, String file) {
         writer = inWriter;
 
         this.filePath = file;
 
-        //convert to tagged value pointable
-        TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
-        tvp.set(treePointable.getByteArray(), 0, treePointable.getLength());
-
         //get bytes and info from doc pointer
         bstart = tvp.getByteArray();
         sstart = tvp.getStartOffset();
@@ -105,7 +97,7 @@ public class IndexDocumentBuilder extends XMLSerializer {
 
         doc = new Document();
 
-        results = new ArrayList<ComplexItem>();
+        results = new ArrayList<>();
 
         pp = PointablePoolFactory.INSTANCE.createPointablePool();
     }
index 623b48c..a3756d5 100644 (file)
@@ -59,7 +59,6 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
 import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
 import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
 import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
 import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
 import org.apache.hyracks.hdfs.ContextFactory;
@@ -129,38 +128,14 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO
             public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
                 fta.reset(buffer);
                 String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
-                Reader input;
+
                 if (!collectionModifiedName.contains("hdfs:/")) {
                     File collectionDirectory = new File(collectionModifiedName);
                     // check if directory is in the local file system
                     if (collectionDirectory.exists()) {
                         // Go through each tuple.
                         if (collectionDirectory.isDirectory()) {
-                            for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
-                                Iterator<File> it = FileUtils.iterateFiles(collectionDirectory,
-                                        new VXQueryIOFileFilter(), TrueFileFilter.INSTANCE);
-                                while (it.hasNext()) {
-                                    File file = it.next();
-                                    String fileName = file.getName().toLowerCase();
-                                    if (fileName.endsWith(".xml")) {
-                                        if (LOGGER.isLoggable(Level.FINE)) {
-                                            LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath());
-                                        }
-                                        parser.parseElements(file, writer, tupleIndex);
-                                    } else if (fileName.endsWith(".json")) {
-                                        if (LOGGER.isLoggable(Level.FINE)) {
-                                            LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath());
-                                        }
-                                        try {
-                                            jsonAbvs.reset();
-                                            input = new InputStreamReader(new FileInputStream(file));
-                                            jparser.parse(input, jsonAbvs, writer, appender);
-                                        } catch (FileNotFoundException e) {
-                                            throw new HyracksDataException(e.toString());
-                                        }
-                                    }
-                                }
-                            }
+                            xmlAndJsonCollection(collectionDirectory);
                         } else {
                             throw new HyracksDataException("Invalid directory parameter (" + nodeId + ":"
                                     + collectionDirectory.getAbsolutePath() + ") passed to collection.");
@@ -272,6 +247,35 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO
                 }
             }
 
+            public void xmlAndJsonCollection(File directory) throws HyracksDataException {
+                Reader input;
+                for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
+                    Iterator<File> it = FileUtils.iterateFiles(directory, new VXQueryIOFileFilter(),
+                            TrueFileFilter.INSTANCE);
+                    while (it.hasNext()) {
+                        File file = it.next();
+                        String fileName = file.getName().toLowerCase();
+                        if (fileName.endsWith(".xml")) {
+                            if (LOGGER.isLoggable(Level.FINE)) {
+                                LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath());
+                            }
+                            parser.parseElements(file, writer, tupleIndex);
+                        } else if (fileName.endsWith(".json")) {
+                            if (LOGGER.isLoggable(Level.FINE)) {
+                                LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath());
+                            }
+                            try {
+                                jsonAbvs.reset();
+                                input = new InputStreamReader(new FileInputStream(file));
+                                jparser.parse(input, jsonAbvs, writer, appender);
+                            } catch (FileNotFoundException e) {
+                                throw new HyracksDataException(e.toString());
+                            }
+                        }
+                    }
+                }
+            }
+
             @Override
             public void fail() throws HyracksDataException {
                 writer.fail();
index ea69cfd..d55530d 100644 (file)
@@ -31,14 +31,11 @@ import org.apache.vxquery.compiler.rewriter.rules.CollectionFileDomain;
  */
 public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource {
 
-    private String elementPath;
     private String function;
 
-    private VXQueryIndexingDataSource(int id, String collection, String elementPath, Object[] types,
-            String functionCall) {
+    private VXQueryIndexingDataSource(int id, String collection, Object[] types, String functionCall) {
         this.dataSourceId = id;
         this.collectionName = collection;
-        this.elementPath = elementPath;
         this.function = functionCall;
         this.collectionPartitions = collectionName.split(DELIMITER);
         this.types = types;
@@ -56,13 +53,8 @@ public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource {
         this.valueSeq = new ArrayList<>();
     }
 
-    public static VXQueryIndexingDataSource create(int id, String collection, String index, Object type,
-            String function) {
-        return new VXQueryIndexingDataSource(id, collection, index, new Object[] { type }, function);
-    }
-
-    public String getElementPath() {
-        return elementPath;
+    public static VXQueryIndexingDataSource create(int id, String collection, Object type, String function) {
+        return new VXQueryIndexingDataSource(id, collection, new Object[] { type }, function);
     }
 
     public String getFunctionCall() {
@@ -71,7 +63,7 @@ public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource {
 
     @Override
     public String toString() {
-        return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + elementPath
+        return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + this.childSeq
                 + ", function=" + function + "]";
     }
 
index ac92a0e..3563713 100644 (file)
  */
 package org.apache.vxquery.metadata;
 
-import java.io.DataInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.Map;
+import java.util.List;
 import java.util.logging.Logger;
 
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.client.NodeControllerInfo;
 import org.apache.hyracks.api.comm.IFrame;
 import org.apache.hyracks.api.comm.IFrameFieldAppender;
 import org.apache.hyracks.api.comm.VSizeFrame;
@@ -35,23 +33,20 @@ import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
 import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
 import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
 import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
 import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
 import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
 import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
 import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
 import org.apache.vxquery.datamodel.values.XDMConstants;
 import org.apache.vxquery.exceptions.SystemException;
 import org.apache.vxquery.functions.BuiltinFunctions;
 import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
 import org.apache.vxquery.runtime.functions.index.VXQueryIndexReader;
-import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
+import org.apache.vxquery.runtime.functions.index.centralizer.IndexCentralizerUtil;
+import org.apache.vxquery.runtime.functions.index.update.IndexUpdater;
 import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
 import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
 
@@ -61,18 +56,18 @@ public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOpe
     private short dataSourceId;
     private short totalDataSources;
     private String[] collectionPartitions;
-    private String elementPath;
     private final String functionCall;
+    private List<Integer> childSeq;
 
     public VXQueryIndexingOperatorDescriptor(IOperatorDescriptorRegistry spec, VXQueryIndexingDataSource ds,
-            RecordDescriptor rDesc, String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos) {
+            RecordDescriptor rDesc) {
         super(spec, 1, 1);
         this.functionCall = ds.getFunctionCall();
         collectionPartitions = ds.getPartitions();
         dataSourceId = (short) ds.getDataSourceId();
         totalDataSources = (short) ds.getTotalDataSources();
         recordDescriptors[0] = rDesc;
-        this.elementPath = ds.getElementPath();
+        childSeq = ds.getChildSeq();
     }
 
     @Override
@@ -87,10 +82,11 @@ public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOpe
         final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources);
         final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
         final String collectionName = collectionPartitions[partition % collectionPartitions.length];
-        String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
+        final String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
         IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
                 ctx.getIOManager().getIODevices().get(0).getMount());
         indexCentralizerUtil.readIndexDirectory();
+        final IPointable result = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
 
         return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
             @Override
@@ -103,100 +99,115 @@ public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOpe
             public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
                 fta.reset(buffer);
 
-                IPointable result = new TaggedValuePointable();
-
-                final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
-                final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY
-                        .createPointable();
-
-                final ByteBufferInputStream bbis = new ByteBufferInputStream();
-                final DataInputStream di = new DataInputStream(bbis);
-                final SequenceBuilder sb = new SequenceBuilder();
                 final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
                 final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
 
-                String indexModifiedName;
+                abvs.reset();
+                abvsFileNode.reset();
+
                 if (collectionModifiedName.contains("hdfs://")) {
                     throw new HyracksDataException("Indexing support for HDFS not yet implemented.");
                 } else {
 
                     if (functionCall.equals(
                             BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier().getName())) {
-                        indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName);
-                        File collectionDirectory = new File(collectionModifiedName);
-
-                        //check if directory is in the local file system
-                        if (collectionDirectory.exists() && collectionDirectory.isDirectory()) {
-                            IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
-                            try {
-                                indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result,
-                                        stringp, bbis, di, sb, abvs, nodeIdProvider, abvsFileNode, nodep, false,
-                                        nodeId);
-                                XDMConstants.setTrue(result);
-                                FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
-                                        result.getStartOffset(), result.getLength());
-                            } catch (SystemException e) {
-                                throw new HyracksDataException("Could not create index for collection: "
-                                        + collectionName + " in dir: " + indexModifiedName + " " + e.getMessage());
-                            }
-                        } else {
-                            throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":"
-                                    + collectionDirectory.getAbsolutePath() + ")");
-                        }
-                    } else if (functionCall
-                            .equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) {
-                        indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
-                        IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
-                                nodeIdProvider, abvsFileNode, nodep, nodeId);
                         try {
-                            updater.setup();
-                            updater.updateIndex();
-                            updater.updateMetadataFile();
-                            updater.exit();
-                            XDMConstants.setTrue(result);
-                            FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
-                                    result.getStartOffset(), result.getLength());
+                            createIndex(result, abvs, abvsFileNode);
                         } catch (IOException e) {
-                            throw new HyracksDataException(
-                                    "Could not update index in " + indexModifiedName + " " + e.getMessage());
+                            throw new HyracksDataException(e);
                         }
                     } else if (functionCall
+                            .equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) {
+                        updateIndex(result, abvs, abvsFileNode);
+                    } else if (functionCall
                             .equals(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier().getName())) {
-                        indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
-                        IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
-                                nodeIdProvider, abvsFileNode, nodep, nodeId);
-                        indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName);
-                        try {
-                            updater.setup();
-                            updater.deleteAllIndexes();
-                            XDMConstants.setTrue(result);
-                            FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
-                                    result.getStartOffset(), result.getLength());
-                        } catch (IOException e) {
-                            throw new HyracksDataException(
-                                    "Could not delete index in " + indexModifiedName + " " + e.getMessage());
-                        }
-
+                        deleteIndex(result, abvs, abvsFileNode);
                     } else if (functionCall
-                            .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier().getName())) {
-                        indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
-                        VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, elementPath);
-                        try {
-                            indexReader.init();
-                            while (indexReader.step(result)) {
-                                FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
-                                        result.getStartOffset(), result.getLength());
-                            }
-                        } catch (AlgebricksException e) {
-                            throw new HyracksDataException("Could not read index.");
-                        }
-
+                            .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_1.getFunctionIdentifier().getName())
+                            || functionCall
+                                    .equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier().getName())) {
+                        usingIndex(result);
                     } else {
                         throw new HyracksDataException("Unsupported function call (" + functionCall + ")");
                     }
                 }
             }
 
+            public void createIndex(IPointable result, ArrayBackedValueStorage abvs,
+                    ArrayBackedValueStorage abvsFileNode) throws IOException {
+                String indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName);
+                File collectionDirectory = new File(collectionModifiedName);
+
+                //check if directory is in the local file system
+                if (collectionDirectory.exists() && collectionDirectory.isDirectory()) {
+                    IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
+                    try {
+                        indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result, abvs,
+                                nodeIdProvider, abvsFileNode, false, nodeId);
+                        XDMConstants.setTrue(result);
+                        FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+                                result.getLength());
+                    } catch (SystemException e) {
+                        throw new HyracksDataException("Could not create index for collection: " + collectionName
+                                + " in dir: " + indexModifiedName + " " + e.getMessage(), e);
+                    }
+                } else {
+                    throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":"
+                            + collectionDirectory.getAbsolutePath() + ")");
+                }
+            }
+
+            public void updateIndex(IPointable result, ArrayBackedValueStorage abvs,
+                    ArrayBackedValueStorage abvsFileNode) throws HyracksDataException {
+                String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+                IndexUpdater updater = new IndexUpdater(indexModifiedName, result, abvs, nodeIdProvider, abvsFileNode,
+                        nodeId);
+                try {
+                    updater.setup();
+                    updater.updateIndex();
+                    updater.updateMetadataFile();
+                    updater.exit();
+                    XDMConstants.setTrue(result);
+                    FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+                            result.getLength());
+                } catch (IOException e) {
+                    throw new HyracksDataException(
+                            "Could not update index in " + indexModifiedName + " " + e.getMessage(), e);
+                }
+            }
+
+            public void deleteIndex(IPointable result, ArrayBackedValueStorage abvs,
+                    ArrayBackedValueStorage abvsFileNode) throws HyracksDataException {
+                String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+                IndexUpdater updater = new IndexUpdater(indexModifiedName, result, abvs, nodeIdProvider, abvsFileNode,
+                        nodeId);
+                indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName);
+                try {
+                    updater.setup();
+                    updater.deleteAllIndexes();
+                    XDMConstants.setTrue(result);
+                    FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+                            result.getLength());
+                } catch (IOException e) {
+                    throw new HyracksDataException(
+                            "Could not delete index in " + indexModifiedName + " " + e.getMessage(), e);
+                }
+            }
+
+            public void usingIndex(IPointable result) throws HyracksDataException {
+                String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+                VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, childSeq, appender);
+                try {
+                    indexReader.init();
+                    for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
+                        while (indexReader.step(result, writer, tupleIndex)) {
+                        }
+                    }
+                } catch (AlgebricksException e) {
+                    throw new HyracksDataException("Could not read index.", e);
+                }
+            }
+
             @Override
             public void fail() throws HyracksDataException {
                 writer.fail();
index f6644d6..5bb9d1a 100644 (file)
@@ -59,15 +59,17 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
     private final Map<String, File> sourceFileMap;
     private final StaticContext staticCtx;
     private final String hdfsConf;
+    private final List<String> collections;
     private final Map<String, NodeControllerInfo> nodeControllerInfos;
 
     public VXQueryMetadataProvider(String[] nodeList, Map<String, File> sourceFileMap, StaticContext staticCtx,
-            String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos) {
+            String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos, List<String> collections) {
         this.nodeList = nodeList;
         this.sourceFileMap = sourceFileMap;
         this.staticCtx = staticCtx;
         this.hdfsConf = hdfsConf;
         this.nodeControllerInfos = nodeControllerInfos;
+        this.collections = collections;
     }
 
     @Override
@@ -111,8 +113,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
             constraint = getClusterLocations(nodeList, ds.getPartitionCount());
         } else {
             rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
-            scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, (VXQueryIndexingDataSource) ds, rDesc,
-                    this.hdfsConf, this.nodeControllerInfos);
+            scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, (VXQueryIndexingDataSource) ds, rDesc);
             constraint = getClusterLocations(nodeList, ds.getPartitionCount());
         }
 
@@ -142,7 +143,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
     @Override
     public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
             int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
-            throws AlgebricksException {
+                    throws AlgebricksException {
         throw new UnsupportedOperationException();
     }
 
@@ -168,7 +169,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
             IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
             List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
             ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
-            throws AlgebricksException {
+                    throws AlgebricksException {
         throw new UnsupportedOperationException();
     }
 
@@ -234,7 +235,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
             JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException {
         throw new UnsupportedOperationException();
     }
-    
+
     @Override
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<String> dataSource,
             IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
@@ -243,7 +244,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
             JobSpecification jobSpec, boolean bulkload) throws AlgebricksException {
         throw new UnsupportedOperationException();
     }
-    
+
     @Override
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getUpsertRuntime(IDataSource<String> dataSource,
             IOperatorSchema inputSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
@@ -252,7 +253,7 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
             JobSpecification jobSpec) throws AlgebricksException {
         throw new UnsupportedOperationException();
     }
-    
+
     @Override
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexUpsertRuntime(
             IDataSourceIndex<String, String> dataSourceIndex, IOperatorSchema propagatedSchema,
@@ -263,11 +264,26 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
             JobSpecification spec) throws AlgebricksException {
         throw new UnsupportedOperationException();
     }
-    
+
     @Override
     public Map<String, String> getConfig() {
         return new HashMap<>();
     }
 
+    public List<String> getIndexCollections() {
+        return collections;
+
+    }
+
+    public boolean hasIndex(ArrayList<String> collections) {
+        boolean indexExists = false;
+        for (String collection : collections) {
+            indexExists = getIndexCollections().contains(collection);
+            if (!indexExists) {
+                break;
+            }
+        }
+        return indexExists;
+    }
 
 }
index 4706496..2c25752 100644 (file)
 */
 package org.apache.vxquery.runtime.functions.index;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.concurrent.ConcurrentHashMap;
+
 import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
 import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -31,38 +35,31 @@ import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
 import org.apache.vxquery.exceptions.ErrorCode;
 import org.apache.vxquery.exceptions.SystemException;
 import org.apache.vxquery.index.IndexDocumentBuilder;
-import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
+import org.apache.vxquery.runtime.functions.index.update.MetaFileUtil;
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
 import org.apache.vxquery.runtime.functions.util.FunctionHelper;
 import org.apache.vxquery.xmlparser.IParser;
 import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
 import org.apache.vxquery.xmlparser.XMLParser;
 
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.text.SimpleDateFormat;
-import java.util.concurrent.ConcurrentHashMap;
-
 public class IndexConstructorUtil {
-     boolean isMetaFilePresent = false;
-     MetaFileUtil metaFileUtil;
-     ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
+    private final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+    private final SequenceBuilder sb = new SequenceBuilder();
+    private boolean isMetaFilePresent = false;
+    private MetaFileUtil metaFileUtil;
+    private ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
 
-    public void evaluate(String collectioFolder, String indexFolder, IPointable result, UTF8StringPointable
-            stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
-            ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
-            boolean isElementPath, String nodeId) throws SystemException {
+    public void evaluate(String collectioFolder, String indexFolder, IPointable result, ArrayBackedValueStorage abvs,
+            ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, boolean isElementPath,
+            String nodeId) throws IOException {
 
-            metaFileUtil = new MetaFileUtil(indexFolder);
-//            metaFileUtil = .create(indexFolder);
-            isMetaFilePresent = metaFileUtil.isMetaFilePresent();
-            metaFileUtil.setCollection(collectioFolder);
+        metaFileUtil = new MetaFileUtil(indexFolder);
+        isMetaFilePresent = metaFileUtil.isMetaFilePresent();
+        metaFileUtil.setCollection(collectioFolder);
 
         File collectionDirectory = new File(collectioFolder);
         if (!collectionDirectory.exists()) {
-            throw new RuntimeException("The collection directory (" + collectioFolder + ") does not exist.");
+            throw new IOException("The collection directory (" + collectioFolder + ") does not exist.");
         }
 
         try {
@@ -80,8 +77,7 @@ public class IndexConstructorUtil {
             IndexWriter writer = new IndexWriter(dir, iwc);
 
             //Add files to index
-            indexXmlFiles(collectionDirectory, writer, isElementPath, nodep, abvsFileNode, nodeIdProvider, sb, bbis, di,
-                    nodeId);
+            indexXmlFiles(collectionDirectory, writer, isElementPath, abvsFileNode, nodeIdProvider, sb, nodeId);
 
             if (!isMetaFilePresent) {
                 // Write metadata map to a file.
@@ -101,14 +97,13 @@ public class IndexConstructorUtil {
         }
     }
 
-    /*This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
+    /*
+     * This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
      * it indexes that document node.
      */
     public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
-            TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider,
-            SequenceBuilder sb, ByteBufferInputStream bbis, DataInputStream di, String nodeId)
-            throws SystemException, IOException {
-
+            ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, SequenceBuilder sb, String nodeId)
+            throws IOException {
         SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy, HH:mm:ss");
 
         for (File file : collectionDirectory.listFiles()) {
@@ -116,8 +111,7 @@ public class IndexConstructorUtil {
             if (readableXmlFile(file.getPath())) {
                 abvsFileNode.reset();
 
-                IndexDocumentBuilder ibuilder = getIndexBuilder(file, writer, nodep, abvsFileNode, nodeIdProvider, bbis,
-                        di, nodeId);
+                IndexDocumentBuilder ibuilder = getIndexBuilder(file, writer, abvsFileNode, nodeIdProvider, nodeId);
 
                 ibuilder.printStart();
                 if (!isMetaFilePresent) {
@@ -131,22 +125,21 @@ public class IndexConstructorUtil {
 
             } else if (file.isDirectory()) {
                 // Consider all XML file in sub directories.
-                indexXmlFiles(file, writer, isElementPath, nodep, abvsFileNode, nodeIdProvider, sb, bbis, di, nodeId);
+                indexXmlFiles(file, writer, isElementPath, abvsFileNode, nodeIdProvider, sb, nodeId);
             }
         }
     }
 
     public boolean readableXmlFile(String path) {
-        return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz"));
+        return path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz");
     }
 
-    public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
-            ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis,
-            DataInputStream di, String nodeId) throws IOException {
+    public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, ArrayBackedValueStorage abvsFileNode,
+            ITreeNodeIdProvider nodeIdProvider, String nodeId) throws IOException {
 
         //Get the document node
         IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
-        FunctionHelper.readInDocFromString(file.getPath(), bbis, di, abvsFileNode, parser);
+        FunctionHelper.readInDocFromString(file.getPath(), abvsFileNode, parser);
 
         nodep.set(abvsFileNode.getByteArray(), abvsFileNode.getStartOffset(), abvsFileNode.getLength());
 
@@ -30,12 +30,12 @@ import org.apache.vxquery.exceptions.ErrorCode;
 import org.apache.vxquery.exceptions.SystemException;
 import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
 import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
+import org.apache.vxquery.runtime.functions.index.centralizer.IndexCentralizerUtil;
 
-public class ShowIndexScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
+public class ShowIndexesScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
     private static final long serialVersionUID = 1L;
 
-    public ShowIndexScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+    public ShowIndexesScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
         super(args);
     }
 
index 8750849..cf781ab 100644 (file)
 */
 package org.apache.vxquery.runtime.functions.index;
 
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.comm.IFrameFieldAppender;
+import org.apache.hyracks.api.comm.IFrameWriter;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
 import org.apache.hyracks.data.std.api.IPointable;
 import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
@@ -32,21 +38,21 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.vxquery.context.DynamicContext;
 import org.apache.vxquery.exceptions.ErrorCode;
 import org.apache.vxquery.exceptions.SystemException;
 import org.apache.vxquery.index.IndexAttributes;
+import org.apache.vxquery.runtime.functions.util.FunctionHelper;
+import org.apache.vxquery.types.ElementType;
+import org.apache.vxquery.types.NameTest;
+import org.apache.vxquery.types.NodeType;
+import org.apache.vxquery.types.SequenceType;
 import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
 import org.apache.vxquery.xmlparser.SAXContentHandler;
 import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
 import org.xml.sax.Attributes;
 import org.xml.sax.SAXException;
 
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
 public class VXQueryIndexReader {
 
     private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage();
@@ -55,10 +61,7 @@ public class VXQueryIndexReader {
     private int indexLength;
     private String elementPath;
     private String indexName;
-
-    private ByteBufferInputStream bbis = new ByteBufferInputStream();
-    private DataInputStream di = new DataInputStream(bbis);
-
+    private List<SequenceType> childSequenceTypes;
     private IndexReader reader;
     private IndexSearcher searcher;
     private QueryParser parser;
@@ -68,14 +71,40 @@ public class VXQueryIndexReader {
     private Document doc;
     private List<IndexableField> fields;
     private IHyracksTaskContext ctx;
+    private String[] childLocalName = null;
+    private IFrameFieldAppender appender;
+    private boolean firstElement;
 
-    public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, String elementPath) {
+    public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, List<Integer> childSeq,
+            IFrameFieldAppender appender) {
         this.ctx = context;
         this.indexName = indexPath;
-        this.elementPath = elementPath;
+        this.appender = appender;
+        final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData();
+        childSequenceTypes = new ArrayList<>();
+        for (int typeCode : childSeq) {
+            childSequenceTypes.add(dCtx.getStaticContext().lookupSequenceType(typeCode));
+        }
+        childLocalName = new String[childSequenceTypes.size()];
+        int index = 0;
+        StringBuilder stb = new StringBuilder();
+        stb.append("/");
+        for (SequenceType sType : childSequenceTypes) {
+            NodeType nodeType = (NodeType) sType.getItemType();
+            ElementType eType = (ElementType) nodeType;
+            NameTest nameTest = eType.getNameTest();
+            childLocalName[index] = FunctionHelper.getStringFromBytes(nameTest.getLocalName());
+
+            stb.append(childLocalName[index]);
+            if (index != childSequenceTypes.size() - 1) {
+                stb.append("/");
+            }
+            ++index;
+        }
+        elementPath = stb.toString();
     }
 
-    public boolean step(IPointable result) throws AlgebricksException {
+    public boolean step(IPointable result, IFrameWriter writer, int tupleIndex) throws AlgebricksException {
         /*each step will create a tuple for a single xml file
         * This is done using the parse function
         * checkoverflow is used throughout. This is because memory might not be
@@ -88,6 +117,8 @@ public class VXQueryIndexReader {
                 //TODO: now we get back the entire document
                 doc = searcher.doc(hits[indexPlace].doc);
                 fields = doc.getFields();
+                handler.setupElementWriter(writer, tupleIndex);
+                this.firstElement = true;
                 parse(nodeAbvs);
             } catch (IOException e) {
                 throw new AlgebricksException(e);
@@ -103,7 +134,7 @@ public class VXQueryIndexReader {
 
         int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
         ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-        handler = new SAXContentHandler(false, nodeIdProvider, true);
+        handler = new SAXContentHandler(false, nodeIdProvider, appender, childSequenceTypes);
 
         nodeAbvs.reset();
         indexPlace = 0;
@@ -125,7 +156,7 @@ public class VXQueryIndexReader {
         String queryString = elementPath.replaceAll("/", ".");
         queryString = "item:" + queryString + "*";
 
-        int lastslash = elementPath.lastIndexOf("/");
+        int lastslash = elementPath.lastIndexOf('/');
         elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1);
         elementPath = elementPath.replaceAll("/", ".") + ".element";
 
@@ -135,31 +166,25 @@ public class VXQueryIndexReader {
 
             //TODO: Right now it only returns 1000000 results
             results = searcher.search(query, 1000000);
-
         } catch (Exception e) {
-            throw new SystemException(null);
+            throw new SystemException(null, e);
         }
 
         hits = results.scoreDocs;
-        System.out.println("found: " + results.totalHits);
         indexPlace = 0;
         indexLength = hits.length;
-
     }
 
     public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException {
         try {
-            handler.startDocument();
-
             for (int i = 0; i < fields.size(); i++) {
                 String fieldValue = fields.get(i).stringValue();
                 if (fieldValue.equals(elementPath)) {
+                    handler.startDocument();
+                    this.firstElement = true;
                     buildElement(abvsFileNode, i);
                 }
             }
-
-            handler.endDocument();
-            handler.writeDocument(abvsFileNode);
         } catch (Exception e) {
             throw new IOException(e);
         }
@@ -167,6 +192,7 @@ public class VXQueryIndexReader {
 
     private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException {
         int whereIFinish = fieldNum;
+        int firstFinish;
         IndexableField field = fields.get(fieldNum);
         String contents = field.stringValue();
         String uri = "";
@@ -176,18 +202,37 @@ public class VXQueryIndexReader {
         String type = contents.substring(lastDot + 1);
         String lastBit = contents.substring(firstColon + 1, lastDot);
 
-        if (type.equals("textnode")) {
+        if (this.firstElement) {
+            this.firstElement = false;
+            firstFinish = whereIFinish - this.childSequenceTypes.size() + 1;
+            String firstBit = contents.substring(1, firstColon);
+            List<String> names = new ArrayList<>();
+            List<String> values = new ArrayList<>();
+            List<String> uris = new ArrayList<>();
+            List<String> localNames = new ArrayList<>();
+            List<String> types = new ArrayList<>();
+            List<String> qNames = new ArrayList<>();
+            firstFinish = findAttributeChildren(firstFinish, names, values, uris, localNames, types, qNames);
+            Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
+
+            handler.startElement(uri, firstBit, firstBit, atts);
+            buildElement(abvsFileNode, firstFinish + 1);
+            handler.endElement(uri, firstBit, firstBit);
+
+        }
+
+        if ("textnode".equals(type)) {
             char[] charContents = lastBit.toCharArray();
             handler.characters(charContents, 0, charContents.length);
 
         }
-        if (type.equals("element")) {
-            List<String> names = new ArrayList<String>();
-            List<String> values = new ArrayList<String>();
-            List<String> uris = new ArrayList<String>();
-            List<String> localNames = new ArrayList<String>();
-            List<String> types = new ArrayList<String>();
-            List<String> qNames = new ArrayList<String>();
+        if ("element".equals(type)) {
+            List<String> names = new ArrayList<>();
+            List<String> values = new ArrayList<>();
+            List<String> uris = new ArrayList<>();
+            List<String> localNames = new ArrayList<>();
+            List<String> types = new ArrayList<>();
+            List<String> qNames = new ArrayList<>();
             whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames);
             Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
 
@@ -264,7 +309,7 @@ public class VXQueryIndexReader {
         String adultPath = adultId.substring(0, lastDotAdult);
         adultPath = adultPath.replaceFirst(":", ".");
 
-        return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."));
+        return childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + ".");
     }
 
     boolean isDirectChildAttribute(IndexableField child, IndexableField adult) {
@@ -278,7 +323,7 @@ public class VXQueryIndexReader {
 
         String childType = childSegments[childSegments.length - 1];
 
-        return (childPath.startsWith(adultPath + ":") && childType.equals("attribute"));
+        return childPath.startsWith(adultPath + ":") && "attribute".equals(childType);
     }
 
 }
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+package org.apache.vxquery.runtime.functions.index.centralizer;
 
 import java.io.DataOutput;
 import java.io.File;
@@ -49,46 +49,54 @@ import org.apache.vxquery.datamodel.values.ValueTag;
  */
 public class IndexCentralizerUtil {
 
-    private final String FILE_NAME = "VXQuery-Index-Directory.xml";
+    private static final String FILE_NAME = "VXQuery-Index-Directory.xml";
     private final List<String> collections = new ArrayList<>();
-    private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
-    private File XML_FILE;
-    private String INDEX_LOCATION;
-    private static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
+    private static final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
+    private File xmlFile;
+    private String indexPath;
+    public static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
+    private static final StringValueBuilder svb = new StringValueBuilder();
+    private final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+    private final DataOutput output = abvs.getDataOutput();
 
     public IndexCentralizerUtil(File index) {
-        this.INDEX_LOCATION = index.getPath();
+        indexPath = index.getPath();
         if (!index.exists()) {
             try {
                 FileUtils.forceMkdir(index);
             } catch (IOException e) {
-                LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e);
+                LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + indexPath + " " + e);
             }
         }
-        XML_FILE = new File(index.getPath() + "/" + FILE_NAME);
+        xmlFile = new File(index.getPath() + "/" + FILE_NAME);
     }
 
     /**
      * Get the index directory containing index of the given collection
      *
-     * @param collection : Collection folder
+     * @param collection
+     *            : Collection folder
      * @return Index folder.
      */
     public String getIndexForCollection(String collection) {
-        return indexCollectionMap.get(collection).getIndex();
+        if (indexCollectionMap.size() > 0 && indexCollectionMap.containsKey(collection)) {
+            return indexCollectionMap.get(collection).getIndex();
+        }
+        return null;
     }
 
     /**
      * Put the index location corresponding to given collection.
      * Index location is created by using the last 100 characters of collection.
      *
-     * @param collection : Collection directory
+     * @param collection
+     *            : Collection directory
      * @return index
      */
     public String putIndexForCollection(String collection) {
         int length = collection.replaceAll("/", "").length();
         String index = collection.replaceAll("/", "");
-        index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index);
+        index = indexPath + "/" + (length > 100 ? index.substring(length - 100) : index);
         IndexLocator il = new IndexLocator();
         il.setCollection(collection);
         il.setIndex(index);
@@ -102,7 +110,8 @@ public class IndexCentralizerUtil {
     /**
      * Remove the entry for given collection directory.
      *
-     * @param collection : Collection directory
+     * @param collection
+     *            : Collection directory
      */
     public void deleteEntryForCollection(String collection) {
         indexCollectionMap.remove(collection);
@@ -110,14 +119,15 @@ public class IndexCentralizerUtil {
 
     /**
      * Prints all collections which have an index created.
-     * @param sb : The output is stored in a sequence
-     * @throws IOException : If writing the dataOutput generates {@link IOException}
+     *
+     * @param sb
+     *            : The output is stored in a sequence
+     * @throws IOException
+     *             : If writing the dataOutput generates {@link IOException}
      */
     public void getAllCollections(SequenceBuilder sb) throws IOException {
         for (String s : collections) {
-            StringValueBuilder svb = new StringValueBuilder();
-            ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-            DataOutput output = abvs.getDataOutput();
+            abvs.reset();
             output.write(ValueTag.XS_STRING_TAG);
             svb.write(s, output);
             sb.addItem(abvs);
@@ -128,11 +138,11 @@ public class IndexCentralizerUtil {
      * Read the collection, index directory file and populate the HashMap.
      */
     public void readIndexDirectory() {
-        if (this.XML_FILE.exists()) {
+        if (xmlFile.exists()) {
             try {
                 JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class);
                 Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
-                IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE);
+                IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(xmlFile);
 
                 for (IndexLocator il : indexDirectory.getDirectory()) {
                     indexCollectionMap.put(il.getCollection(), il);
@@ -153,7 +163,7 @@ public class IndexCentralizerUtil {
         List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values());
         id.setDirectory(indexLocators);
         try {
-            FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE);
+            FileOutputStream fileOutputStream = new FileOutputStream(this.xmlFile);
             JAXBContext context = JAXBContext.newInstance(IndexDirectory.class);
             Marshaller jaxbMarshaller = context.createMarshaller();
             jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+package org.apache.vxquery.runtime.functions.index.centralizer;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
 
 @XmlRootElement(name = "indexes")
 @XmlAccessorType(XmlAccessType.FIELD)
-public class IndexDirectory implements Serializable{
+public class IndexDirectory implements Serializable {
+    private static final long serialVersionUID = 1L;
 
     @XmlElement(name = "index", type = IndexLocator.class)
     private List<IndexLocator> directory = new ArrayList<>();
@@ -35,7 +37,6 @@ public class IndexDirectory implements Serializable{
         return directory;
     }
 
-
     public void setDirectory(List<IndexLocator> directory) {
         this.directory = directory;
     }
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+package org.apache.vxquery.runtime.functions.index.centralizer;
+
+import java.io.Serializable;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
 
 @XmlAccessorType(XmlAccessType.FIELD)
 @XmlRootElement(name = "Entry")
-public class IndexLocator implements Serializable{
+public class IndexLocator implements Serializable {
+    private static final long serialVersionUID = 1L;
 
     @XmlAttribute
     private String collection;
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
 
 /**
  * Constants used in updating index
  */
 public class Constants {
-    public static String FIELD_PATH = "path";
-    public static String META_FILE_NAME = "vxquery_index.xml";
+    public static final String FIELD_PATH = "path";
+    public static final String META_FILE_NAME = "vxquery_index.xml";
+
+    private Constants() {
+    }
 }
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
 import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.lucene.index.IndexWriter;
@@ -27,7 +34,6 @@ import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
 import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
 import org.apache.vxquery.exceptions.ErrorCode;
 import org.apache.vxquery.exceptions.SystemException;
@@ -36,16 +42,6 @@ import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer;
 import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
 import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
 
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.text.SimpleDateFormat;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
 /**
  * Update the index if the source files are changed.
  */
@@ -53,48 +49,37 @@ public class IndexUpdater {
     private MetaFileUtil metaFileUtil;
     private ConcurrentHashMap<String, XmlMetadata> metadataMap;
     private IPointable result;
-    private ByteBufferInputStream bbis;
-    private DataInputStream di;
-    private SequenceBuilder sb;
+    private final SequenceBuilder sb = new SequenceBuilder();
     private ArrayBackedValueStorage abvs;
     private ITreeNodeIdProvider nodeIdProvider;
     private ArrayBackedValueStorage abvsFileNode;
-    private TaggedValuePointable nodep;
     private String nodeId;
     private IndexWriter indexWriter;
     private Set<String> pathsFromFileList;
     private String collectionFolder;
     private String indexFolder;
-    private Logger LOGGER = Logger.getLogger("Index Updater");
+    private final Logger LOGGER = Logger.getLogger("Index Updater");
     private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
     private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
 
-    public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp, ByteBufferInputStream bbis,
-            DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider,
-            ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, String nodeId) {
+    public IndexUpdater(String indexFolder, IPointable result, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider,
+            ArrayBackedValueStorage abvsFileNode, String nodeId) {
         this.indexFolder = indexFolder;
         this.result = result;
-        this.bbis = bbis;
-        this.di = di;
-        this.sb = sb;
         this.abvs = abvs;
         this.nodeIdProvider = nodeIdProvider;
         this.abvsFileNode = abvsFileNode;
-        this.nodep = nodep;
         this.nodeId = nodeId;
         this.pathsFromFileList = new HashSet<>();
     }
 
     /**
      * Perform the initial configuration for index update/ delete processes.
-     * 
-     * @throws SystemException
-     *             : If getting the index folder generates {@link SystemException}
+     *
      * @throws IOException
      *             : If getting the index folder generates {@link IOException}
      */
-    public void setup() throws SystemException, IOException {
-
+    public void setup() throws IOException {
         // Read the metadata file and load the metadata map into memory.
         metaFileUtil = new MetaFileUtil(indexFolder);
         metaFileUtil.readMetadataFile();
@@ -114,14 +99,14 @@ public class IndexUpdater {
 
     /**
      * Wrapper for update index function.
-     * 
+     *
      * @throws IOException
      *             : If the directory doesn't exist
      */
     public void updateIndex() throws IOException {
         File collectionDirectory = new File(collectionFolder);
         if (!collectionDirectory.exists()) {
-            throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
+            throw new IOException("The collection directory (" + collectionFolder + ") does not exist.");
         }
 
         //Execute update index process
@@ -134,7 +119,7 @@ public class IndexUpdater {
 
     /**
      * Close opened IndexWriter and terminate the index update/ delete process.
-     * 
+     *
      * @throws IOException
      *             : If exiting the index folder generates {@link IOException}
      */
@@ -149,7 +134,7 @@ public class IndexUpdater {
 
     /**
      * Functional wrapper to update Metadata file.
-     * 
+     *
      * @throws IOException
      *             : If updating metadata folder generates {@link IOException}
      */
@@ -190,8 +175,8 @@ public class IndexUpdater {
 
                         //Update index corresponding to the xml file.
                         indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath()));
-                        indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep,
-                                abvsFileNode, nodeIdProvider, bbis, di, nodeId);
+                        indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode,
+                                nodeIdProvider, nodeId);
                         indexDocumentBuilder.printStart();
 
                         if (LOGGER.isDebugEnabled()) {
@@ -207,8 +192,8 @@ public class IndexUpdater {
 
                     // In this case, the xml file has not added to the index. (It is a newly added file)
                     // Therefore generate a new index for this file and add it to the existing index.
-                    indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, abvsFileNode,
-                            nodeIdProvider, bbis, di, nodeId);
+                    indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode, nodeIdProvider,
+                            nodeId);
                     indexDocumentBuilder.printStart();
 
                     if (LOGGER.isDebugEnabled()) {
@@ -235,8 +220,8 @@ public class IndexUpdater {
      * @throws IOException
      *             : If getting the file info generates {@link IOException}
      */
-    private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException {
-
+    private XmlMetadata updateEntry(File file, XmlMetadata metadataArg) throws IOException {
+        XmlMetadata metadata = metadataArg;
         if (metadata == null) {
             metadata = new XmlMetadata();
         }
@@ -286,7 +271,7 @@ public class IndexUpdater {
      * When deleting indexes, if any error occurred, the process will be rolled back and all the indexes will be
      * restored.
      * Otherwise the changes will be committed.
-     * 
+     *
      * @throws SystemException
      *             : An attempt to divide by zero
      */
@@ -310,7 +295,7 @@ public class IndexUpdater {
                 sb.finish();
                 result.set(abvs);
             } catch (IOException e1) {
-                throw new SystemException(ErrorCode.FOAR0001);
+                throw new SystemException(ErrorCode.FOAR0001, e1);
             }
         }
 
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
 
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import javax.xml.bind.DatatypeConverter;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
@@ -37,13 +29,22 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import javax.xml.bind.DatatypeConverter;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
 /**
  * Utility class for writing, reading metadata file and generating checksum.
  */
 public class MetaFileUtil {
 
     private File metaFile;
-    private Logger LOGGER = Logger.getLogger("MetadataFileUtil");
+    private static final Logger LOGGER = Logger.getLogger("MetadataFileUtil");
     private String index;
     private String collection;
     private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>();
@@ -65,8 +66,11 @@ public class MetaFileUtil {
      * Update the content of the metadata map.
      * If the current collection data is present, replace it.
      * Otherwise insert new.
-     * @param metadataMap : Set of XmlMetaData objects.
-     * @param index : The path to index location.
+     *
+     * @param metadataMap
+     *            : Set of XmlMetaData objects.
+     * @param index
+     *            : The path to index location.
      */
     public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) {
         this.indexMap = metadataMap;
@@ -109,22 +113,22 @@ public class MetaFileUtil {
      * Write the content of the ConcurrentHashMap to the xml metadata file.
      */
     public void writeMetadataToFile() {
-        XmlMetadataCollection collection = new XmlMetadataCollection();
+        XmlMetadataCollection xmlMetadataCollection = new XmlMetadataCollection();
         List<XmlMetadata> metadataList = new ArrayList<>();
 
         for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) {
             metadataList.add(entry.getValue());
         }
 
-        collection.setMetadataList(metadataList);
-        collection.setCollection(this.collection);
-        collection.setIndexLocation(this.index);
-        try{
+        xmlMetadataCollection.setMetadataList(metadataList);
+        xmlMetadataCollection.setCollection(collection);
+        xmlMetadataCollection.setIndexLocation(this.index);
+        try {
             FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
             JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
             Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
             jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
-            jaxbMarshaller.marshal(collection, fileOutputStream);
+            jaxbMarshaller.marshal(xmlMetadataCollection, fileOutputStream);
 
             if (LOGGER.isDebugEnabled()) {
                 LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
@@ -135,18 +139,18 @@ public class MetaFileUtil {
             }
         }
 
-
     }
 
-
     /**
      * Generate MD5 checksum string for a given file.
      *
-     * @param file : File which the checksum should be generated.
+     * @param file
+     *            : File which the checksum should be generated.
      * @return : Checksum String
-     * @throws IOException : The file is not available
+     * @throws IOException
+     *             : The file is not available
      */
-    public String generateMD5(File file) throws  IOException {
+    public String generateMD5(File file) throws IOException {
         try {
             MessageDigest md = MessageDigest.getInstance("MD5");
             md.update(Files.readAllBytes(file.toPath()));
@@ -168,12 +172,12 @@ public class MetaFileUtil {
     public boolean deleteMetaDataFile() {
         try {
             Files.delete(Paths.get(metaFile.getCanonicalPath()));
-            if (LOGGER.isDebugEnabled()){
+            if (LOGGER.isDebugEnabled()) {
                 LOGGER.log(Level.DEBUG, "Metadata file deleted!");
             }
             return true;
         } catch (IOException e) {
-            if (LOGGER.isTraceEnabled()){
+            if (LOGGER.isTraceEnabled()) {
                 LOGGER.log(Level.ERROR, "Metadata file could not be deleted!");
             }
             return false;
@@ -182,6 +186,7 @@ public class MetaFileUtil {
 
     /**
      * Get the collection for a given index location.
+     *
      * @return collection folder for a given index.
      */
     public String getCollection() {
@@ -190,7 +195,9 @@ public class MetaFileUtil {
 
     /**
      * Set the entry for given index and collection.
-     * @param collection : path to corresponding collection
+     *
+     * @param collection
+     *            : path to corresponding collection
      */
     public void setCollection(String collection) {
         this.collection = collection;
@@ -14,7 +14,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
+
+import java.io.Serializable;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
 
 /**
  * Class to store metadata related to an XML file.
@@ -32,6 +33,7 @@ import java.io.Serializable;
 @XmlRootElement(name = "file")
 @XmlAccessorType(XmlAccessType.FIELD)
 public class XmlMetadata implements Serializable {
+    private static final long serialVersionUID = 1L;
 
     private String path;
     private String md5;
@@ -14,7 +14,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
index 665c812..85ef4ca 100644 (file)
@@ -64,7 +64,7 @@ public class JnDocScalarEvaluatorFactory extends AbstractTaggedValueArgumentScal
                 tvp.getValue(stringp);
                 try {
                     IParser parser = new JSONParser();
-                    FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+                    FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
                 } catch (IOException e) {
                     throw new SystemException(ErrorCode.FODC0002, e);
                 }
index db908f6..15fd624 100644 (file)
@@ -78,7 +78,7 @@ public class FnDocAvailableScalarEvaluatorFactory extends AbstractTaggedValueArg
                 tvp.getValue(stringp);
                 try {
                     IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
-                    FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+                    FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
                     XDMConstants.setTrue(result);
                 } catch (Exception e) {
                     XDMConstants.setFalse(result);
index 5f08a8e..2fd1755 100644 (file)
@@ -79,7 +79,7 @@ public class FnDocScalarEvaluatorFactory extends AbstractTaggedValueArgumentScal
                 try {
                     // Only one document should be parsed so its ok to have a unique parser.
                     IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
-                    FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+                    FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
                 } catch (Exception e) {
                     throw new SystemException(ErrorCode.SYSE0001, e);
                 }
index 6558274..1d66c4e 100644 (file)
@@ -16,7 +16,6 @@
  */
 package org.apache.vxquery.runtime.functions.util;
 
-import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.File;
 import java.io.FileInputStream;
@@ -37,7 +36,7 @@ import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
 import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
 import org.apache.hyracks.data.std.util.GrowableArray;
 import org.apache.hyracks.data.std.util.UTF8StringBuilder;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.hyracks.util.string.UTF8StringUtil;
 import org.apache.vxquery.context.DynamicContext;
 import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
 import org.apache.vxquery.datamodel.accessors.TypedPointables;
@@ -485,6 +484,15 @@ public class FunctionHelper {
         return true;
     }
 
+    public static String getStringFromBytes(byte[] bytes) {
+        if (bytes == null) {
+            return null;
+        }
+        StringBuilder sb = new StringBuilder();
+        UTF8StringUtil.toString(sb, bytes, 0);
+        return sb.toString();
+    }
+
     public static boolean compareTaggedValues(AbstractValueComparisonOperation aOp, TaggedValuePointable tvp1,
             TaggedValuePointable tvp2, DynamicContext dCtx, TypedPointables tp1, TypedPointables tp2)
             throws SystemException {
@@ -1215,13 +1223,12 @@ public class FunctionHelper {
         System.err.println(" printUTF8String END");
     }
 
-    public static void readInDocFromPointable(UTF8StringPointable stringp, ByteBufferInputStream bbis,
-            DataInputStream di, ArrayBackedValueStorage abvs, IParser parser) throws IOException {
-        readInDocFromString(stringp.toString(), bbis, di, abvs, parser);
+    public static void readInDocFromPointable(UTF8StringPointable stringp, ArrayBackedValueStorage abvs,
+            IParser parser) throws IOException {
+        readInDocFromString(stringp.toString(), abvs, parser);
     }
 
-    public static void readInDocFromString(String fName, ByteBufferInputStream bbis, DataInputStream di,
-            ArrayBackedValueStorage abvs, IParser parser) throws IOException {
+    public static void readInDocFromString(String fName, ArrayBackedValueStorage abvs, IParser parser) throws IOException {
         Reader input;
         if (!fName.contains("hdfs:/")) {
             File file = new File(fName);
index 84c8ddf..9e21f53 100644 (file)
@@ -129,8 +129,6 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
     public SAXContentHandler(boolean attachTypes, ITreeNodeIdProvider nodeIdProvider, IFrameFieldAppender appender,
             List<SequenceType> childSequenceTypes) {
         this(attachTypes, nodeIdProvider, false);
-
-        // Frame writing variables
         this.appender = appender;
         setChildPathSteps(childSequenceTypes);
     }
@@ -297,10 +295,11 @@ public class SAXContentHandler implements ContentHandler, LexicalHandler {
 
     /**
      * The filter settings here are similar to one in the class linked below.
-     *
+     * 
+     * @throws SAXException
      * @see org.apache.vxquery.runtime.functions.step.NodeTestFilter.java
      */
-    private boolean startElementChildPathStep(String uri, String localName) {
+    private boolean startElementChildPathStep(String uri, String localName) throws SAXException {
         if (subElement != null && depth <= subElement.length) {
             // Check path step if it exists.
             subElement[depth - 1] = true;
index 0c252b4..d3d02ae 100644 (file)
@@ -222,7 +222,7 @@ public class XMLQueryCompiler {
         }
     }
 
-    public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel)
+    public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel, List<String> collections)
             throws AlgebricksException, SystemException {
         moduleNode = XMLQueryParser.parse(name, query);
         listener.notifyParseResult(moduleNode);
@@ -230,7 +230,7 @@ public class XMLQueryCompiler {
         pprinter = new LogicalOperatorPrettyPrintVisitor(new AlgebricksAppendable(),
                 new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext()));
         VXQueryMetadataProvider mdProvider = new VXQueryMetadataProvider(nodeList, ccb.getSourceFileMap(),
-                module.getModuleContext(), this.hdfsConf, nodeControllerInfos);
+                module.getModuleContext(), this.hdfsConf, nodeControllerInfos, collections);
         compiler = cFactory.createCompiler(module.getBody(), mdProvider, 0);
         listener.notifyTranslationResult(module);
         XMLQueryTypeChecker.typeCheckModule(module);
index 60f39f8..45d553f 100644 (file)
@@ -23,8 +23,8 @@ import java.util.concurrent.ConcurrentHashMap;
 import javax.xml.bind.JAXBException;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
+import org.apache.vxquery.runtime.functions.index.update.MetaFileUtil;
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
index b79107e..1b3e0b7 100644 (file)
@@ -14,8 +14,6 @@
  */
 package org.apache.vxquery.indexing;
 
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
-
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.File;
@@ -27,6 +25,8 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
+
 /**
  * TestConstants and methods which will be used in indexing test cases.
  */
index 7646f97..1ee35a4 100644 (file)
@@ -142,6 +142,6 @@ public class SimpleXQueryTest {
         XMLQueryCompiler compiler = new XMLQueryCompiler(null, nodeControllerInfos, 65536);
         CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
                 new ResultSetId(System.nanoTime()), null);
-        compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE);
+        compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE, null);
     }
 }
index e4ba6eb..fa0a900 100644 (file)
  */
 package org.apache.vxquery.xtest;
 
+import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.client.NodeControllerInfo;
 import org.apache.hyracks.api.comm.IFrame;
@@ -53,13 +59,14 @@ import org.apache.vxquery.xmlquery.query.XMLQueryCompiler;
 
 public class TestRunner {
     private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})");
-
+    private List<String> collectionList;
     private XTestOptions opts;
     private IHyracksClientConnection hcc;
     private IHyracksDataset hds;
 
     public TestRunner(XTestOptions opts) throws UnknownHostException {
         this.opts = opts;
+        this.collectionList = new ArrayList<String>();
     }
 
     public void open() throws Exception {
@@ -67,8 +74,46 @@ public class TestRunner {
         hds = TestClusterUtil.getDataset();
     }
 
+    protected static TestConfiguration getIndexConfiguration(TestCase testCase) {
+        XTestOptions opts = new XTestOptions();
+        opts.verbose = false;
+        opts.threads = 1;
+        opts.showQuery = true;
+        opts.showResult = true;
+        opts.hdfsConf = "src/test/resources/hadoop/conf";
+        opts.catalog = StringUtils.join(new String[] { "src", "test", "resources", "VXQueryCatalog.xml" },
+                File.separator);
+        TestConfiguration indexConf = new TestConfiguration();
+        indexConf.options = opts;
+        String baseDir = new File(opts.catalog).getParent();
+        try {
+            String root = new File(baseDir).getCanonicalPath();
+            indexConf.testRoot = new File(root + "/./");
+            indexConf.resultOffsetPath = new File(root + "/./ExpectedResults/");
+            indexConf.sourceFileMap = testCase.getSourceFileMap();
+            indexConf.xqueryFileExtension = ".xq";
+            indexConf.xqueryxFileExtension = "xqx";
+            indexConf.xqueryQueryOffsetPath = new File(root + "/./Queries/XQuery/");
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return indexConf;
+
+    }
+
     public TestCaseResult run(final TestCase testCase) {
         TestCaseResult res = new TestCaseResult(testCase);
+        TestCase testCaseIndex = new TestCase(getIndexConfiguration(testCase));
+        testCaseIndex.setFolder("Indexing/Partition-1/");
+        testCaseIndex.setName("showIndexes");
+        runQuery(testCaseIndex, res);
+        String[] collections = res.result.split("\n");
+        this.collectionList = Arrays.asList(collections);
+        runQueries(testCase, res);
+        return res;
+    }
+
+    public void runQuery(TestCase testCase, TestCaseResult res) {
         if (opts.verbose) {
             System.err.println("Starting " + testCase.getXQueryDisplayName());
         }
@@ -78,6 +123,7 @@ public class TestRunner {
         try {
             try {
                 if (opts.showQuery) {
+
                     FileInputStream query = new FileInputStream(testCase.getXQueryFile());
                     System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": ");
                     System.err.println(IOUtils.toString(query, "UTF-8"));
@@ -98,7 +144,7 @@ public class TestRunner {
                 CompilerControlBlock ccb = new CompilerControlBlock(
                         new StaticContextImpl(RootStaticContextImpl.INSTANCE),
                         new ResultSetId(testCase.getXQueryDisplayName().hashCode()), testCase.getSourceFileMap());
-                compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel);
+                compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel, collectionList);
                 JobSpecification spec = compiler.getModule().getHyracksJobSpecification();
                 in.close();
 
@@ -172,7 +218,11 @@ public class TestRunner {
                 System.err.println(res.result);
             }
         }
-        return res;
+
+    }
+
+    public void runQueries(TestCase testCase, TestCaseResult res) {
+        runQuery(testCase, res);
     }
 
     public void close() throws Exception {
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt
new file mode 100644 (file)
index 0000000..baf9dca
--- /dev/null
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt
new file mode 100644 (file)
index 0000000..baf9dca
--- /dev/null
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt
new file mode 100644 (file)
index 0000000..baf9dca
--- /dev/null
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
index 63fdda7..96a7671 100644 (file)
    under the License. :)
    
 (: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
 let $datetime := xs:dateTime(fn:data($r/date))
 where $r/station eq "GHCND:AS000000003" 
     and fn:year-from-dateTime($datetime) ge 2000
     and fn:month-from-dateTime($datetime) eq 3 
     and fn:day-from-dateTime($datetime) eq 3
-return $r
\ No newline at end of file
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq
new file mode 100644 (file)
index 0000000..7a2bb2c
--- /dev/null
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+   
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003" 
+    and fn:year-from-dateTime($datetime) ge 2000
+    and fn:month-from-dateTime($datetime) eq 3 
+    and fn:day-from-dateTime($datetime) eq 3
+return $r
index cf41536..464c1cc 100644 (file)
@@ -19,6 +19,6 @@
 (: Find all reading for hurricane force wind warning or extreme wind warning. :)
 (: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744     :)
 (: meters per second). (Wind value is in tenth of a meter per second)         :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
 where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
 return $r
index 5c99d9a..eeb019b 100644 (file)
@@ -19,7 +19,7 @@
 (: Find the annual precipitation (PRCP) for a Seattle using the airport       :)
 (: station (US000000002) for 2002.                                            :)
 fn:sum(
-    for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+    for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
     where $r/station eq "GHCND:US000000002" 
         and $r/dataType eq "PRCP" 
         and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
index 39e5d17..06284ae 100644 (file)
@@ -18,7 +18,7 @@
 (: Search Lucene Index :)
 (: Find the highest recorded temperature (TMAX) in Celsius.                   :)
 fn:max(
-    for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+    for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
     where $r/dataType eq "TMAX"
     return $r/value
 ) div 10
index 63aeca5..3a4ae05 100644 (file)
@@ -18,6 +18,6 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
 where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
 return $r
index c81f271..9090edd 100644 (file)
@@ -18,7 +18,7 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd")/stationCollection/station
 where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
 order by $s/id
 return $s
index dd6b5f9..7c703b6 100644 (file)
@@ -18,8 +18,8 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
     
 where $s/id eq $r/station 
     and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
index fecb56d..4718240 100644 (file)
@@ -16,7 +16,7 @@
    under the License. :)
    
 (: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
 let $datetime := xs:dateTime(fn:data($r/date))
 where $r/station eq "GHCND:AS000000003" 
     and fn:year-from-dateTime($datetime) ge 2000
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq
new file mode 100644 (file)
index 0000000..0e42155
--- /dev/null
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+   
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003" 
+    and fn:year-from-dateTime($datetime) ge 2000
+    and fn:month-from-dateTime($datetime) eq 3 
+    and fn:day-from-dateTime($datetime) eq 3
+return $r
index 75c7a64..37e5626 100644 (file)
@@ -19,6 +19,6 @@
 (: Find all reading for hurricane force wind warning or extreme wind warning. :)
 (: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744     :)
 (: meters per second). (Wind value is in tenth of a meter per second)         :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
 where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
 return $r
index 28f7473..358d3f3 100644 (file)
@@ -19,7 +19,7 @@
 (: Find the annual precipitation (PRCP) for a Seattle using the airport       :)
 (: station (US000000002) for 2002.                                            :)
 fn:sum(
-    for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+    for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
     where $r/station eq "GHCND:US000000002" 
         and $r/dataType eq "PRCP" 
         and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
index 317a141..bd5ba1c 100644 (file)
@@ -18,7 +18,7 @@
 (: Search Lucene Index :)
 (: Find the highest recorded temperature (TMAX) in Celsius.                   :)
 fn:max(
-    for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+    for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
     where $r/dataType eq "TMAX"
     return $r/value
 ) div 10
index 2deb4c3..77f6c2f 100644 (file)
@@ -18,6 +18,6 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
 where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
 return $r
index a0ce1e9..c1a45f4 100644 (file)
@@ -18,6 +18,6 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/stationCollection/station
 where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
 return $s
index b3e622c..a776ab9 100644 (file)
@@ -18,8 +18,8 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
     
 where $s/id eq $r/station 
     and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
index 0cccbc5..2bc9ce7 100644 (file)
@@ -16,7 +16,7 @@
    under the License. :)
    
 (: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
 let $datetime := xs:dateTime(fn:data($r/date))
 where $r/station eq "GHCND:AS000000003" 
     and fn:year-from-dateTime($datetime) ge 2000
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq
new file mode 100644 (file)
index 0000000..e740365
--- /dev/null
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+   
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003" 
+    and fn:year-from-dateTime($datetime) ge 2000
+    and fn:month-from-dateTime($datetime) eq 3 
+    and fn:day-from-dateTime($datetime) eq 3
+return $r
index c282e31..a1b86ac 100644 (file)
@@ -19,6 +19,6 @@
 (: Find all reading for hurricane force wind warning or extreme wind warning. :)
 (: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744     :)
 (: meters per second). (Wind value is in tenth of a meter per second)         :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
 where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
 return $r
index 33ea1c9..9cc2b8e 100644 (file)
@@ -19,7 +19,7 @@
 (: Find the annual precipitation (PRCP) for a Seattle using the airport       :)
 (: station (US000000002) for 2002.                                            :)
 fn:sum(
-    for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+    for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
     where $r/station eq "GHCND:US000000002" 
         and $r/dataType eq "PRCP" 
         and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
index d213082..dd26e87 100644 (file)
@@ -18,7 +18,7 @@
 (: Search Lucene Index :)
 (: Find the highest recorded temperature (TMAX) in Celsius.                   :)
 fn:max(
-    for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+    for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
     where $r/dataType eq "TMAX"
     return $r/value
 ) div 10
index 1d98682..4a9d224 100644 (file)
@@ -18,6 +18,6 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
 where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
 return $r
index abe2184..4407079 100644 (file)
@@ -18,6 +18,6 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/stationCollection/station
 where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
 return $s
index 7b40ca0..664b150 100644 (file)
@@ -18,8 +18,8 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
     
 where $s/id eq $r/station 
     and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
index 5ecdb94..8e28135 100644 (file)
          &LibrariesInJSONiq;
         </test-group>
     </test-group>
-   <test-group name="SerializationQueries" featureOwner="Christina Pavlopoulou">
+    <test-group name="SerializationQueries" featureOwner="Christina Pavlopoulou">
       <GroupInfo>
          <title>Serialize Function Queries</title>
          <description/>
          </GroupInfo>
          &SerializationQueries;
       </test-group>
-   </test-group>
+    </test-group>
     <test-group name="XMLInJSONQueries" featureOwner="Riyafa Abdul Hameed">
         <GroupInfo>
             <title>XML in JSON</title>
index 7cf6bf6..cc6b65b 100644 (file)
       <query name="useIndex1" date="2016-05-26"/>
       <output-file compare="Text">useIndex1.txt</output-file>
    </test-case>
+   <test-case name="use-index-1-user" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex1_user" date="2016-05-26"/>
+      <output-file compare="Text">useIndex1_user.txt</output-file>
+   </test-case>
    <test-case name="use-index-2" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex2" date="2016-05-26"/>
       <query name="useIndex1" date="2016-05-26"/>
       <output-file compare="Text">useIndex1.txt</output-file>
    </test-case>
+   <test-case name="use-index-1-user" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex1_user" date="2016-05-26"/>
+      <output-file compare="Text">useIndex1_user.txt</output-file>
+   </test-case>
    <test-case name="use-index-2" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex2" date="2016-05-26"/>
       <query name="useIndex1" date="2016-05-26"/>
       <output-file compare="Text">useIndex1.txt</output-file>
    </test-case>
+   <test-case name="use-index-1-user" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex1_user" date="2016-05-26"/>
+      <output-file compare="Text">useIndex1_user.txt</output-file>
+   </test-case>
    <test-case name="use-index-2" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex2" date="2016-05-26"/>