VXQUERY-207 VXQUERY-209 Parallel Index creation and access, and index centralization 153/head 154/head
authorSteven Glenn Jacobs <sjaco002@ucr.edu>
Wed, 7 Sep 2016 22:44:47 +0000 (15:44 -0700)
committerSteven Glenn Jacobs <sjaco002@ucr.edu>
Wed, 7 Sep 2016 23:09:07 +0000 (16:09 -0700)
Fixed issue in Tests where no result file meant success

102 files changed:
src/site/apt/user_indexing.apt [new file with mode: 0644]
vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
vxquery-core/src/main/java/org/apache/vxquery/metadata/AbstractVXQueryDataSource.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CollectionFromIndexUnnestingEvaluatorFactory.java [deleted file]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java [deleted file]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java [deleted file]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java [moved from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorScalarEvaluatorFactory.java with 58% similarity]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java [new file with mode: 0644]
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
vxquery-server/src/main/resources/conf/cluster_example.xml
vxquery-server/src/main/resources/conf/local.xml
vxquery-server/src/main/resources/scripts/cluster_actions.py
vxquery-server/src/main/resources/scripts/cluster_information.py
vxquery-server/src/main/resources/scripts/startnc.sh
vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/createIndex.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/deleteIndex.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/updateIndex.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex1.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex2.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex3.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex4.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex5.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex6.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt [moved from vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/useIndex7.txt with 100% similarity]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/showIndex1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/updateIndex.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex2.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex3.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex4.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex5.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex6.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex7.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/createIndex.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/showIndex1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/updateIndex.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex2.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex3.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex4.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex5.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex6.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex7.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/project.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/remove_keys.txt [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/createIndex.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/createIndex.xq with 89% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/deleteIndex.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/deleteIndex.xq with 93% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndex.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/updateIndex.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/updateIndex.xq with 93% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex1.xq with 90% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex2.xq with 91% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex3.xq with 91% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex4.xq with 90% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex5.xq with 91% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex6.xq with 91% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq [moved from vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/useIndex7.xq with 85% similarity]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/createIndex.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/showIndex.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/updateIndex.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/createIndex.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/showIndex.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/updateIndex.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq [new file with mode: 0644]
vxquery-xtest/src/test/resources/cat/IndexingQueries.xml

diff --git a/src/site/apt/user_indexing.apt b/src/site/apt/user_indexing.apt
new file mode 100644 (file)
index 0000000..b572231
--- /dev/null
@@ -0,0 +1,202 @@
+~~ Licensed to the Apache Software Foundation (ASF) under one or more
+~~ contributor license agreements.  See the NOTICE file distributed with
+~~ this work for additional information regarding copyright ownership.
+~~ The ASF licenses this file to You under the Apache License, Version 2.0
+~~ (the "License"); you may not use this file except in compliance with
+~~ the License.  You may obtain a copy of the License at
+~~
+~~     http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License.
+
+How to use Indexing Features in VXQuery.
+
+In VXQuery, all the indexes are created in user specified directory. In order to use indexing,
+you will need to set this directory in your cluster configuration file.
+
+*** Configuring VXQuery to use indexing functions.
+Add the following line to your cluster configuration (e.g. cluster.xml)
+
+--------
+    <index_directory><path_to_index_directory></index_directory>
+--------
+
+(You should create this index_directory)
+
+** Using indexing queries.
+
+VXQuery offers following indexing functionality.
+
+[[a]] Create an index for collection.
+[[b]] Use the index in executing a query.
+[[c]] Update the index.
+[[d]] Delete the index.
+[[e]] View existing indexes.
+
+*1. Scenario I - When collection is a single directory.
+In this scenario, all the XML files are stored in a single directory. (There can be sub directories)
+
+*** Creating an index for collection
+If I need to create index for xml collection stored in <path_1>/collection1,
+
+Query structure:
+--------
+build-index-on-collection("collection")
+--------
+
+You can see the index has created in a new sub-directory in the index_directory specified in local.xml
+
+Example:
+--------
+build-index-on-collection("<path_1>/collection1")
+--------
+This function takes the collection path as an argument.
+
+*** Using index in query.
+If we need to use the index and execute a query, use the following structure.
+
+------
+for $r in collection-from-index("<path1>/collection1", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
+------
+Here the index access function is,
+
+------
+collection-from-index
+------
+
+which takes two arguments, collection folder and the path element.
+
+Result
+
+------
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
+------
+
+*** Updating the index.
+A collection can be modified or changed by following ways.
+[[1]] Inserting new XML files.
+[[2]] Deleting files.
+[[3]] Add/ remove or modify the content of XML files.
+
+In this type of situation, the index corresponding to the modified collection must also be modified. To achieve this
+the update-index function can be used.
+
+Query structure:
+--------
+update-index("<path_to_collection>")
+--------
+
+Example:
+-------
+update-index("<path_1>/collection1")
+-------
+
+This function takes the collection which was modified.
+
+*** Deleting the index.
+If we want to delete the entire index created for a collection, the delete-index function can be used.
+This function also takes the collection path of which the index is needed to be deleted.
+
+Query structure:
+--------
+delete-index("<path_to_collection>")
+--------
+
+Example:
+-------
+delete-index("<path_11>/collection1")
+-------
+
+*2. Scenario II - When the collection is distributed.
+In this scenario, the collection is distributed among several directories. We can distribute the queries among
+partitions.
+
+*** Creating indexes for collections.
+
+Query structure:
+--------
+build-index-on-collection("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>")
+--------
+
+In here the parameter contains the list of collection partitions separated by '|' character.
+
+Example:
+Consider the collection has now distributed among four directories, <path_1>/collection1, <path_2>/collection2,
+<path_3>/collection3 and <path_4>/collection4.
+
+To create indexes for all of the above collections,
+-------
+build-index-on-collection("<path_1>/collection1|<path_2>/collection2|<path_3>/collection3|<path_4>/collection4")
+-------
+
+In this case, all indexes will be created in separate sub-directories corresponding to each partition. Also note that
+ this query requires each node to have four partitions available
+
+*** Using the indexes in query.
+In this case, suppose you need to run a query on indexes of two collection partitions.
+
+Example:
+-----
+for $r in collection-from-index("<path_1>/collection1|<path-2>collection2", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
+-----
+
+The result will be taken from the indexes of both collection1 and collection2.
+
+Result:
+------
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
+------
+
+*** Updating the indexes.
+In cases of updating the collection files stored in several partitions, we can use this function to update the
+indexes of those directories.
+
+In this case, give a '|' separated list of collection directories.
+Query structure:
+--------
+update-index("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>")
+--------
+
+Example:
+Suppose that we need to update the indexes in partition1 and partition4
+--------
+update-index("<path_1>/collection1|<path_4>/collection4")
+--------
+
+*** Deleting the indexes.
+If we want to delete indexes of collections in several partitions, we can use this function.
+Query structure:
+--------
+delete-index("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>")
+--------
+
+Example:
+Suppose that we need to update the indexes in collection2 and collection3
+--------
+delete-index("<path_2>/collection2|<path_3>/collection3")
+--------
+
+** Viewing Index information.
+Suppose you need to check, what are the collections have indexes created. To do this, the show-index function can be
+used. This function takes no arguments and returns a sequence of collection paths, which an index is already created.
+If there are no indexes created for any collection, the result will be null.
+
+Suppose we have two collections, <path_1/collection1>, <path_2/collection2>  have indexes created.
+Example:
+------
+show-index()
+------
+
+Result:
+------
+<path_1/collection1>
+<path_2/collection2>
+------
index e18332e..e0e3843 100644 (file)
@@ -305,6 +305,7 @@ public class VXQuery {
             ncConfig.dataIPAddress = localAddress;
             ncConfig.resultIPAddress = localAddress;
             ncConfig.nodeId = "nc" + (i + 1);
+            //TODO: enable index folder as a cli option for on-the-fly indexing queries
             ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString();
             ncs[i] = new NodeControllerService(ncConfig);
             ncs[i].start();
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
new file mode 100644 (file)
index 0000000..ceaf3c7
--- /dev/null
@@ -0,0 +1,43 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.common;
+
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.vxquery.functions.BuiltinFunctions;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class VXQueryCommons {
+
+    public static final Set<FunctionIdentifier> collectionFunctions = new HashSet<>();
+
+    static {
+        collectionFunctions.add(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier());
+        collectionFunctions.add(BuiltinFunctions.FN_COLLECTION_WITH_TAG_2.getFunctionIdentifier());
+    }
+
+    public static final Set<FunctionIdentifier> indexingFunctions = new HashSet<>();
+
+    static {
+        indexingFunctions.add(BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier());
+        indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier());
+        indexingFunctions.add(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier());
+        indexingFunctions.add(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier());
+    }
+
+}
index 205e0b2..d5909c6 100644 (file)
@@ -28,6 +28,7 @@ import org.apache.vxquery.compiler.rewriter.rules.EliminateSubplanForSingleItems
 import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSequencesRule;
 import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSubplanRule;
 import org.apache.vxquery.compiler.rewriter.rules.IntroduceCollectionRule;
+import org.apache.vxquery.compiler.rewriter.rules.IntroduceIndexingRule;
 import org.apache.vxquery.compiler.rewriter.rules.IntroduceTwoStepAggregateRule;
 import org.apache.vxquery.compiler.rewriter.rules.PushChildIntoDataScanRule;
 import org.apache.vxquery.compiler.rewriter.rules.PushFunctionsOntoEqJoinBranches;
@@ -130,6 +131,7 @@ public class RewriteRuleset {
         normalization.add(new SetCollectionDataSourceRule());
         normalization.add(new IntroduceCollectionRule());
         normalization.add(new RemoveUnusedAssignAndAggregateRule());
+        normalization.add(new IntroduceIndexingRule());
 
         normalization.add(new ConsolidateDescandantChild());
 
index 74220da..eff7a6e 100644 (file)
@@ -20,6 +20,7 @@ import java.io.DataInputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Set;
 
 import org.apache.commons.lang3.mutable.Mutable;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -30,8 +31,11 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
 import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
 import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
@@ -40,7 +44,6 @@ import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue;
 import org.apache.vxquery.compiler.rewriter.rules.util.OperatorToolbox;
 import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
 import org.apache.vxquery.datamodel.values.ValueTag;
-import org.apache.vxquery.functions.BuiltinFunctions;
 import org.apache.vxquery.types.BuiltinTypeRegistry;
 import org.apache.vxquery.types.Quantifier;
 import org.apache.vxquery.types.SequenceType;
@@ -50,6 +53,7 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule {
     final DataInputStream di = new DataInputStream(bbis);
     final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
     final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+    public static AbstractFunctionCallExpression functionCall;
 
     /**
      * Get the arguments for the collection and collection-with-tag. Return null for not a collection.
@@ -58,8 +62,7 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule {
      *            Logical operator
      * @return collection name
      */
-    protected String[] getCollectionName(Mutable<ILogicalOperator> opRef) {
-
+    protected String[] getFunctionalArguments(Mutable<ILogicalOperator> opRef, Set<FunctionIdentifier> functions) {
         AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
             return null;
@@ -78,11 +81,9 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule {
         if (logicalExpression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
             return null;
         }
-        AbstractFunctionCallExpression functionCall = (AbstractFunctionCallExpression) logicalExpression;
-        if (!functionCall.getFunctionIdentifier()
-                .equals(BuiltinFunctions.FN_COLLECTION_WITH_TAG_2.getFunctionIdentifier())
-                && !functionCall.getFunctionIdentifier()
-                        .equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier())) {
+        functionCall = (AbstractFunctionCallExpression) logicalExpression;
+
+        if (!functions.contains(functionCall.getFunctionIdentifier())) {
             return null;
         }
 
@@ -144,6 +145,20 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule {
         return null;
     }
 
+    protected boolean setDataSourceScan(IDataSource<String> ids, Mutable<ILogicalOperator> opRef) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        UnnestOperator unnest = (UnnestOperator) op;
+        Mutable<ILogicalOperator> opRef2 = unnest.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        AssignOperator assign = (AssignOperator) op2;
+
+        DataSourceScanOperator opNew = new DataSourceScanOperator(assign.getVariables(), ids);
+        opNew.getInputs().addAll(assign.getInputs());
+        opRef2.setValue(opNew);
+
+        return true;
+    }
+
     @Override
     public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
index 8ed8bb1..42d59aa 100644 (file)
 package org.apache.vxquery.compiler.rewriter.rules;
 
 import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.vxquery.common.VXQueryCommons;
 import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext;
 import org.apache.vxquery.metadata.VXQueryCollectionDataSource;
 import org.apache.vxquery.types.AnyItemType;
 import org.apache.vxquery.types.Quantifier;
 import org.apache.vxquery.types.SequenceType;
 
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-
 /**
  * Find the default query plan created for collection and updated it to use
  * parallelization. The rule searches for unnest followed by an assign for the
@@ -61,10 +56,11 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperat
  * @author prestonc
  */
 public class IntroduceCollectionRule extends AbstractCollectionRule {
+
     @Override
     public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
-        String args[] = getCollectionName(opRef);
+        String args[] = getFunctionalArguments(opRef, VXQueryCommons.collectionFunctions);
 
         if (args != null) {
             String collectionName = args[0];
@@ -82,16 +78,7 @@ public class IntroduceCollectionRule extends AbstractCollectionRule {
                 }
 
                 // Known to be true because of collection name.
-                AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-                UnnestOperator unnest = (UnnestOperator) op;
-                Mutable<ILogicalOperator> opRef2 = unnest.getInputs().get(0);
-                AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-                AssignOperator assign = (AssignOperator) op2;
-
-                DataSourceScanOperator opNew = new DataSourceScanOperator(assign.getVariables(), ds);
-                opNew.getInputs().addAll(assign.getInputs());
-                opRef2.setValue(opNew);
-                return true;
+                return setDataSourceScan(ds, opRef);
             }
         }
         return false;
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
new file mode 100644 (file)
index 0000000..5b96131
--- /dev/null
@@ -0,0 +1,58 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.compiler.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.vxquery.common.VXQueryCommons;
+import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext;
+import org.apache.vxquery.metadata.VXQueryIndexingDataSource;
+import org.apache.vxquery.types.AnyItemType;
+import org.apache.vxquery.types.Quantifier;
+import org.apache.vxquery.types.SequenceType;
+
+/**
+ *
+ */
+public class IntroduceIndexingRule extends AbstractCollectionRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
+        String args[] = getFunctionalArguments(opRef, VXQueryCommons.indexingFunctions);
+
+        if (args != null) {
+
+            String collection = args[0];
+            String elementPath = args.length > 1?args[1]:null;
+
+            // Build the new operator and update the query plan.
+            int collectionId = vxqueryContext.newCollectionId();
+            VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection, elementPath,
+                    SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR), functionCall.getFunctionIdentifier().getName());
+            if (ids != null) {
+                ids.setTotalDataSources(vxqueryContext.getTotalDataSources());
+
+                // Known to be true because of collection name.
+                return setDataSourceScan(ids, opRef);
+            }
+        }
+        return false;
+    }
+}
index 7ffcd90..6060c19 100644 (file)
@@ -34,6 +34,7 @@ import org.apache.vxquery.compiler.rewriter.rules.util.ExpressionToolbox;
 import org.apache.vxquery.context.StaticContext;
 import org.apache.vxquery.functions.BuiltinOperators;
 import org.apache.vxquery.metadata.VXQueryCollectionDataSource;
+import org.apache.vxquery.metadata.VXQueryIndexingDataSource;
 import org.apache.vxquery.metadata.VXQueryMetadataProvider;
 import org.apache.vxquery.types.ElementType;
 
@@ -86,8 +87,16 @@ public class PushChildIntoDataScanRule extends AbstractUsedVariablesProcessingRu
         DataSourceScanOperator datascan = (DataSourceScanOperator) op2;
 
         if (!usedVariables.contains(datascan.getVariables())) {
+            VXQueryCollectionDataSource ds = null;
+            VXQueryIndexingDataSource ids = null;
+
             // Find all child functions.
-            VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) datascan.getDataSource();
+            try {
+                ids = (VXQueryIndexingDataSource) datascan.getDataSource();
+            } catch (ClassCastException e) {
+                ds = (VXQueryCollectionDataSource) datascan.getDataSource();
+            }
+
             if (!updateDataSource(ds, unnest.getExpressionRef())) {
                 return false;
             }
index 10ca007..688e86e 100644 (file)
         <!-- Collection operator is added during the rewrite rules phase.  -->
     </function>
 
-    <!-- fn:build-index-on-collection($arg  as xs:string?, $indexFolder  as xs:string?) as  node()* -->
+    <!-- fn:build-index-on-collection($arg  as xs:string?, $indexFolder  as xs:string?) as  boolean -->
     <function name="fn:build-index-on-collection">
         <param name="collection-folder" type="xs:string?"/>
-        <param name="index-folder" type="xs:string?"/>
         <return type="xs:boolean"/>
-        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.IndexConstructorScalarEvaluatorFactory"/>
+        <!-- build-index-on-collection operator is added during the rewrite rules phase.  -->
     </function>
 
-    <!-- fn:update-index($indexFolder  as xs:string?) as  node()* -->
+    <!-- fn:update-index($indexFolder  as xs:string?) as  boolean -->
     <function name="fn:update-index">
-        <param name="index-folder" type="xs:string?"/>
+        <param name="collection-folder" type="xs:string?"/>
         <return type="xs:boolean"/>
-        <runtime type="scalar"
-                 class="org.apache.vxquery.runtime.functions.index.IndexUpdaterEvaluatorFactory"/>
+        <!-- update-index operator is added during the rewrite rules phase.  -->
     </function>
 
-    <!-- fn:delete-index($indexFolder  as xs:string?) as  node()* -->
+    <!-- fn:delete-index($indexFolder  as xs:string?) as  boolean -->
     <function name="fn:delete-index">
-        <param name="index-folder" type="xs:string?"/>
+        <param name="collection-folder" type="xs:string?"/>
         <return type="xs:boolean"/>
-        <runtime type="scalar"
-                 class="org.apache.vxquery.runtime.functions.index.IndexDeleteEvaluatorFactory"/>
+        <!-- delete-index operator is added during the rewrite rules phase.  -->
     </function>
 
     <!-- fn:collection-from-index($indexfolder  as xs:string?, $elementpath as xs:string?) as  node()* -->
         <param name="index-folder" type="xs:string?"/>
         <param name="element-path" type="xs:string?"/>
         <return type="node()*"/>
-        <runtime type="unnesting" class="org.apache.vxquery.runtime.functions.index.CollectionFromIndexUnnestingEvaluatorFactory"/>
         <property type="DocumentOrder" class="org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.InputPropertyPropagationPolicy">
             <argument value="0"/>
         </property>
         <property type="UniqueNodes" class="org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.InputPropertyPropagationPolicy">
             <argument value="0"/>
         </property>
+        <!-- collection-from-index operator is added during the rewrite rules phase.  -->
+    </function>
+
+    <!-- fn:show-index as node()* -->
+    <function name="fn:show-index">
+        <return type="node()*"/>
+        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.ShowIndexScalarEvaluatorFactory"/>
     </function>
 
     <!-- fn:collection-with-tag($arg1  as xs:string?, $arg2 as xs:string?) as  node()* -->
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/AbstractVXQueryDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/AbstractVXQueryDataSource.java
new file mode 100644 (file)
index 0000000..dd9898c
--- /dev/null
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.metadata;
+
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+
+import java.util.List;
+
+public abstract class AbstractVXQueryDataSource {
+    protected static final String DELIMITER = "\\|";
+    protected int dataSourceId;
+    protected String collectionName;
+    protected String[] collectionPartitions;
+    protected String elementPath;
+    protected List<Integer> childSeq;
+    protected int totalDataSources;
+    protected String tag;
+    protected String function;
+
+    protected Object[] types;
+
+    protected IDataSourcePropertiesProvider propProvider;
+
+    public abstract String getFunctionCall();
+}
index 4d0e7a4..b4bc858 100644 (file)
@@ -30,18 +30,7 @@ import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalProperties
 import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
 import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
 
-public class VXQueryCollectionDataSource implements IDataSource<String> {
-    private static final String DELIMITER = "\\|";
-    private final int dataSourceId;
-    private final String collectionName;
-    private String[] collectionPartitions;
-    private final List<Integer> childSeq;
-    private int totalDataSources;
-    private String tag;
-
-    private final Object[] types;
-
-    private IDataSourcePropertiesProvider propProvider;
+public class VXQueryCollectionDataSource extends AbstractVXQueryDataSource implements IDataSource<String> {
 
     private VXQueryCollectionDataSource(int id, String file, Object[] types) {
         this.dataSourceId = id;
@@ -128,4 +117,9 @@ public class VXQueryCollectionDataSource implements IDataSource<String> {
     public String toString() {
         return "VXQueryCollectionDataSource [collectionName=" + collectionName + ", childSeq=" + childSeq + "]";
     }
+
+    @Override
+    public String getFunctionCall() {
+        return function;
+    }
 }
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
new file mode 100644 (file)
index 0000000..da75108
--- /dev/null
@@ -0,0 +1,137 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.metadata;
+
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import org.apache.vxquery.compiler.rewriter.rules.CollectionFileDomain;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Datasource object for indexing.
+ */
+public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource implements IDataSource<String> {
+
+    protected Object[] types;
+
+    protected IDataSourcePropertiesProvider propProvider;
+    private VXQueryIndexingDataSource(int id, String collection, String elementPath, Object[] types,
+            String functionCall) {
+        this.dataSourceId = id;
+        this.collectionName = collection;
+        this.elementPath = elementPath;
+        this.function = functionCall;
+        this.collectionPartitions = collectionName.split(DELIMITER);
+
+        this.types = types;
+        final IPhysicalPropertiesVector vec = new StructuralPropertiesVector(
+                new RandomPartitioningProperty(new CollectionFileDomain(collectionName)),
+                new ArrayList<>());
+        propProvider = new IDataSourcePropertiesProvider() {
+            @Override
+            public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables) {
+                return vec;
+            }
+        };
+        this.tag = null;
+        this.childSeq = new ArrayList<>();
+    }
+
+    public static VXQueryIndexingDataSource create(int id, String collection, String index, Object type, String
+            function) {
+        return new VXQueryIndexingDataSource(id, collection, index, new Object[] { type }, function);
+    }
+
+    public int getTotalDataSources() {
+        return totalDataSources;
+    }
+
+    public void setTotalDataSources(int totalDataSources) {
+        this.totalDataSources = totalDataSources;
+    }
+
+    public int getDataSourceId() {
+        return dataSourceId;
+    }
+
+    public String getElementPath() {
+        return elementPath;
+    }
+
+    public String[] getCollectionPartitions() {
+        return collectionPartitions;
+    }
+
+    public void setCollectionPartitions(String[] collectionPartitions) {
+        this.collectionPartitions = collectionPartitions;
+    }
+
+    public int getPartitionCount() {
+        return collectionPartitions.length;
+    }
+
+    public String getTag() {
+        return this.tag;
+    }
+
+    public void setTag(String tag) {
+        this.tag = tag;
+    }
+
+    @Override
+    public String getId() {
+        return collectionName;
+    }
+
+    @Override
+    public Object[] getSchemaTypes() {
+        return types;
+    }
+
+    @Override
+    public IDataSourcePropertiesProvider getPropertiesProvider() {
+        return propProvider;
+    }
+
+    @Override
+    public void computeFDs(List scanVariables, List fdList) {
+    }
+
+    @Override
+    public String toString() {
+        return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + elementPath + " "
+                + "function=" + function + "]";
+    }
+
+    @Override
+    public String getFunctionCall() {
+        return function;
+    }
+
+    public List<Integer> getChildSeq() {
+        return childSeq;
+    }
+
+}
+
+
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
new file mode 100644 (file)
index 0000000..ae637ac
--- /dev/null
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.metadata;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.client.NodeControllerInfo;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameFieldAppender;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import org.apache.vxquery.context.DynamicContext;
+import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.datamodel.values.XDMConstants;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.functions.BuiltinFunctions;
+import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
+import org.apache.vxquery.runtime.functions.index.VXQueryIndexReader;
+import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
+import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
+import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
+import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
+
+public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+    protected static final Logger LOGGER = Logger.getLogger(VXQueryCollectionOperatorDescriptor.class.getName());
+    private static final long serialVersionUID = 1L;
+    private short dataSourceId;
+    private short totalDataSources;
+    private String[] collectionPartitions;
+    private String elementPath;
+    private final String functionCall;
+
+    public VXQueryIndexingOperatorDescriptor(IOperatorDescriptorRegistry spec, VXQueryIndexingDataSource ds,
+            RecordDescriptor rDesc, String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos) {
+        super(spec, 1, 1);
+        this.functionCall = ds.getFunctionCall();
+        collectionPartitions = ds.getCollectionPartitions();
+        dataSourceId = (short) ds.getDataSourceId();
+        totalDataSources = (short) ds.getTotalDataSources();
+        recordDescriptors[0] = rDesc;
+        this.elementPath = ds.getElementPath();
+    }
+
+    @Override
+    public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+            IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+        final FrameTupleAccessor fta = new FrameTupleAccessor(
+                recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
+        final int fieldOutputCount = recordDescProvider.getOutputRecordDescriptor(getActivityId(), 0).getFieldCount();
+        final IFrame frame = new VSizeFrame(ctx);
+        final IFrameFieldAppender appender = new FrameFixedFieldTupleAppender(fieldOutputCount);
+        final short partitionId = (short) ctx.getTaskAttemptId().getTaskId().getPartition();
+        final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources);
+        final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
+        final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData();
+        final String collectionName = collectionPartitions[partition % collectionPartitions.length];
+        String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
+        IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
+                ctx.getIOManager().getIODevices().get(0).getPath());
+        indexCentralizerUtil.readIndexDirectory();
+
+        return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
+            @Override
+            public void open() throws HyracksDataException {
+                appender.reset(frame, true);
+                writer.open();
+            }
+
+            @Override
+            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+                fta.reset(buffer);
+
+                IPointable result = new TaggedValuePointable();
+
+                final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+                final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY
+                        .createPointable();
+
+                final ByteBufferInputStream bbis = new ByteBufferInputStream();
+                final DataInputStream di = new DataInputStream(bbis);
+                final SequenceBuilder sb = new SequenceBuilder();
+                final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+                final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
+
+                String indexModifiedName;
+                if (collectionModifiedName.contains("hdfs://")) {
+                    throw new HyracksDataException("Indexing support for HDFS not yet implemented.");
+                } else {
+
+                    if (functionCall.equals(
+                            BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier().getName())) {
+                        indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName);
+                        File collectionDirectory = new File(collectionModifiedName);
+
+                        //check if directory is in the local file system
+                        if (collectionDirectory.exists() && collectionDirectory.isDirectory()) {
+                            IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
+                            try {
+                                indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result,
+                                        stringp, bbis, di, sb, abvs, nodeIdProvider, abvsFileNode, nodep, false,
+                                        nodeId);
+                                XDMConstants.setTrue(result);
+                                FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
+                                        result.getStartOffset(), result.getLength());
+                            } catch (SystemException e) {
+                                throw new HyracksDataException("Could not create index for collection: "
+                                        + collectionName + " in dir: " + indexModifiedName + " " + e.getMessage());
+                            }
+                        } else {
+                            throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":"
+                                    + collectionDirectory.getAbsolutePath() + ")");
+                        }
+                    } else if (functionCall
+                            .equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) {
+                        indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+                        IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
+                                nodeIdProvider, abvsFileNode, nodep, nodeId);
+                        try {
+                            updater.setup();
+                            updater.updateIndex();
+                            updater.updateMetadataFile();
+                            updater.exit();
+                            XDMConstants.setTrue(result);
+                            FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
+                                    result.getStartOffset(), result.getLength());
+                        } catch (IOException | SystemException e) {
+                            throw new HyracksDataException(
+                                    "Could not update index in " + indexModifiedName + " " + e.getMessage());
+                        }
+                    } else if (functionCall
+                            .equals(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier().getName())) {
+                        indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+                        IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
+                                nodeIdProvider, abvsFileNode, nodep, nodeId);
+                        indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName);
+                        try {
+                            updater.setup();
+                            updater.deleteAllIndexes();
+                            XDMConstants.setTrue(result);
+                            FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
+                                    result.getStartOffset(), result.getLength());
+                        } catch (SystemException | IOException e) {
+                            throw new HyracksDataException(
+                                    "Could not delete index in " + indexModifiedName + " " + e.getMessage());
+                        }
+
+                    } else if (functionCall
+                            .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier().getName())) {
+                        indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+                        VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, elementPath);
+                        try {
+                            indexReader.init();
+                            while (indexReader.step(result)) {
+                                FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
+                                        result.getStartOffset(), result.getLength());
+                            }
+                        } catch (AlgebricksException e) {
+                            throw new HyracksDataException("Could not read index.");
+                        }
+
+                    } else {
+                        throw new HyracksDataException("Unsupported function call (" + functionCall + ")");
+                    }
+                }
+            }
+
+            @Override
+            public void fail() throws HyracksDataException {
+                writer.fail();
+            }
+
+            @Override
+            public void close() throws HyracksDataException {
+                // Check if needed?
+                if (appender.getTupleCount() > 0) {
+                    appender.flush(writer, true);
+                }
+                writer.close();
+                indexCentralizerUtil.writeIndexDirectory();
+            }
+        };
+    }
+}
index 820c365..b7b37b9 100644 (file)
@@ -88,22 +88,44 @@ public class VXQueryMetadataProvider implements IMetadataProvider<String, String
             List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
             IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
                     throws AlgebricksException {
-        VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) dataSource;
+        VXQueryCollectionDataSource ds = null;
+        VXQueryIndexingDataSource ids = null;
+
+        try {
+            ids = (VXQueryIndexingDataSource) dataSource;
+        } catch (ClassCastException e) {
+            ds = (VXQueryCollectionDataSource) dataSource;
+        }
         if (sourceFileMap != null) {
-            final int len = ds.getPartitions().length;
+            final int len = ds != null ? ds.getPartitions().length : ids.getCollectionPartitions().length;
             String[] collectionPartitions = new String[len];
             for (int i = 0; i < len; ++i) {
-                String partition = ds.getPartitions()[i];
+                String partition = ds != null ? ds.getPartitions()[i] : ids.getCollectionPartitions()[i];
                 File mapped = sourceFileMap.get(partition);
                 collectionPartitions[i] = mapped != null ? mapped.toString() : partition;
             }
-            ds.setPartitions(collectionPartitions);
+            if (ds != null) {
+                ds.setPartitions(collectionPartitions);
+            } else {
+                ids.setCollectionPartitions(collectionPartitions);
+            }
+        }
+        RecordDescriptor rDesc;
+        IOperatorDescriptor scanner;
+        AlgebricksPartitionConstraint constraint;
+
+        if (ds != null) {
+            rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
+            scanner = new VXQueryCollectionOperatorDescriptor(jobSpec, ds, rDesc, this.hdfsConf,
+                    this.nodeControllerInfos);
+            constraint = getClusterLocations(nodeList, ds.getPartitionCount());
+        } else {
+            rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
+            scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, ids, rDesc, this.hdfsConf,
+                    this.nodeControllerInfos);
+            constraint = getClusterLocations(nodeList, ids.getPartitionCount());
         }
-        RecordDescriptor rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
-        IOperatorDescriptor scanner = new VXQueryCollectionOperatorDescriptor(jobSpec, ds, rDesc, this.hdfsConf,
-                this.nodeControllerInfos);
 
-        AlgebricksPartitionConstraint constraint = getClusterLocations(nodeList, ds.getPartitionCount());
         return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint);
     }
 
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CollectionFromIndexUnnestingEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CollectionFromIndexUnnestingEvaluatorFactory.java
deleted file mode 100644 (file)
index ea78de5..0000000
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluator;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.values.ValueTag;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.index.IndexAttributes;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentUnnestingEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentUnnestingEvaluatorFactory;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.SAXContentHandler;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class CollectionFromIndexUnnestingEvaluatorFactory extends AbstractTaggedValueArgumentUnnestingEvaluatorFactory {
-    private static final long serialVersionUID = 1L;
-
-    public CollectionFromIndexUnnestingEvaluatorFactory(IScalarEvaluatorFactory[] args) {
-        super(args);
-    }
-
-    @Override
-    protected IUnnestingEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
-            throws AlgebricksException {
-
-        return new AbstractTaggedValueArgumentUnnestingEvaluator(args) {
-
-            private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage();
-
-            private int indexPlace;
-            private int indexLength;
-            private String elementPath;
-            private String indexName;
-
-            private UTF8StringPointable stringIndexFolder = (UTF8StringPointable) UTF8StringPointable.FACTORY
-                    .createPointable();
-            private UTF8StringPointable stringElementPath = (UTF8StringPointable) UTF8StringPointable.FACTORY
-                    .createPointable();
-            private ByteBufferInputStream bbis = new ByteBufferInputStream();
-            private DataInputStream di = new DataInputStream(bbis);
-
-            private IndexReader reader;
-            private IndexSearcher searcher;
-            private Analyzer analyzer;
-            private QueryParser parser;
-            private ScoreDoc[] hits;
-            private SAXContentHandler handler;
-            private Query query;
-            private Document doc;
-            private List<IndexableField> fields;
-
-            @Override
-            public boolean step(IPointable result) throws AlgebricksException {
-                /* each step will create a tuple for a single xml file
-                 * This is done using the parse function
-                 * checkoverflow is used throughout. This is because memory might not be
-                 * able to hold all of the results at once, so we return 1 million at
-                 * a time and check when we need to get more
-                 */
-                if (indexPlace < indexLength) {
-                    nodeAbvs.reset();
-                    try {
-                        //TODO: now we get back the entire document
-                        doc = searcher.doc(hits[indexPlace].doc);
-                        fields = doc.getFields();
-                        parse(nodeAbvs);
-                    } catch (IOException e) {
-                        throw new AlgebricksException(e);
-                    }
-                    indexPlace += 1;
-                    result.set(nodeAbvs.getByteArray(), nodeAbvs.getStartOffset(), nodeAbvs.getLength());
-                    return true;
-                }
-                return false;
-            }
-
-            @Override
-            protected void init(TaggedValuePointable[] args) throws SystemException {
-
-                int partition = ctxview.getTaskAttemptId().getTaskId().getPartition();
-                ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-                handler = new SAXContentHandler(false, nodeIdProvider, true);
-
-                nodeAbvs.reset();
-                indexPlace = 0;
-                TaggedValuePointable tvp1 = args[0];
-                TaggedValuePointable tvp2 = args[1];
-
-                if (tvp1.getTag() != ValueTag.XS_STRING_TAG || tvp2.getTag() != ValueTag.XS_STRING_TAG) {
-                    throw new SystemException(ErrorCode.FORG0006);
-                }
-                tvp1.getValue(stringIndexFolder);
-                tvp2.getValue(stringElementPath);
-                //This whole loop is to get the string arguments, indexFolder, elementPath, and match option
-                try {
-                    // Get the list of files.
-                    bbis.setByteBuffer(ByteBuffer.wrap(
-                            Arrays.copyOfRange(stringIndexFolder.getByteArray(), stringIndexFolder.getStartOffset(),
-                                    stringIndexFolder.getLength() + stringIndexFolder.getStartOffset())),
-                            0);
-                    indexName = di.readUTF();
-                    bbis.setByteBuffer(ByteBuffer.wrap(
-                            Arrays.copyOfRange(stringElementPath.getByteArray(), stringElementPath.getStartOffset(),
-                                    stringElementPath.getLength() + stringElementPath.getStartOffset())),
-                            0);
-                    elementPath = di.readUTF();
-
-                    indexPlace = 0;
-
-                    //Create the index reader.
-                    reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexName)));
-                } catch (IOException e) {
-                    throw new SystemException(ErrorCode.SYSE0001, e);
-                }
-
-                searcher = new IndexSearcher(reader);
-                analyzer = new CaseSensitiveAnalyzer();
-
-                parser = new CaseSensitiveQueryParser("item", analyzer);
-
-                String queryString = elementPath.replaceAll("/", ".");
-                queryString = "item:" + queryString + "*";
-
-                int lastslash = elementPath.lastIndexOf("/");
-                elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1);
-                elementPath = elementPath.replaceAll("/", ".") + ".element";
-
-                TopDocs results = null;
-                try {
-                    query = parser.parse(queryString);
-
-                    //TODO: Right now it only returns 1000000 results
-                    results = searcher.search(query, 1000000);
-
-                } catch (Exception e) {
-                    throw new SystemException(null);
-                }
-
-                hits = results.scoreDocs;
-                System.out.println("found: " + results.totalHits);
-                indexPlace = 0;
-                indexLength = hits.length;
-
-            }
-
-            public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException {
-                try {
-                    handler.startDocument();
-
-                    for (int i = 0; i < fields.size(); i++) {
-                        String fieldValue = fields.get(i).stringValue();
-                        if (fieldValue.equals(elementPath)) {
-                            buildElement(abvsFileNode, i);
-                        }
-                    }
-
-                    handler.endDocument();
-                    handler.writeDocument(abvsFileNode);
-                } catch (Exception e) {
-                    throw new IOException(e);
-                }
-            }
-
-            private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException {
-                int whereIFinish = fieldNum;
-                IndexableField field = fields.get(fieldNum);
-                String contents = field.stringValue();
-                String uri = "";
-
-                int firstColon = contents.indexOf(':');
-                int lastDot = contents.lastIndexOf('.');
-                String type = contents.substring(lastDot + 1);
-                String lastBit = contents.substring(firstColon + 1, lastDot);
-
-                if (type.equals("textnode")) {
-                    char[] charContents = lastBit.toCharArray();
-                    handler.characters(charContents, 0, charContents.length);
-
-                }
-                if (type.equals("element")) {
-                    List<String> names = new ArrayList<String>();
-                    List<String> values = new ArrayList<String>();
-                    List<String> uris = new ArrayList<String>();
-                    List<String> localNames = new ArrayList<String>();
-                    List<String> types = new ArrayList<String>();
-                    List<String> qNames = new ArrayList<String>();
-                    whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames);
-                    Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
-
-                    handler.startElement(uri, lastBit, lastBit, atts);
-
-                    boolean noMoreChildren = false;
-
-                    while (whereIFinish + 1 < fields.size() && !noMoreChildren) {
-                        if (isChild(fields.get(whereIFinish + 1), field)) {
-                            whereIFinish = buildElement(abvsFileNode, whereIFinish + 1);
-                        } else {
-                            noMoreChildren = true;
-                        }
-                    }
-
-                    handler.endElement(uri, lastBit, lastBit);
-
-                }
-                return whereIFinish;
-            }
-
-            /*This function creates the attribute children for an element node
-             * 
-             */
-            int findAttributeChildren(int fieldnum, List<String> n, List<String> v, List<String> u, List<String> l,
-                    List<String> t, List<String> q) {
-                int nextindex = fieldnum + 1;
-                boolean foundattributes = false;
-                if (nextindex < fields.size()) {
-                    IndexableField nextguy;
-
-                    while (nextindex < fields.size()) {
-                        nextguy = fields.get(nextindex);
-                        String contents = nextguy.stringValue();
-                        int firstcolon = contents.indexOf(':');
-                        int lastdot = contents.lastIndexOf('.');
-                        String lastbit = contents.substring(firstcolon + 1, lastdot);
-
-                        if (isDirectChildAttribute(nextguy, fields.get(fieldnum))) {
-                            foundattributes = true;
-                            n.add(lastbit);
-                            IndexableField nextnextguy = fields.get(nextindex + 1);
-                            contents = nextnextguy.stringValue();
-                            firstcolon = contents.indexOf(':');
-                            lastdot = contents.lastIndexOf('.');
-                            String nextlastbit = contents.substring(firstcolon + 1, lastdot);
-                            v.add(nextlastbit);
-                            u.add(lastbit);
-                            l.add(lastbit);
-                            t.add(lastbit);
-                            q.add(lastbit);
-                        } else {
-                            break;
-                        }
-                        nextindex += 2;
-                    }
-                }
-                if (foundattributes) {
-                    return nextindex - 1;
-
-                } else {
-                    return fieldnum;
-                }
-            }
-
-            boolean isChild(IndexableField child, IndexableField adult) {
-                String childId = child.stringValue();
-                String adultId = adult.stringValue();
-
-                int lastDotChild = childId.lastIndexOf('.');
-                int lastDotAdult = adultId.lastIndexOf('.');
-
-                String childPath = childId.substring(0, lastDotChild);
-                String adultPath = adultId.substring(0, lastDotAdult);
-                adultPath = adultPath.replaceFirst(":", ".");
-
-                return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."));
-            }
-
-            boolean isDirectChildAttribute(IndexableField child, IndexableField adult) {
-                String childId = child.stringValue();
-                String adultId = adult.stringValue();
-
-                String childPath = childId.substring(0, childId.lastIndexOf('.'));
-                String adultPath = adultId.substring(0, adultId.lastIndexOf('.'));
-                adultPath = adultPath.replaceFirst(":", ".");
-                String[] childSegments = child.stringValue().split("\\.");
-
-                String childType = childSegments[childSegments.length - 1];
-
-                return (childPath.startsWith(adultPath + ":") && childType.equals("attribute"));
-            }
-
-        };
-    }
-}
\ No newline at end of file
index 126ddf5..2f22ade 100644 (file)
@@ -28,7 +28,6 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
 import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.ValueTag;
 import org.apache.vxquery.exceptions.ErrorCode;
 import org.apache.vxquery.exceptions.SystemException;
 import org.apache.vxquery.index.IndexDocumentBuilder;
@@ -39,58 +38,31 @@ import org.apache.vxquery.xmlparser.IParser;
 import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
 import org.apache.vxquery.xmlparser.XMLParser;
 
-import javax.xml.bind.JAXBException;
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.nio.file.Paths;
-import java.security.NoSuchAlgorithmException;
 import java.text.SimpleDateFormat;
-import java.util.Arrays;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class IndexConstructorUtil {
-    static boolean isMetaFilePresent = false;
-    static MetaFileUtil metaFileUtil;
-    static ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
+     boolean isMetaFilePresent = false;
+     MetaFileUtil metaFileUtil;
+     ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
 
-    public static void evaluate(TaggedValuePointable[] args, IPointable result, UTF8StringPointable stringp,
-            ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
+    public void evaluate(String collectioFolder, String indexFolder, IPointable result, UTF8StringPointable
+            stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
             ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
-            boolean isElementPath, String nodeId) throws SystemException, JAXBException {
-        String collectionFolder;
-        String indexFolder;
-        TaggedValuePointable collectionTVP = args[0];
-        TaggedValuePointable indexTVP = args[1];
-
-        if (collectionTVP.getTag() != ValueTag.XS_STRING_TAG || indexTVP.getTag() != ValueTag.XS_STRING_TAG) {
-            throw new SystemException(ErrorCode.FORG0006);
-        }
+            boolean isElementPath, String nodeId) throws SystemException {
 
-        try {
-            // Get the list of files.
-            collectionTVP.getValue(stringp);
-            bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
-                    stringp.getLength() + stringp.getStartOffset())), 0);
-            collectionFolder = di.readUTF();
-
-            // Get the index folder
-            indexTVP.getValue(stringp);
-            bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
-                    stringp.getLength() + stringp.getStartOffset())), 0);
-            indexFolder = di.readUTF();
-
-            metaFileUtil = MetaFileUtil.create(indexFolder);
+            metaFileUtil = new MetaFileUtil(indexFolder);
+//            metaFileUtil = .create(indexFolder);
             isMetaFilePresent = metaFileUtil.isMetaFilePresent();
-            metaFileUtil.setCollectionForIndex(indexFolder, collectionFolder);
+            metaFileUtil.setCollection(collectioFolder);
 
-        } catch (IOException e) {
-            throw new SystemException(ErrorCode.SYSE0001, e);
-        }
-        File collectionDirectory = new File(collectionFolder);
+        File collectionDirectory = new File(collectioFolder);
         if (!collectionDirectory.exists()) {
-            throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
+            throw new RuntimeException("The collection directory (" + collectioFolder + ") does not exist.");
         }
 
         try {
@@ -132,7 +104,7 @@ public class IndexConstructorUtil {
     /*This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
      * it indexes that document node.
      */
-    public static void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
+    public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
             TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider,
             SequenceBuilder sb, ByteBufferInputStream bbis, DataInputStream di, String nodeId)
             throws SystemException, IOException {
@@ -153,11 +125,7 @@ public class IndexConstructorUtil {
                     xmlMetadata.setPath(file.getCanonicalPath());
                     xmlMetadata.setFileName(file.getName());
                     xmlMetadata.setLastModified(sdf.format(file.lastModified()));
-                    try {
-                        xmlMetadata.setMd5(metaFileUtil.generateMD5(file));
-                    } catch (NoSuchAlgorithmException e) {
-                        throw new SystemException(ErrorCode.SYSE0001, e);
-                    }
+                    xmlMetadata.setMd5(metaFileUtil.generateMD5(file));
                     metadataMap.put(file.getCanonicalPath(), xmlMetadata);
                 }
 
@@ -168,14 +136,14 @@ public class IndexConstructorUtil {
         }
     }
 
-    public static boolean readableXmlFile(String path) {
+    public boolean readableXmlFile(String path) {
         return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz"));
     }
 
     /**
      * Separated from create index method so that it could be used as a helper function in IndexUpdater
      */
-    public static IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
+    public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
             ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis,
             DataInputStream di, String nodeId) throws IOException {
 
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java
deleted file mode 100644 (file)
index e713b20..0000000
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.XDMConstants;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-
-import javax.xml.bind.JAXBException;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * Delete the index of a given index directory
- */
-public class IndexDeleteEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
-    public IndexDeleteEvaluatorFactory(IScalarEvaluatorFactory[] args) {
-        super(args);
-    }
-
-    @Override
-    protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
-            throws AlgebricksException {
-        final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-        final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
-        final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
-        final ByteBufferInputStream bbis = new ByteBufferInputStream();
-        final DataInputStream di = new DataInputStream(bbis);
-        final SequenceBuilder sb = new SequenceBuilder();
-        final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
-        final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
-        final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
-        final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-
-        return new AbstractTaggedValueArgumentScalarEvaluator(args) {
-
-            @Override
-            protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
-                IndexUpdater updater = new IndexUpdater(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider,
-                        abvsFileNode, nodep, nodeId);
-                try {
-                    updater.setup();
-                    updater.deleteAllIndexes();
-                    XDMConstants.setTrue(result);
-                } catch (IOException | NoSuchAlgorithmException | JAXBException e) {
-                    throw new SystemException(ErrorCode.SYSE0001, e);
-                }
-
-            }
-
-        };
-    }
-}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java
deleted file mode 100644 (file)
index 3fbc279..0000000
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.vxquery.runtime.functions.index;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.XDMConstants;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
-import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-
-import javax.xml.bind.JAXBException;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * Update the index of collection
- */
-public class IndexUpdaterEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
-    public IndexUpdaterEvaluatorFactory(IScalarEvaluatorFactory[] args) {
-        super(args);
-    }
-
-    @Override
-    protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
-            throws AlgebricksException {
-        final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-        final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
-        final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
-        final ByteBufferInputStream bbis = new ByteBufferInputStream();
-        final DataInputStream di = new DataInputStream(bbis);
-        final SequenceBuilder sb = new SequenceBuilder();
-        final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
-        final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
-        final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
-        final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
-
-        return new AbstractTaggedValueArgumentScalarEvaluator(args) {
-
-            @Override
-            protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
-                IndexUpdater updater = new IndexUpdater(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider,
-                        abvsFileNode, nodep, nodeId);
-                try {
-                    updater.setup();
-                    updater.updateIndex();
-                    updater.updateMetadataFile();
-                    updater.exit();
-                    XDMConstants.setTrue(result);
-                } catch (IOException | NoSuchAlgorithmException | JAXBException e) {
-                    throw new SystemException(ErrorCode.SYSE0001, e);
-                }
-            }
-
-        };
-    }
-}
 */
 package org.apache.vxquery.runtime.functions.index;
 
+import java.io.IOException;
+
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
 import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
 import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
 import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
 import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
 import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.XDMConstants;
 import org.apache.vxquery.exceptions.ErrorCode;
 import org.apache.vxquery.exceptions.SystemException;
 import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
 import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
-
-import javax.xml.bind.JAXBException;
-import java.io.DataInputStream;
-
-public class IndexConstructorScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
-    //Creates one Lucene doc per file
-
-    private static final long serialVersionUID = 1L;
+import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
 
-    public IndexConstructorScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+public class ShowIndexScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
+    public ShowIndexScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
         super(args);
     }
 
@@ -50,30 +41,25 @@ public class IndexConstructorScalarEvaluatorFactory extends AbstractTaggedValueA
     protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
             throws AlgebricksException {
         final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-        final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
-        final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
-        final ByteBufferInputStream bbis = new ByteBufferInputStream();
-        final DataInputStream di = new DataInputStream(bbis);
         final SequenceBuilder sb = new SequenceBuilder();
-        final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
-        final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
-        final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
-        final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
 
         return new AbstractTaggedValueArgumentScalarEvaluator(args) {
-
             @Override
             protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
                 try {
-                    IndexConstructorUtil
-                            .evaluate(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider, abvsFileNode,
-                            nodep, false, nodeId);
-                    XDMConstants.setTrue(result);
-                } catch (JAXBException e) {
+                    abvs.reset();
+                    sb.reset(abvs);
+                    IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
+                            ctx.getIOManager().getIODevices().get(0).getPath());
+                    indexCentralizerUtil.readIndexDirectory();
+                    indexCentralizerUtil.getAllCollections(sb);
+                    sb.finish();
+                    result.set(abvs);
+                } catch (IOException e) {
                     throw new SystemException(ErrorCode.SYSE0001, e);
                 }
-            }
 
+            }
         };
     }
 }
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
new file mode 100644 (file)
index 0000000..8750849
--- /dev/null
@@ -0,0 +1,284 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.vxquery.runtime.functions.index;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.index.IndexAttributes;
+import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
+import org.apache.vxquery.xmlparser.SAXContentHandler;
+import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+public class VXQueryIndexReader {
+
+    private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage();
+
+    private int indexPlace;
+    private int indexLength;
+    private String elementPath;
+    private String indexName;
+
+    private ByteBufferInputStream bbis = new ByteBufferInputStream();
+    private DataInputStream di = new DataInputStream(bbis);
+
+    private IndexReader reader;
+    private IndexSearcher searcher;
+    private QueryParser parser;
+    private ScoreDoc[] hits;
+    private SAXContentHandler handler;
+    private Query query;
+    private Document doc;
+    private List<IndexableField> fields;
+    private IHyracksTaskContext ctx;
+
+    public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, String elementPath) {
+        this.ctx = context;
+        this.indexName = indexPath;
+        this.elementPath = elementPath;
+    }
+
+    public boolean step(IPointable result) throws AlgebricksException {
+        /*each step will create a tuple for a single xml file
+        * This is done using the parse function
+        * checkoverflow is used throughout. This is because memory might not be
+        * able to hold all of the results at once, so we return 1 million at
+        * a time and check when we need to get more
+        */
+        if (indexPlace < indexLength) {
+            nodeAbvs.reset();
+            try {
+                //TODO: now we get back the entire document
+                doc = searcher.doc(hits[indexPlace].doc);
+                fields = doc.getFields();
+                parse(nodeAbvs);
+            } catch (IOException e) {
+                throw new AlgebricksException(e);
+            }
+            indexPlace += 1;
+            result.set(nodeAbvs.getByteArray(), nodeAbvs.getStartOffset(), nodeAbvs.getLength());
+            return true;
+        }
+        return false;
+    }
+
+    public void init() throws SystemException {
+
+        int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
+        ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
+        handler = new SAXContentHandler(false, nodeIdProvider, true);
+
+        nodeAbvs.reset();
+        indexPlace = 0;
+
+        try {
+            indexPlace = 0;
+
+            //Create the index reader.
+            reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexName)));
+        } catch (IOException e) {
+            throw new SystemException(ErrorCode.SYSE0001, e);
+        }
+
+        searcher = new IndexSearcher(reader);
+        Analyzer analyzer = new CaseSensitiveAnalyzer();
+
+        parser = new CaseSensitiveQueryParser("item", analyzer);
+
+        String queryString = elementPath.replaceAll("/", ".");
+        queryString = "item:" + queryString + "*";
+
+        int lastslash = elementPath.lastIndexOf("/");
+        elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1);
+        elementPath = elementPath.replaceAll("/", ".") + ".element";
+
+        TopDocs results = null;
+        try {
+            query = parser.parse(queryString);
+
+            //TODO: Right now it only returns 1000000 results
+            results = searcher.search(query, 1000000);
+
+        } catch (Exception e) {
+            throw new SystemException(null);
+        }
+
+        hits = results.scoreDocs;
+        System.out.println("found: " + results.totalHits);
+        indexPlace = 0;
+        indexLength = hits.length;
+
+    }
+
+    public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException {
+        try {
+            handler.startDocument();
+
+            for (int i = 0; i < fields.size(); i++) {
+                String fieldValue = fields.get(i).stringValue();
+                if (fieldValue.equals(elementPath)) {
+                    buildElement(abvsFileNode, i);
+                }
+            }
+
+            handler.endDocument();
+            handler.writeDocument(abvsFileNode);
+        } catch (Exception e) {
+            throw new IOException(e);
+        }
+    }
+
+    private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException {
+        int whereIFinish = fieldNum;
+        IndexableField field = fields.get(fieldNum);
+        String contents = field.stringValue();
+        String uri = "";
+
+        int firstColon = contents.indexOf(':');
+        int lastDot = contents.lastIndexOf('.');
+        String type = contents.substring(lastDot + 1);
+        String lastBit = contents.substring(firstColon + 1, lastDot);
+
+        if (type.equals("textnode")) {
+            char[] charContents = lastBit.toCharArray();
+            handler.characters(charContents, 0, charContents.length);
+
+        }
+        if (type.equals("element")) {
+            List<String> names = new ArrayList<String>();
+            List<String> values = new ArrayList<String>();
+            List<String> uris = new ArrayList<String>();
+            List<String> localNames = new ArrayList<String>();
+            List<String> types = new ArrayList<String>();
+            List<String> qNames = new ArrayList<String>();
+            whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames);
+            Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
+
+            handler.startElement(uri, lastBit, lastBit, atts);
+
+            boolean noMoreChildren = false;
+
+            while (whereIFinish + 1 < fields.size() && !noMoreChildren) {
+                if (isChild(fields.get(whereIFinish + 1), field)) {
+                    whereIFinish = buildElement(abvsFileNode, whereIFinish + 1);
+                } else {
+                    noMoreChildren = true;
+                }
+            }
+
+            handler.endElement(uri, lastBit, lastBit);
+
+        }
+        return whereIFinish;
+    }
+
+    /*This function creates the attribute children for an element node
+     *
+     */
+    int findAttributeChildren(int fieldnum, List<String> n, List<String> v, List<String> u, List<String> l,
+            List<String> t, List<String> q) {
+        int nextindex = fieldnum + 1;
+        boolean foundattributes = false;
+        if (nextindex < fields.size()) {
+            IndexableField nextguy;
+
+            while (nextindex < fields.size()) {
+                nextguy = fields.get(nextindex);
+                String contents = nextguy.stringValue();
+                int firstcolon = contents.indexOf(':');
+                int lastdot = contents.lastIndexOf('.');
+                String lastbit = contents.substring(firstcolon + 1, lastdot);
+
+                if (isDirectChildAttribute(nextguy, fields.get(fieldnum))) {
+                    foundattributes = true;
+                    n.add(lastbit);
+                    IndexableField nextnextguy = fields.get(nextindex + 1);
+                    contents = nextnextguy.stringValue();
+                    firstcolon = contents.indexOf(':');
+                    lastdot = contents.lastIndexOf('.');
+                    String nextlastbit = contents.substring(firstcolon + 1, lastdot);
+                    v.add(nextlastbit);
+                    u.add(lastbit);
+                    l.add(lastbit);
+                    t.add(lastbit);
+                    q.add(lastbit);
+                } else {
+                    break;
+                }
+                nextindex += 2;
+            }
+        }
+        if (foundattributes) {
+            return nextindex - 1;
+
+        } else {
+            return fieldnum;
+        }
+    }
+
+    boolean isChild(IndexableField child, IndexableField adult) {
+        String childId = child.stringValue();
+        String adultId = adult.stringValue();
+
+        int lastDotChild = childId.lastIndexOf('.');
+        int lastDotAdult = adultId.lastIndexOf('.');
+
+        String childPath = childId.substring(0, lastDotChild);
+        String adultPath = adultId.substring(0, lastDotAdult);
+        adultPath = adultPath.replaceFirst(":", ".");
+
+        return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."));
+    }
+
+    boolean isDirectChildAttribute(IndexableField child, IndexableField adult) {
+        String childId = child.stringValue();
+        String adultId = adult.stringValue();
+
+        String childPath = childId.substring(0, childId.lastIndexOf('.'));
+        String adultPath = adultId.substring(0, adultId.lastIndexOf('.'));
+        adultPath = adultPath.replaceFirst(":", ".");
+        String[] childSegments = child.stringValue().split("\\.");
+
+        String childType = childSegments[childSegments.length - 1];
+
+        return (childPath.startsWith(adultPath + ":") && childType.equals("attribute"));
+    }
+
+}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
new file mode 100644 (file)
index 0000000..38dcc2a
--- /dev/null
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.datamodel.values.ValueTag;
+
+/**
+ * Class for maintaining the centralized index information file.
+ * Index centralization procedure.
+ * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = <index_directory>.
+ * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections.
+ * There will be a single xml file, located in the directory specified in local.xml, which contains all information
+ * about the existing indexes.
+ * This class can be used to read, add, delete, modify the entries and write the file back to the disk.
+ */
+public class IndexCentralizerUtil {
+
+    private final String FILE_NAME = "VXQuery-Index-Directory.xml";
+    private final List<String> collections = new ArrayList<>();
+    private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
+    private File XML_FILE;
+    private String INDEX_LOCATION;
+    private static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
+
+    public IndexCentralizerUtil(File index) {
+        this.INDEX_LOCATION = index.getPath();
+        if (!index.exists()) {
+            try {
+                FileUtils.forceMkdir(index);
+            } catch (IOException e) {
+                LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e);
+            }
+        }
+        XML_FILE = new File(index.getPath() + "/" + FILE_NAME);
+    }
+
+    /**
+     * Get the index directory containing index of the given collection
+     *
+     * @param collection : Collection folder
+     * @return Index folder.
+     */
+    public String getIndexForCollection(String collection) {
+        return indexCollectionMap.get(collection).getIndex();
+    }
+
+    /**
+     * Put the index location corresponding to given collection.
+     * Index location is created by using the last 100 characters of collection.
+     *
+     * @param collection : Collection directory
+     */
+    public String putIndexForCollection(String collection) {
+        int length = collection.replaceAll("/", "").length();
+        String index = collection.replaceAll("/", "");
+        index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index);
+        IndexLocator il = new IndexLocator();
+        il.setCollection(collection);
+        il.setIndex(index);
+        if (indexCollectionMap.get(collection) != null) {
+            return index;
+        }
+        indexCollectionMap.put(collection, il);
+        return index;
+    }
+
+    /**
+     * Remove the entry for given collection directory.
+     *
+     * @param collection : Collection directory
+     */
+    public void deleteEntryForCollection(String collection) {
+        indexCollectionMap.remove(collection);
+    }
+
+    /**
+     * Prints all collections which have an index created.
+     *
+     * @throws IOException
+     */
+    public void getAllCollections(SequenceBuilder sb) throws IOException {
+        for (String s : collections) {
+            StringValueBuilder svb = new StringValueBuilder();
+            ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+            DataOutput output = abvs.getDataOutput();
+            output.write(ValueTag.XS_STRING_TAG);
+            svb.write(s, output);
+            sb.addItem(abvs);
+        }
+    }
+
+    /**
+     * Read the collection, index directory file and populate the HashMap.
+     */
+    public void readIndexDirectory() {
+        if (this.XML_FILE.exists()) {
+            try {
+                JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class);
+                Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+                IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE);
+
+                for (IndexLocator il : indexDirectory.getDirectory()) {
+                    indexCollectionMap.put(il.getCollection(), il);
+                    this.collections.add(il.getCollection());
+                }
+            } catch (JAXBException e) {
+                LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
+            }
+        }
+
+    }
+
+    /**
+     * Write back the contents of the HashMap to the file.
+     */
+    public void writeIndexDirectory() {
+        IndexDirectory id = new IndexDirectory();
+        List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values());
+        id.setDirectory(indexLocators);
+        try {
+            FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE);
+            JAXBContext context = JAXBContext.newInstance(IndexDirectory.class);
+            Marshaller jaxbMarshaller = context.createMarshaller();
+            jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+            jaxbMarshaller.marshal(id, fileOutputStream);
+        } catch (JAXBException | FileNotFoundException e) {
+            LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e);
+        }
+    }
+}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
new file mode 100644 (file)
index 0000000..54d9ad9
--- /dev/null
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+@XmlRootElement(name = "indexes")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class IndexDirectory implements Serializable{
+
+    @XmlElement(name = "index", type = IndexLocator.class)
+    private List<IndexLocator> directory = new ArrayList<>();
+
+    public List<IndexLocator> getDirectory() {
+        return directory;
+    }
+
+
+    public void setDirectory(List<IndexLocator> directory) {
+        this.directory = directory;
+    }
+}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
new file mode 100644 (file)
index 0000000..1a33c8b
--- /dev/null
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.Serializable;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "Entry")
+public class IndexLocator implements Serializable{
+
+    @XmlAttribute
+    private String collection;
+
+    @XmlAttribute
+    private String index;
+
+    public String getCollection() {
+        return collection;
+    }
+
+    public void setCollection(String collection) {
+        this.collection = collection;
+    }
+
+    public String getIndex() {
+        return index;
+    }
+
+    public void setIndex(String index) {
+        this.index = index;
+    }
+}
index 9aebfb4..2a45747 100644 (file)
@@ -22,5 +22,4 @@ package org.apache.vxquery.runtime.functions.index.updateIndex;
 public class Constants {
     public static String FIELD_PATH = "path";
     public static String META_FILE_NAME = "vxquery_index.xml";
-    public static String COLLECTION_ENTRY = "collection";
 }
index 4588282..ba7cd88 100644 (file)
@@ -29,7 +29,6 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
 import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.datamodel.values.ValueTag;
 import org.apache.vxquery.exceptions.ErrorCode;
 import org.apache.vxquery.exceptions.SystemException;
 import org.apache.vxquery.index.IndexDocumentBuilder;
@@ -37,16 +36,12 @@ import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer;
 import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
 import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
 
-import javax.xml.bind.JAXBException;
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.security.NoSuchAlgorithmException;
 import java.text.SimpleDateFormat;
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -57,9 +52,7 @@ import java.util.concurrent.ConcurrentHashMap;
 public class IndexUpdater {
     private MetaFileUtil metaFileUtil;
     private ConcurrentHashMap<String, XmlMetadata> metadataMap;
-    private TaggedValuePointable[] args;
     private IPointable result;
-    private UTF8StringPointable stringp;
     private ByteBufferInputStream bbis;
     private DataInputStream di;
     private SequenceBuilder sb;
@@ -71,19 +64,17 @@ public class IndexUpdater {
     private IndexWriter indexWriter;
     private Set<String> pathsFromFileList;
     private String collectionFolder;
-    private XmlMetadata collectionMetadata;
     private String indexFolder;
     private Logger LOGGER = Logger.getLogger("Index Updater");
     private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
+    private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
 
-    //TODO : Implement for paralleizing
-    public IndexUpdater(TaggedValuePointable[] args, IPointable result, UTF8StringPointable stringp,
+    public  IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp,
             ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
             ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
             String nodeId) {
-        this.args = args;
+        this.indexFolder = indexFolder;
         this.result = result;
-        this.stringp = stringp;
         this.bbis = bbis;
         this.di = di;
         this.sb = sb;
@@ -100,35 +91,17 @@ public class IndexUpdater {
      *
      * @throws SystemException
      * @throws IOException
-     * @throws NoSuchAlgorithmException
      */
-    public void setup() throws SystemException, IOException, NoSuchAlgorithmException, JAXBException {
+    public void setup() throws SystemException, IOException {
 
-        TaggedValuePointable indexTVP = args[0];
+        // Read the metadata file and load the metadata map into memory.
+        metaFileUtil = new MetaFileUtil(indexFolder);
+        metaFileUtil.readMetadataFile();
+        metadataMap = metaFileUtil.getMetadata();
 
-        if (indexTVP.getTag() != ValueTag.XS_STRING_TAG) {
-            throw new SystemException(ErrorCode.FORG0006);
-        }
-
-        try {
-            // Get the index folder
-            indexTVP.getValue(stringp);
-            bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
-                    stringp.getLength() + stringp.getStartOffset())), 0);
-            indexFolder = di.readUTF();
-
-            // Read the metadata file and load the metadata map into memory.
-            metaFileUtil = MetaFileUtil.create(indexFolder);
-            metaFileUtil.readMetadataFile();
-            metadataMap = metaFileUtil.getMetadata(indexFolder);
-
-            // Retrieve the collection folder path.
-            // Remove the entry for ease of the next steps.
-            collectionFolder = metaFileUtil.getCollection(indexFolder);
-
-        } catch (IOException | ClassNotFoundException e) {
-            throw new SystemException(ErrorCode.SYSE0001, e);
-        }
+        // Retrieve the collection folder path.
+        // Remove the entry for ease of the next steps.
+        collectionFolder = metaFileUtil.getCollection();
 
         abvs.reset();
         sb.reset(abvs);
@@ -142,9 +115,8 @@ public class IndexUpdater {
      * Wrapper for update index function.
      *
      * @throws IOException
-     * @throws NoSuchAlgorithmException
      */
-    public void updateIndex() throws IOException, NoSuchAlgorithmException {
+    public void updateIndex() throws IOException {
         File collectionDirectory = new File(collectionFolder);
         if (!collectionDirectory.exists()) {
             throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
@@ -155,6 +127,7 @@ public class IndexUpdater {
 
         //Detect deleted files and execute the delete index process.
         deleteIndexOfDeletedFiles(metadataMap.keySet(), pathsFromFileList);
+        updateMetadataFile();
     }
 
     /**
@@ -176,7 +149,7 @@ public class IndexUpdater {
      *
      * @throws IOException
      */
-    public synchronized void updateMetadataFile() throws IOException, JAXBException {
+    public synchronized void updateMetadataFile() throws IOException {
         //Write the updated metadata to the file.
         metaFileUtil.updateMetadataMap(metadataMap, indexFolder);
         metaFileUtil.writeMetadataToFile();
@@ -188,14 +161,14 @@ public class IndexUpdater {
      *
      * @param collection : Collection folder path
      */
-    private void updateIndex(File collection) throws IOException, NoSuchAlgorithmException {
+    private void updateIndex(File collection) throws IOException {
 
         File[] list = collection.listFiles();
 
         assert list != null;
         for (File file : list) {
             pathsFromFileList.add(file.getCanonicalPath());
-            if (IndexConstructorUtil.readableXmlFile(file.getCanonicalPath())) {
+            if (indexConstructorUtil.readableXmlFile(file.getCanonicalPath())) {
                 XmlMetadata data = metadataMap.get(file.getCanonicalPath());
                 String md5 = metaFileUtil.generateMD5(file);
 
@@ -212,7 +185,7 @@ public class IndexUpdater {
 
                         //Update index corresponding to the xml file.
                         indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath()));
-                        indexDocumentBuilder = IndexConstructorUtil
+                        indexDocumentBuilder = indexConstructorUtil
                                 .getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di,
                                         nodeId);
                         indexDocumentBuilder.printStart();
@@ -230,7 +203,7 @@ public class IndexUpdater {
 
                     // In this case, the xml file has not added to the index. (It is a newly added file)
                     // Therefore generate a new index for this file and add it to the existing index.
-                    indexDocumentBuilder = IndexConstructorUtil
+                    indexDocumentBuilder = indexConstructorUtil
                             .getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di, nodeId);
                     indexDocumentBuilder.printStart();
 
@@ -254,15 +227,14 @@ public class IndexUpdater {
      * @param metadata : Existing metadata object
      * @return : XML metadata object with updated fields.
      * @throws IOException
-     * @throws NoSuchAlgorithmException
      */
-    private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException, NoSuchAlgorithmException {
+    private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException {
 
-        if (metadata == null)
+        if (metadata == null) {
             metadata = new XmlMetadata();
-
-        metadata.setPath(file.getCanonicalPath());
+        }
         metadata.setFileName(file.getName());
+        metadata.setPath(file.getCanonicalPath());
         metadata.setMd5(metaFileUtil.generateMD5(file));
         metadata.setLastModified(sdf.format(file.lastModified()));
         return metadata;
index 53b02df..dd099b5 100644 (file)
@@ -19,7 +19,11 @@ package org.apache.vxquery.runtime.functions.index.updateIndex;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 
-import javax.xml.bind.*;
+import javax.xml.bind.DatatypeConverter;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
@@ -40,17 +44,14 @@ public class MetaFileUtil {
 
     private File metaFile;
     private Logger LOGGER = Logger.getLogger("MetadataFileUtil");
-    private Map<String, ConcurrentHashMap<String, XmlMetadata>> indexes = new ConcurrentHashMap<>();
-    private Map<String, String> indexToCollection = new ConcurrentHashMap<>();
+    private String index;
+    private String collection;
+    private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>();
 
-    private MetaFileUtil(String indexFolder) {
+    public MetaFileUtil(String indexFolder) {
         this.metaFile = new File(indexFolder + "/" + Constants.META_FILE_NAME);
     }
 
-    public static MetaFileUtil create(String indexFolder) {
-        return new MetaFileUtil(indexFolder);
-    }
-
     /**
      * Checks for existing metadata file.
      *
@@ -66,88 +67,74 @@ public class MetaFileUtil {
      * Otherwise insert new.
      * @param metadataMap : Set of XmlMetaData objects.
      * @param index : The path to index location.
-     * @throws IOException
      */
-    public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) throws
-            IOException, JAXBException {
-
-        if (this.indexes.get(index) == null) {
-            this.indexes.put(index, metadataMap);
-        } else {
-            this.indexes.replace(index, metadataMap);
-        }
+    public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) {
+        this.indexMap = metadataMap;
+        this.index = index;
 
     }
 
     /**
      * Method to get the set of xml metadata for a given collection
      *
-     * @param index : The collection from which the metadata should be read.
-     * @return : Map containing the set of XmlMetadata objects.
-     * @throws IOException
-     * @throws ClassNotFoundException
+     * @return : Map containing the set of XmlMetadata objects.\
      */
-    public ConcurrentHashMap<String, XmlMetadata> getMetadata(String index)
-            throws IOException, ClassNotFoundException, JAXBException {
-
-        return this.indexes.get(index);
+    public ConcurrentHashMap<String, XmlMetadata> getMetadata() {
+        return this.indexMap;
     }
 
     /**
      * Read the metadata file and create an in-memory map containing collection paths and xml files.
-     * @throws JAXBException
      */
-    public void readMetadataFile() throws JAXBException {
-        JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
-        Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
-        VXQueryIndex indexes = (VXQueryIndex) jaxbUnmarshaller.unmarshal(metaFile);
-
-        List<XmlMetadataCollection> list = indexes.getIndex();
-
-
-        for (XmlMetadataCollection collection : list) {
-            String indexPath = collection.getIndexLocation();
-            ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
-            List<XmlMetadata> metadata = collection.getMetadataList();
+    public void readMetadataFile() {
+        try {
+            JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
+            Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+            XmlMetadataCollection indexes = (XmlMetadataCollection) jaxbUnmarshaller.unmarshal(metaFile);
 
-            this.indexToCollection.put(indexPath, collection.getCollection());
+            this.collection = indexes.getCollection();
+            this.index = indexes.getIndexLocation();
 
-            for (XmlMetadata mData : metadata) {
-                metadataMap.put(mData.getPath(), mData);
+            for (XmlMetadata metadata : indexes.getMetadataList()) {
+                this.indexMap.put(index, metadata);
+            }
+        } catch (JAXBException e) {
+            if (LOGGER.isTraceEnabled()) {
+                LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
             }
-            this.indexes.put(indexPath, metadataMap);
         }
     }
 
     /**
      * Write the content of the ConcurrentHashMap to the xml metadata file.
-     * @throws FileNotFoundException
-     * @throws JAXBException
      */
-    public void writeMetadataToFile() throws FileNotFoundException, JAXBException {
-        VXQueryIndex index = new VXQueryIndex();
-        List<XmlMetadataCollection> xmlMetadataCollections = new ArrayList<>();
-
-        for (Map.Entry<String, ConcurrentHashMap<String, XmlMetadata>> entry : indexes.entrySet()) {
-            XmlMetadataCollection metadataCollection = new XmlMetadataCollection();
-            List<XmlMetadata> metadataList = new ArrayList<>();
-            metadataCollection.setIndexLocation(entry.getKey());
-            metadataCollection.setCollection(indexToCollection.get(entry.getKey()));
-            metadataList.addAll(entry.getValue().values());
-            metadataCollection.setMetadataList(metadataList);
-            xmlMetadataCollections.add(metadataCollection);
-        }
-        index.setIndex(xmlMetadataCollections);
+    public void writeMetadataToFile() {
+        XmlMetadataCollection collection = new XmlMetadataCollection();
+        List<XmlMetadata> metadataList = new ArrayList<>();
 
+        for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) {
+            metadataList.add(entry.getValue());
+        }
 
-        FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
-        JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
-        Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
-        jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
-        jaxbMarshaller.marshal(index, fileOutputStream);
+        collection.setMetadataList(metadataList);
+        collection.setCollection(this.collection);
+        collection.setIndexLocation(this.index);
+        try{
+            FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
+            JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
+            Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+            jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+            jaxbMarshaller.marshal(collection, fileOutputStream);
+
+            if (LOGGER.isDebugEnabled()) {
+                LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
+            }
+        } catch (JAXBException | FileNotFoundException e) {
+            if (LOGGER.isTraceEnabled()) {
+                LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e);
+            }
+        }
 
-        if (LOGGER.isDebugEnabled())
-            LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
 
     }
 
@@ -157,14 +144,20 @@ public class MetaFileUtil {
      *
      * @param file : File which the checksum should be generated.
      * @return : Checksum String
-     * @throws NoSuchAlgorithmException
      * @throws IOException
      */
-    public String generateMD5(File file) throws NoSuchAlgorithmException, IOException {
-        MessageDigest md = MessageDigest.getInstance("MD5");
-        md.update(Files.readAllBytes(file.toPath()));
-        byte[] md5 = md.digest();
-        return DatatypeConverter.printHexBinary(md5);
+    public String generateMD5(File file) throws  IOException {
+        try {
+            MessageDigest md = MessageDigest.getInstance("MD5");
+            md.update(Files.readAllBytes(file.toPath()));
+            byte[] md5 = md.digest();
+            return DatatypeConverter.printHexBinary(md5);
+        } catch (NoSuchAlgorithmException e) {
+            if (LOGGER.isTraceEnabled()) {
+                LOGGER.log(Level.ERROR, "No Such Algorithm Error " + e.getMessage());
+            }
+            return null;
+        }
     }
 
     /**
@@ -189,21 +182,17 @@ public class MetaFileUtil {
 
     /**
      * Get the collection for a given index location.
-     * @param index : path to index
-     * @return
+     * @return collection folder for a given index.
      */
-    public String getCollection(String index) {
-        return this.indexToCollection.get(index);
+    public String getCollection() {
+        return this.collection;
     }
 
     /**
      * Set the entry for given index and collection.
-     * @param index : path to index
      * @param collection : path to corresponding collection
      */
-    public void setCollectionForIndex(String index, String collection) {
-        if (this.indexToCollection.get(index)==null) {
-            this.indexToCollection.put(index, collection);
-        }
+    public void setCollection(String collection) {
+        this.collection = collection;
     }
 }
index 270fb8d..1f5c3e9 100644 (file)
 */
 package org.apache.vxquery.runtime.functions.index.updateIndex;
 
-import javax.xml.bind.annotation.*;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
 import java.util.List;
 
 /**
index 6fa92e1..543719d 100644 (file)
  */
 package org.apache.vxquery.indexing;
 
-import junit.framework.Assert;
+import java.io.File;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.xml.bind.JAXBException;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
 import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
@@ -24,12 +31,7 @@ import org.junit.FixMethodOrder;
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
 
-import javax.xml.bind.JAXBException;
-import java.io.File;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
+import junit.framework.Assert;
 
 /**
  * Test cases for testing MetaFileUtil functions.
@@ -49,7 +51,7 @@ public class MetaFileUtilTest {
     @BeforeClass
     public static void setup() {
         new File(TestConstants.INDEX_DIR).mkdir();
-        metaFileUtil = MetaFileUtil.create(TestConstants.INDEX_DIR);
+        metaFileUtil = new MetaFileUtil(TestConstants.INDEX_DIR);
         initialMap = TestConstants.getInitialMap();
         modifiedMap = TestConstants.getModifiedMap();
     }
@@ -58,7 +60,7 @@ public class MetaFileUtilTest {
      * Test case for generating MD5 string for an XML file.
      */
     @Test
-    public void step1_testGenerateMD5ForXML() throws IOException, NoSuchAlgorithmException {
+    public void step1_testGenerateMD5ForXML() throws IOException {
         TestConstants.createXML("catalog.xml");
         File xml = new File(TestConstants.XML_FILE);
         String md5 = metaFileUtil.generateMD5(xml);
@@ -71,7 +73,7 @@ public class MetaFileUtilTest {
      * Test the creation of metadata file.
      */
     @Test
-    public void step2_testCreateMetaDataFile() throws IOException, JAXBException {
+    public void step2_testCreateMetaDataFile() {
         ConcurrentHashMap<String, XmlMetadata> initialMap = TestConstants.getInitialMap();
         metaFileUtil.updateMetadataMap(initialMap, "");
         metaFileUtil.writeMetadataToFile();
@@ -82,8 +84,8 @@ public class MetaFileUtilTest {
      * Validate the content of the file.
      */
     @Test
-    public void step3_testValidateMetadataFile() throws IOException, ClassNotFoundException, JAXBException {
-        ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata("");
+    public void step3_testValidateMetadataFile() {
+        ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata();
         Set<String> from = fromFile.keySet();
         Set<String> initial = initialMap.keySet();
 
@@ -100,7 +102,7 @@ public class MetaFileUtilTest {
      * Change the xml file and test whether the changes are detected.
      */
     @Test
-    public void step4_testDetectFileChanges() throws IOException, NoSuchAlgorithmException {
+    public void step4_testDetectFileChanges() throws IOException {
         TestConstants.createXML("catalog_edited.xml");
         File xml = new File(TestConstants.XML_FILE);
         Assert.assertTrue(metaFileUtil.generateMD5(xml).equals(TestConstants.CHANGED_MD5));
@@ -112,7 +114,7 @@ public class MetaFileUtilTest {
     @Test
     public void step5_testUpdateMetadata()
             throws IOException, ClassNotFoundException, NoSuchAlgorithmException, JAXBException {
-        ConcurrentHashMap<String, XmlMetadata> fromFileMap = metaFileUtil.getMetadata("");
+        ConcurrentHashMap<String, XmlMetadata> fromFileMap = metaFileUtil.getMetadata();
         XmlMetadata modified = fromFileMap.get(TestConstants.XML_FILE);
 
         File xml = new File(TestConstants.XML_FILE);
@@ -122,7 +124,7 @@ public class MetaFileUtilTest {
 
         metaFileUtil.updateMetadataMap(fromFileMap, TestConstants.INDEX_DIR);
 
-        Assert.assertNotNull(metaFileUtil.getMetadata(TestConstants.INDEX_DIR));
+        Assert.assertNotNull(metaFileUtil.getMetadata());
 
     }
 
@@ -130,8 +132,8 @@ public class MetaFileUtilTest {
      * Validate the updated metadata.
      */
     @Test
-    public void step6_testVerifyMetadataChange() throws IOException, ClassNotFoundException, JAXBException {
-        ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata(TestConstants.INDEX_DIR);
+    public void step6_testVerifyMetadataChange() {
+        ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata();
         Set<String> from = fromFile.keySet();
         Set<String> modified = modifiedMap.keySet();
 
index 68d40b5..b79107e 100644 (file)
@@ -16,7 +16,13 @@ package org.apache.vxquery.indexing;
 
 import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.concurrent.ConcurrentHashMap;
index 18d9173..f43b22a 100644 (file)
@@ -17,6 +17,7 @@
 <cluster xmlns="cluster">
     <name>local</name>
     <username>joe</username>
+    <index_directory>/tmp/indexFolder</index_directory>
     <master_node>
         <id>master</id>
         <client_ip>128.195.52.177</client_ip>
index 4a48620..e0d07f8 100644 (file)
@@ -16,6 +16,7 @@
 -->
 <cluster xmlns="cluster">
     <name>local</name>
+    <index_directory>/tmp/indexFolder</index_directory>
     <master_node>
         <id>master</id>
         <client_ip>127.0.0.1</client_ip>
index deeee33..8ce1607 100644 (file)
@@ -120,7 +120,7 @@ class ClusterActions:
     def start_nc(self, machine, cc):
         print "Start Node Controller."
         print "  " + machine.get_id() + " " + machine.get_ip()
-        command = "./vxquery-server/target/appassembler/bin/startnc.sh " + machine.get_id() + " " + machine.get_ip() + " " + cc.get_client_ip() + " \"" + cc.get_client_port() + "\" \"" + machine.get_java_opts() + "\""
+        command = "./vxquery-server/target/appassembler/bin/startnc.sh " + machine.get_id() + " " + machine.get_ip() + " " + cc.get_client_ip() + " \"" + cc.get_client_port() + "\" \"" + machine.get_java_opts() + "\" \"" + self.ci.get_index_directory() + "\""
         self.run_remote_command(machine.get_username(), machine.get_id(), command)
 
     def stop_cc_and_all_ncs(self, machine):
index 94b231d..d70828d 100644 (file)
@@ -28,6 +28,9 @@ class ClusterInformation:
     def get_java_opts(self):
         return get_tag_text(self.config, "java_opts")
 
+    def get_index_directory(self):
+        return get_tag_text(self.config, "index_directory");
+
     def get_master_node_machine(self):
         master_node = self.config.getElementsByTagName("master_node")[0]
         id = NodeXmlReader.get_cluster_id(master_node)
index a199ec8..d719d22 100755 (executable)
@@ -24,6 +24,7 @@ IPADDR=$2
 CCHOST=$3
 CCPORT=$4
 J_OPTS=$5
+IO_DEVICES=$6
 
 #Import cluster properties
 MYDIR="$(dirname -- $0)"
@@ -62,6 +63,9 @@ NC_OPTIONS=" -cc-host ${CCHOST} -cluster-net-ip-address ${IPADDR}  -data-ip-addr
 
     NC_OPTIONS=" ${NC_OPTIONS} ${NCOPTS} "
 
+[ "${IO_DEVICES}" ] &&
+    NC_OPTIONS=" ${NC_OPTIONS} -iodevices ${IO_DEVICES} "
+
 
 echo "${JAVA_OPTS}" &> ${NCLOGS_DIR}/nc.log
 
index 75dce1b..d7db8a8 100644 (file)
 
 package org.apache.vxquery.xtest;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
 import org.apache.hyracks.control.cc.ClusterControllerService;
 import org.apache.hyracks.control.common.controllers.CCConfig;
 import org.apache.hyracks.control.common.controllers.NCConfig;
 import org.apache.hyracks.control.nc.NodeControllerService;
 
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
 public class TestClusterUtil {
 
     private static int clientNetPort = 39000;
@@ -34,6 +34,7 @@ public class TestClusterUtil {
     private static int profileDumpPeriod = 10000;
     private static String ccHost = "localhost";
     private static String nodeId = "nc1";
+    private static String ioDevices = "target/tmp/indexFolder";
 
     private TestClusterUtil() {
     }
@@ -58,6 +59,7 @@ public class TestClusterUtil {
         ncConfig1.dataIPAddress = publicAddress;
         ncConfig1.resultIPAddress = publicAddress;
         ncConfig1.nodeId = nodeId;
+        ncConfig1.ioDevices = ioDevices;
         return ncConfig1;
     }
 
index c6b339b..78ed8ff 100644 (file)
@@ -80,6 +80,7 @@ public abstract class AbstractXQueryTest {
             case EXPECTED_RESULT_GOT_SAME_RESULT:
                 break;
             case NO_RESULT_FILE:
+                fail(result.state + " (" + result.time + " ms): " + result.testCase.getXQueryDisplayName());
                 break;
         }
     }
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt
new file mode 100644 (file)
index 0000000..17d6fec
--- /dev/null
@@ -0,0 +1 @@
+src/test/resources/TestSources/ghcnd
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt
new file mode 100644 (file)
index 0000000..8b13789
--- /dev/null
@@ -0,0 +1 @@
+
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt
new file mode 100644 (file)
index 0000000..0be5d98
--- /dev/null
@@ -0,0 +1,2 @@
+true
+true
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/showIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/showIndex1.txt
new file mode 100644 (file)
index 0000000..31d670b
--- /dev/null
@@ -0,0 +1,2 @@
+src/test/resources/TestSources/ghcnd/half_1
+src/test/resources/TestSources/ghcnd/half_2
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/updateIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/updateIndex.txt
new file mode 100644 (file)
index 0000000..0be5d98
--- /dev/null
@@ -0,0 +1,2 @@
+true
+true
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1.txt
new file mode 100644 (file)
index 0000000..baf9dca
--- /dev/null
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex2.txt
new file mode 100644 (file)
index 0000000..ef8dde4
--- /dev/null
@@ -0,0 +1 @@
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex3.txt
new file mode 100644 (file)
index 0000000..d8263ee
--- /dev/null
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex4.txt
new file mode 100644 (file)
index 0000000..f30101c
--- /dev/null
@@ -0,0 +1 @@
+3.3
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex5.txt
new file mode 100644 (file)
index 0000000..c84c360
--- /dev/null
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex6.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex6.txt
new file mode 100644 (file)
index 0000000..9abedff
--- /dev/null
@@ -0,0 +1,2 @@
+<station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
+<station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex7.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex7.txt
new file mode 100644 (file)
index 0000000..c84c360
--- /dev/null
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/createIndex.txt
new file mode 100644 (file)
index 0000000..4b1e4bc
--- /dev/null
@@ -0,0 +1,4 @@
+true
+true
+true
+true
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/showIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/showIndex1.txt
new file mode 100644 (file)
index 0000000..e696773
--- /dev/null
@@ -0,0 +1,6 @@
+src/test/resources/TestSources/ghcnd/half_2/quarter_3
+src/test/resources/TestSources/ghcnd/half_1
+src/test/resources/TestSources/ghcnd/half_2/quarter_4
+src/test/resources/TestSources/ghcnd/half_2
+src/test/resources/TestSources/ghcnd/half_1/quarter_2
+src/test/resources/TestSources/ghcnd/half_1/quarter_1
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/updateIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/updateIndex.txt
new file mode 100644 (file)
index 0000000..4b1e4bc
--- /dev/null
@@ -0,0 +1,4 @@
+true
+true
+true
+true
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1.txt
new file mode 100644 (file)
index 0000000..baf9dca
--- /dev/null
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex2.txt
new file mode 100644 (file)
index 0000000..ef8dde4
--- /dev/null
@@ -0,0 +1 @@
+<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex3.txt
new file mode 100644 (file)
index 0000000..d8263ee
--- /dev/null
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex4.txt
new file mode 100644 (file)
index 0000000..f30101c
--- /dev/null
@@ -0,0 +1 @@
+3.3
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex5.txt
new file mode 100644 (file)
index 0000000..c84c360
--- /dev/null
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex6.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex6.txt
new file mode 100644 (file)
index 0000000..9abedff
--- /dev/null
@@ -0,0 +1,2 @@
+<station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
+<station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex7.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex7.txt
new file mode 100644 (file)
index 0000000..c84c360
--- /dev/null
@@ -0,0 +1,3 @@
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/project.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/project.txt
new file mode 100644 (file)
index 0000000..6bb7afe
--- /dev/null
@@ -0,0 +1,13 @@
+{"Captain":"Kirk","First Officer":"Spock","XQuery Evangelist":"XQuery Evangelist"}
+[1,2,3,4]
+{"Captain":"Archer","First Officer":"First Officer","XQuery Evangelist":"XQuery Evangelist"}
+true
+1
+null
+{"Captain":"Archer"}
+true
+1
+null
+true
+1
+null
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/remove_keys.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Libraries/remove_keys.txt
new file mode 100644 (file)
index 0000000..41732ea
--- /dev/null
@@ -0,0 +1,12 @@
+[1,2,3,4]
+true
+1
+null
+{"Engineer":"Trip"}
+true
+1
+null
+{"Captain":"Archer"}
+true
+1
+null
\ No newline at end of file
@@ -16,5 +16,5 @@
    under the License. :)
    
 (: Build Lucene Index :)
-build-index-on-collection( "src/test/resources/TestSources/ghcnd", "target/tmp/indexFolder")
-   
\ No newline at end of file
+build-index-on-collection("src/test/resources/TestSources/ghcnd")
+
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndex.xq
new file mode 100644 (file)
index 0000000..5ca9c2f
--- /dev/null
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Show indexes after createion :)
+show-index()
\ No newline at end of file
@@ -16,7 +16,7 @@
    under the License. :)
    
 (: Search Lucene Index :)
-for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
 let $datetime := xs:dateTime(fn:data($r/date))
 where $r/station eq "GHCND:AS000000003" 
     and fn:year-from-dateTime($datetime) ge 2000
@@ -19,6 +19,6 @@
 (: Find all reading for hurricane force wind warning or extreme wind warning. :)
 (: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744     :)
 (: meters per second). (Wind value is in tenth of a meter per second)         :)
-for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
 where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
 return $r
@@ -19,7 +19,7 @@
 (: Find the annual precipitation (PRCP) for a Seattle using the airport       :)
 (: station (US000000002) for 2002.                                            :)
 fn:sum(
-    for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
+    for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
     where $r/station eq "GHCND:US000000002" 
         and $r/dataType eq "PRCP" 
         and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
@@ -18,7 +18,7 @@
 (: Search Lucene Index :)
 (: Find the highest recorded temperature (TMAX) in Celsius.                   :)
 fn:max(
-    for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
+    for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
     where $r/dataType eq "TMAX"
     return $r/value
 ) div 10
@@ -18,6 +18,6 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
 where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
 return $r
@@ -18,6 +18,6 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $s in collection-from-index("target/tmp/indexFolder", "/stationCollection/station")/station
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
 where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
 return $s
@@ -18,8 +18,8 @@
 (: Search Lucene Index :)
 (: Find all the weather readings for Washington state for a specific day    :)
 (: 2002-2-2.                                                                  :)
-for $s in collection-from-index("target/tmp/indexFolder", "/stationCollection/station")/station
-for $r in collection-from-index("target/tmp/indexFolder", "/dataCollection/data")/data
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
     
 where $s/id eq $r/station 
     and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/createIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/createIndex.xq
new file mode 100644 (file)
index 0000000..d0ca5cc
--- /dev/null
@@ -0,0 +1,20 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+   
+(: Build Lucene Index :)
+build-index-on-collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")
+
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/showIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/showIndex.xq
new file mode 100644 (file)
index 0000000..5ca9c2f
--- /dev/null
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Show indexes after createion :)
+show-index()
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/updateIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/updateIndex.xq
new file mode 100644 (file)
index 0000000..6a4e6f7
--- /dev/null
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Update Lucene Index :)
+update-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
new file mode 100644 (file)
index 0000000..fecb56d
--- /dev/null
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+   
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003" 
+    and fn:year-from-dateTime($datetime) ge 2000
+    and fn:month-from-dateTime($datetime) eq 3 
+    and fn:day-from-dateTime($datetime) eq 3
+return $r
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
new file mode 100644 (file)
index 0000000..75c7a64
--- /dev/null
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find all reading for hurricane force wind warning or extreme wind warning. :)
+(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744     :)
+(: meters per second). (Wind value is in tenth of a meter per second)         :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
new file mode 100644 (file)
index 0000000..28f7473
--- /dev/null
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find the annual precipitation (PRCP) for a Seattle using the airport       :)
+(: station (US000000002) for 2002.                                            :)
+fn:sum(
+    for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+    where $r/station eq "GHCND:US000000002" 
+        and $r/dataType eq "PRCP" 
+        and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
+    return $r/value
+) div 10
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
new file mode 100644 (file)
index 0000000..317a141
--- /dev/null
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find the highest recorded temperature (TMAX) in Celsius.                   :)
+fn:max(
+    for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+    where $r/dataType eq "TMAX"
+    return $r/value
+) div 10
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
new file mode 100644 (file)
index 0000000..2deb4c3
--- /dev/null
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day    :)
+(: 2002-2-2.                                                                  :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
new file mode 100644 (file)
index 0000000..a0ce1e9
--- /dev/null
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day    :)
+(: 2002-2-2.                                                                  :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
+where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+return $s
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
new file mode 100644 (file)
index 0000000..b3e622c
--- /dev/null
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day    :)
+(: 2002-2-2.                                                                  :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+    
+where $s/id eq $r/station 
+    and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+    and xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/createIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/createIndex.xq
new file mode 100644 (file)
index 0000000..f2f0d52
--- /dev/null
@@ -0,0 +1,20 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+   
+(: Build Lucene Index :)
+build-index-on-collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")
+
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/showIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/showIndex.xq
new file mode 100644 (file)
index 0000000..5ca9c2f
--- /dev/null
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Show indexes after createion :)
+show-index()
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/updateIndex.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/updateIndex.xq
new file mode 100644 (file)
index 0000000..12f5677
--- /dev/null
@@ -0,0 +1,19 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Update Lucene Index :)
+update-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
new file mode 100644 (file)
index 0000000..0cccbc5
--- /dev/null
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+   
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003" 
+    and fn:year-from-dateTime($datetime) ge 2000
+    and fn:month-from-dateTime($datetime) eq 3 
+    and fn:day-from-dateTime($datetime) eq 3
+return $r
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
new file mode 100644 (file)
index 0000000..c282e31
--- /dev/null
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find all reading for hurricane force wind warning or extreme wind warning. :)
+(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744     :)
+(: meters per second). (Wind value is in tenth of a meter per second)         :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
new file mode 100644 (file)
index 0000000..33ea1c9
--- /dev/null
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find the annual precipitation (PRCP) for a Seattle using the airport       :)
+(: station (US000000002) for 2002.                                            :)
+fn:sum(
+    for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+    where $r/station eq "GHCND:US000000002" 
+        and $r/dataType eq "PRCP" 
+        and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
+    return $r/value
+) div 10
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
new file mode 100644 (file)
index 0000000..d213082
--- /dev/null
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find the highest recorded temperature (TMAX) in Celsius.                   :)
+fn:max(
+    for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+    where $r/dataType eq "TMAX"
+    return $r/value
+) div 10
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
new file mode 100644 (file)
index 0000000..1d98682
--- /dev/null
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day    :)
+(: 2002-2-2.                                                                  :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
new file mode 100644 (file)
index 0000000..abe2184
--- /dev/null
@@ -0,0 +1,23 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day    :)
+(: 2002-2-2.                                                                  :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
+where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+return $s
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
new file mode 100644 (file)
index 0000000..7b40ca0
--- /dev/null
@@ -0,0 +1,27 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+   
+     http://www.apache.org/licenses/LICENSE-2.0
+   
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License. :)
+
+(: Search Lucene Index :)
+(: Find all the weather readings for Washington state for a specific day    :)
+(: 2002-2-2.                                                                  :)
+for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+    
+where $s/id eq $r/station 
+    and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
+    and xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
+return $r
index 763d8c3..fc668ba 100644 (file)
       <title>Indexing</title>
       <description/>
    </GroupInfo>
-   <test-case name="create-index" FilePath="Indexing/" Creator="Steven Jacobs">
+   <test-case name="create-index" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Create Lucene Index from Collection.</description>
       <query name="createIndex" date="2016-05-26"/>
       <output-file compare="Text">createIndex.txt</output-file>
    </test-case>
-   <test-case name="use-index-1" FilePath="Indexing/" Creator="Steven Jacobs">
+   <test-case name="show-index-1" FilePath="Indexing/Partition-1/" Creator="Menaka Jayawardena">
+      <description>Show indexes after creating index.</description>
+      <query name="showIndex" date="2016-08-18"/>
+      <output-file compare="Text">showIndex1.txt</output-file>
+   </test-case>
+   <test-case name="use-index-1" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex1" date="2016-05-26"/>
       <output-file compare="Text">useIndex1.txt</output-file>
    </test-case>
-   <test-case name="use-index-2" FilePath="Indexing/" Creator="Steven Jacobs">
+   <test-case name="use-index-2" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex2" date="2016-05-26"/>
       <output-file compare="Text">useIndex2.txt</output-file>
    </test-case>
-   <test-case name="use-index-3" FilePath="Indexing/" Creator="Steven Jacobs">
+   <test-case name="use-index-3" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex3" date="2016-05-26"/>
       <output-file compare="Text">useIndex3.txt</output-file>
    </test-case>
-   <test-case name="use-index-4" FilePath="Indexing/" Creator="Steven Jacobs">
+   <test-case name="use-index-4" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex4" date="2016-05-26"/>
       <output-file compare="Text">useIndex4.txt</output-file>
    </test-case>
-   <test-case name="use-index-5" FilePath="Indexing/" Creator="Steven Jacobs">
+   <test-case name="use-index-5" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex5" date="2016-05-26"/>
       <output-file compare="Text">useIndex5.txt</output-file>
    </test-case>
-   <!--<test-case name="use-index-6" FilePath="Indexing/" Creator="Steven Jacobs">
+   <test-case name="use-index-6" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex6" date="2016-05-26"/>
       <output-file compare="Text">useIndex6.txt</output-file>
-   </test-case>-->
-   <test-case name="use-index-7" FilePath="Indexing/" Creator="Steven Jacobs">
+   </test-case>
+   <test-case name="use-index-7" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
       <description>Get Collection From Lucene Index</description>
       <query name="useIndex7" date="2016-05-26"/>
       <output-file compare="Text">useIndex7.txt</output-file>
    </test-case>
-   <test-case name="update-index" FilePath="Indexing/" Creator="Menaka Jayawardena">
+   <test-case name="update-index" FilePath="Indexing/Partition-1/" Creator="Menaka Jayawardena">
       <description>Update the existing index</description>
       <query name="updateIndex" date="2016-06-24"/>
       <output-file compare="Text">updateIndex.txt</output-file>
    </test-case>
-   <test-case name="delete-index" FilePath="Indexing/" Creator="Menaka Jayawardena">
+   <test-case name="delete-index" FilePath="Indexing/Partition-1/" Creator="Menaka Jayawardena">
       <description>Delete the existing index</description>
       <query name="deleteIndex" date="2016-06-24"/>
       <output-file compare="Text">deleteIndex.txt</output-file>
    </test-case>
+   <test-case name="show-index-2" FilePath="Indexing/Partition-1/" Creator="Menaka Jayawardena">
+      <description>Show indexes after deletion.</description>
+      <query name="showIndex" date="2016-08-18"/>
+      <output-file compare="Text">showIndex2.txt</output-file>
+   </test-case>
+   <test-case name="create-index" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Create Lucene Index from Collection.</description>
+      <query name="createIndex" date="2016-05-26"/>
+      <output-file compare="Text">createIndex.txt</output-file>
+   </test-case>
+   <test-case name="show-index-1" FilePath="Indexing/Partition-2/" Creator="Menaka Jayawardena">
+      <description>Show indexes after creating index.</description>
+      <query name="showIndex" date="2016-08-18"/>
+      <output-file compare="Text">showIndex1.txt</output-file>
+   </test-case>
+   <test-case name="use-index-1" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex1" date="2016-05-26"/>
+      <output-file compare="Text">useIndex1.txt</output-file>
+   </test-case>
+   <test-case name="use-index-2" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex2" date="2016-05-26"/>
+      <output-file compare="Text">useIndex2.txt</output-file>
+   </test-case>
+   <test-case name="use-index-3" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex3" date="2016-05-26"/>
+      <output-file compare="Text">useIndex3.txt</output-file>
+   </test-case>
+   <test-case name="use-index-4" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex4" date="2016-05-26"/>
+      <output-file compare="Text">useIndex4.txt</output-file>
+   </test-case>
+   <test-case name="use-index-5" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex5" date="2016-05-26"/>
+      <output-file compare="Text">useIndex5.txt</output-file>
+   </test-case>
+   <test-case name="use-index-6" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex6" date="2016-05-26"/>
+      <output-file compare="Text">useIndex6.txt</output-file>
+   </test-case>
+   <test-case name="use-index-7" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex7" date="2016-05-26"/>
+      <output-file compare="Text">useIndex7.txt</output-file>
+   </test-case>
+   <test-case name="update-index" FilePath="Indexing/Partition-2/" Creator="Menaka Jayawardena">
+      <description>Update the existing index</description>
+      <query name="updateIndex" date="2016-06-24"/>
+      <output-file compare="Text">updateIndex.txt</output-file>
+   </test-case>
+   <test-case name="create-index" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Create Lucene Index from Collection.</description>
+      <query name="createIndex" date="2016-05-26"/>
+      <output-file compare="Text">createIndex.txt</output-file>
+   </test-case>
+   <test-case name="show-index-1" FilePath="Indexing/Partition-4/" Creator="Menaka Jayawardena">
+      <description>Show indexes after creating index.</description>
+      <query name="showIndex" date="2016-08-18"/>
+      <output-file compare="Text">showIndex1.txt</output-file>
+   </test-case>
+   <test-case name="use-index-1" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex1" date="2016-05-26"/>
+      <output-file compare="Text">useIndex1.txt</output-file>
+   </test-case>
+   <test-case name="use-index-2" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex2" date="2016-05-26"/>
+      <output-file compare="Text">useIndex2.txt</output-file>
+   </test-case>
+   <test-case name="use-index-3" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex3" date="2016-05-26"/>
+      <output-file compare="Text">useIndex3.txt</output-file>
+   </test-case>
+   <test-case name="use-index-4" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex4" date="2016-05-26"/>
+      <output-file compare="Text">useIndex4.txt</output-file>
+   </test-case>
+   <test-case name="use-index-5" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex5" date="2016-05-26"/>
+      <output-file compare="Text">useIndex5.txt</output-file>
+   </test-case>
+   <test-case name="use-index-6" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex6" date="2016-05-26"/>
+      <output-file compare="Text">useIndex6.txt</output-file>
+   </test-case>
+   <test-case name="use-index-7" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+      <description>Get Collection From Lucene Index</description>
+      <query name="useIndex7" date="2016-05-26"/>
+      <output-file compare="Text">useIndex7.txt</output-file>
+   </test-case>
+   <test-case name="update-index" FilePath="Indexing/Partition-4/" Creator="Menaka Jayawardena">
+      <description>Update the existing index</description>
+      <query name="updateIndex" date="2016-06-24"/>
+      <output-file compare="Text">updateIndex.txt</output-file>
+   </test-case>
 </test-group>