merged master 153/head
authorJörg Unbehauen <joerg@unbehauen.net>
Fri, 4 Aug 2017 13:31:21 +0000 (15:31 +0200)
committerJörg Unbehauen <joerg@unbehauen.net>
Fri, 4 Aug 2017 13:31:21 +0000 (15:31 +0200)
1  2 
core/src/main/java/org/apache/metamodel/MetaModelHelper.java
core/src/test/java/org/apache/metamodel/MetaModelHelperTest.java
jdbc/src/main/java/org/apache/metamodel/jdbc/JdbcMetadataLoader.java
jdbc/src/test/java/org/apache/metamodel/jdbc/H2databaseTest.java
jdbc/src/test/java/org/apache/metamodel/jdbc/HsqldbTest.java
pojo/src/main/java/org/apache/metamodel/pojo/PojoDataContext.java

   */
  package org.apache.metamodel;
  
- import java.util.ArrayList;
- import java.util.Arrays;
- import java.util.Collection;
- import java.util.Collections;
- import java.util.Comparator;
- import java.util.HashMap;
- import java.util.HashSet;
- import java.util.List;
- import java.util.Map;
+ import java.util.*;
  import java.util.Map.Entry;
  import java.util.stream.Collectors;
 +import java.util.stream.Stream;
 +
  import org.apache.metamodel.data.CachingDataSetHeader;
  import org.apache.metamodel.data.DataSet;
  import org.apache.metamodel.data.DataSetHeader;
@@@ -184,67 -180,86 +182,87 @@@ public final class MetaModelHelper 
          if (fromDataSets.length == 1) {
              return getFiltered(fromDataSets[0], whereItems);
          }
+         // do a nested loop join, no matter what
+         Iterator<DataSet> dsIter = Arrays.asList(fromDataSets).iterator();
  
-         List<SelectItem> selectItems = new ArrayList<SelectItem>();
-         for (DataSet dataSet : fromDataSets) {
-             for (int i = 0; i < dataSet.getSelectItems().size(); i++) {
-                 SelectItem item = dataSet.getSelectItems().get(i);
-                 selectItems.add(item);
-             }
-         }
 +
-         int selectItemOffset = 0;
-         List<Object[]> data = new ArrayList<Object[]>();
-         for (int fromDataSetIndex = 0; fromDataSetIndex < fromDataSets.length; fromDataSetIndex++) {
-             DataSet fromDataSet = fromDataSets[fromDataSetIndex];
-             List<SelectItem> fromSelectItems = fromDataSet.getSelectItems();
-             if (fromDataSetIndex == 0) {
-                 while (fromDataSet.next()) {
-                     Object[] values = fromDataSet.getRow().getValues();
-                     Object[] row = new Object[selectItems.size()];
-                     System.arraycopy(values, 0, row, selectItemOffset, values.length);
-                     data.add(row);
-                 }
-                 fromDataSet.close();
-             } else {
-                 List<Object[]> fromDataRows = new ArrayList<Object[]>();
-                 while (fromDataSet.next()) {
-                     fromDataRows.add(fromDataSet.getRow().getValues());
-                 }
-                 fromDataSet.close();
-                 for (int i = 0; i < data.size(); i = i + fromDataRows.size()) {
-                     Object[] originalRow = data.get(i);
-                     data.remove(i);
-                     for (int j = 0; j < fromDataRows.size(); j++) {
-                         Object[] newRow = fromDataRows.get(j);
-                         System.arraycopy(newRow, 0, originalRow, selectItemOffset, newRow.length);
-                         data.add(i + j, originalRow.clone());
-                     }
-                 }
-             }
-             selectItemOffset += fromSelectItems.size();
-         }
+         DataSet joined = dsIter.next();
+         while (dsIter.hasNext()) {
+             joined = nestedLoopJoin(dsIter.next(), joined, (whereItems));
  
-         if (data.isEmpty()) {
-             return new EmptyDataSet(selectItems);
          }
  
-         final DataSetHeader header = new CachingDataSetHeader(selectItems);
-         final List<Row> rows = new ArrayList<Row>(data.size());
-         for (Object[] objects : data) {
-             rows.add(new DefaultRow(header, objects, null));
+         return joined;
+     }
+     /**
+      * Executes a simple nested loop join. The innerLoopDs will be copied in an
+      * in-memory dataset.
+      *
+      */
+     public static InMemoryDataSet nestedLoopJoin(DataSet innerLoopDs, DataSet outerLoopDs,
+             Iterable<FilterItem> filtersIterable) {
+         List<FilterItem> filters = new ArrayList<>();
+         for (FilterItem fi : filtersIterable) {
+             filters.add(fi);
          }
 -        List<SelectItem> allItems = new ArrayList<>(Arrays.asList(outerLoopDs.getSelectItems()));
 -        allItems.addAll(Arrays.asList(innerLoopDs.getSelectItems()));
+         List<Row> innerRows = innerLoopDs.toRows();
++        List<SelectItem> allItems = new ArrayList<>(outerLoopDs.getSelectItems());
++        allItems.addAll(innerLoopDs.getSelectItems());
  
-         DataSet result = new InMemoryDataSet(header, rows);
-         if (whereItems != null) {
-             DataSet filteredResult = getFiltered(result, whereItems);
-             result = filteredResult;
+         Set<FilterItem> applicableFilters = applicableFilters(filters, allItems);
+         DataSetHeader jointHeader = new CachingDataSetHeader(allItems);
+         List<Row> resultRows = new ArrayList<>();
+         for (Row outerRow : outerLoopDs) {
+             for (Row innerRow : innerRows) {
+                 Object[] joinedRowObjects = new Object[outerRow.getValues().length + innerRow.getValues().length];
+                 System.arraycopy(outerRow.getValues(), 0, joinedRowObjects, 0, outerRow.getValues().length);
+                 System.arraycopy(innerRow.getValues(), 0, joinedRowObjects, outerRow.getValues().length, innerRow
+                         .getValues().length);
+                 Row joinedRow = new DefaultRow(jointHeader, joinedRowObjects);
+                 if (applicableFilters.isEmpty() || applicableFilters.stream().allMatch(fi -> fi.accept(joinedRow))) {
+                     resultRows.add(joinedRow);
+                 }
+             }
          }
-         return result;
+         return new InMemoryDataSet(jointHeader, resultRows);
      }
  
-     public static DataSet getCarthesianProduct(DataSet[] fromDataSets, FilterItem... filterItems) {
-         return getCarthesianProduct(fromDataSets, Arrays.asList(filterItems));
+     /**
+      * Filters the FilterItems such that only the FilterItems are returned,
+      * which contain SelectItems that are contained in selectItemList
+      * 
+      * @param filters
+      * @param selectItemList
+      * @return
+      */
+     private static Set<FilterItem> applicableFilters(Collection<FilterItem> filters,
+             Collection<SelectItem> selectItemList) {
+         Set<SelectItem> items = new HashSet<SelectItem>(selectItemList);
+         return filters.stream().filter(fi -> {
+             Collection<SelectItem> fiSelectItems = new ArrayList<>();
+             fiSelectItems.add(fi.getSelectItem());
+             Object operand = fi.getOperand();
+             if (operand instanceof SelectItem) {
+                 fiSelectItems.add((SelectItem) operand);
+             }
+             return items.containsAll(fiSelectItems);
+         }).collect(Collectors.toSet());
      }
  
      public static DataSet getFiltered(DataSet dataSet, Iterable<FilterItem> filterItems) {
@@@ -116,21 -115,19 +116,21 @@@ public class MetaModelHelperTest extend
  
      public void testSimpleCarthesianProduct() throws Exception {
          DataSet dataSet = MetaModelHelper.getCarthesianProduct(createDataSet1(), createDataSet2());
+         List<String> results = new ArrayList<String>();
  
 -        assertEquals(2, dataSet.getSelectItems().length);
++
+         while (dataSet.next()) {
+             results.add(dataSet.getRow().toString());
+         }
-         assertTrue(dataSet.next());
-         assertEquals("Row[values=[f, b]]", dataSet.getRow().toString());
-         assertTrue(dataSet.next());
-         assertEquals("Row[values=[f, a]]", dataSet.getRow().toString());
-         assertTrue(dataSet.next());
-         assertTrue(dataSet.next());
-         assertTrue(dataSet.next());
-         assertTrue(dataSet.next());
-         assertTrue(dataSet.next());
-         assertTrue(dataSet.next());
-         assertTrue(dataSet.next());
-         assertEquals("Row[values=[o, r]]", dataSet.getRow().toString());
-         assertFalse(dataSet.next());
 +        assertEquals(2, dataSet.getSelectItems().size());
+         assertEquals(9, results.size());
+         assertTrue(results.contains("Row[values=[f, b]]"));
+         assertTrue(results.contains("Row[values=[f, a]]"));
+         assertTrue(results.contains("Row[values=[f, r]]"));
+         assertTrue(results.contains("Row[values=[o, b]]"));
+         assertTrue(results.contains("Row[values=[o, a]]"));
+         assertTrue(results.contains("Row[values=[o, r]]"));
++
      }
  
      public void testTripleCarthesianProduct() throws Exception {
          data1.add(new Object[] { "f" });
          data1.add(new Object[] { "o" });
          data1.add(new Object[] { "o" });
 -        DataSet dataSet1 = createDataSet(new SelectItem[] { new SelectItem(new MutableColumn("foo",
 -                ColumnType.VARCHAR)) }, data1);
++
 +        DataSet dataSet1 = createDataSet(
 +                Lists.newArrayList( new SelectItem(new MutableColumn("foo", ColumnType.VARCHAR)) ), data1);
++
          return dataSet1;
      }
  
          List<Object[]> data3 = new ArrayList<Object[]>();
          data3.add(new Object[] { "w00p", true });
          data3.add(new Object[] { "yippie", false });
 -        DataSet dataSet3 = createDataSet(new SelectItem[] { new SelectItem("expression", "e"), new SelectItem("webish?",
 -                "w") }, data3);
++
 +        DataSet dataSet3 = createDataSet(Lists.newArrayList(new SelectItem("expression", "e"),
 +                new SelectItem("webish?", "w") ), data3);
++
          return dataSet3;
      }
  
          return dataSet4;
      }
  
 -        DataSet dataSet5 = createDataSet(new SelectItem[] { new SelectItem(new MutableColumn("nr", ColumnType.BIGINT)),
+     private int bigDataSetSize = 3000;
+     /**
+      * 
+      * @return a big dataset, mocking an employee table
+      */
+     private DataSet createDataSet5() {
+         List<Object[]> data5 = new ArrayList<Object[]>();
+         for (int i = 0; i < bigDataSetSize; i++) {
+             data5.add(new Object[] { i, "Person_" + i, bigDataSetSize - (i + 1) });
+         }
 -                        ColumnType.BIGINT)) }, data5);
++        DataSet dataSet5 = createDataSet(Lists.newArrayList( new SelectItem(new MutableColumn("nr", ColumnType.BIGINT)),
+                 new SelectItem(new MutableColumn("name", ColumnType.STRING)), new SelectItem(new MutableColumn("dnr",
 -        DataSet dataSet6 = createDataSet(new SelectItem[] { new SelectItem(new MutableColumn("nr", ColumnType.BIGINT)),
 -                new SelectItem(new MutableColumn("name", ColumnType.STRING)), }, data6);
++                        ColumnType.BIGINT)) ), data5);
+         return dataSet5;
+     }
+     /**
+      * 
+      * @return a big dataset, mocking an department table
+      */
+     private DataSet createDataSet6() {
+         List<Object[]> data6 = new ArrayList<Object[]>();
+         for (int i = 0; i < bigDataSetSize; i++) {
+             data6.add(new Object[] { i, "Department_" + i });
+         }
++        DataSet dataSet6 = createDataSet(Lists.newArrayList(new SelectItem(new MutableColumn("nr", ColumnType.BIGINT)),
++                new SelectItem(new MutableColumn("name", ColumnType.STRING))), data6);
+         return dataSet6;
+     }
      public void testGetTables() throws Exception {
          MutableTable table1 = new MutableTable("table1");
          MutableTable table2 = new MutableTable("table2");
          assertEquals("Row[values=[1, 2, null]]", joinedDs.getRow().toString());
          assertFalse(joinedDs.next());
      }
 -        FilterItem fi = new FilterItem(employees.getSelectItems()[2], OperatorType.EQUALS_TO, departmens
 -                .getSelectItems()[0]);
+     public void testCarthesianProductScalability() {
+         DataSet employees = createDataSet5();
+         DataSet departmens = createDataSet6();
++        FilterItem fi = new FilterItem(employees.getSelectItems().get(2), OperatorType.EQUALS_TO, departmens
++                .getSelectItems().get(0));
+         DataSet joined = MetaModelHelper.getCarthesianProduct(new DataSet[] { employees, departmens }, fi);
+         int count = 0;
+         while (joined.next()) {
+             count++;
+         }
+         assertTrue(count == bigDataSetSize);
+     }
  }
@@@ -452,9 -468,42 +467,42 @@@ final class JdbcMetadataLoader implemen
                  logger.error("pkColumn={}", pkColumn);
                  logger.error("fkColumn={}", fkColumn);
              } else {
-                 MutableRelationship.createRelationship(pkColumn, fkColumn);
+                 if (!relations.containsKey(pkTable)) {
+                     relations.put(pkTable, new HashMap<>());
+                 }
+                 // get or init the columns tuple
+                 ColumnsTuple ct = relations.get(pkTable).get(fkTable);
+                 if (Objects.isNull(ct)) {
+                     ct = new ColumnsTuple();
+                     relations.get(pkTable).put(fkTable, ct);
+                 }
+                 // we can now safely add the columns
+                 ct.getPkCols().add(pkColumn);
+                 ct.getFkCols().add(fkColumn);
              }
          }
 -                .createRelationship(ct.getPkCols().toArray(new Column[0]), ct.getFkCols().toArray(new Column[0])));
+         relations.values().stream().flatMap(map -> map.values().stream()).forEach(ct -> MutableRelationship
++                .createRelationship(ct.getPkCols(), ct.getFkCols()));
+     }
+     /**
+      * Represents the columns of a relationship while it is being built from a
+      * {@link ResultSet}.
+      */
+     private static class ColumnsTuple {
+         private final List<Column> pkCols = new ArrayList<>();
+         private final List<Column> fkCols = new ArrayList<>();
+         public List<Column> getFkCols() {
+             return fkCols;
+         }
+         public List<Column> getPkCols() {
+             return pkCols;
+         }
      }
  
  }
@@@ -276,14 -281,17 +281,17 @@@ public class H2databaseTest extends Tes
  
              @Override
              public void run(UpdateCallback cb) {
-                 JdbcCreateTableBuilder createTableBuilder = (JdbcCreateTableBuilder) cb.createTable(schema, "test_table");
+                 JdbcCreateTableBuilder createTableBuilder = (JdbcCreateTableBuilder) cb.createTable(schema,
+                         "test_table");
                  Table writtenTable = createTableBuilder.withColumn("id").asPrimaryKey().ofType(ColumnType.INTEGER)
-                         .withColumn("name").ofSize(255).ofType(ColumnType.VARCHAR).withColumn("age").ofType(ColumnType.INTEGER)
-                         .execute();
+                         .withColumn("name").ofSize(255).ofType(ColumnType.VARCHAR).withColumn("age").ofType(
+                                 ColumnType.INTEGER).execute();
                  String sql = createTableBuilder.createSqlStatement();
-                 assertEquals("CREATE TABLE PUBLIC.test_table (id INTEGER, name VARCHAR(255), age INTEGER, PRIMARY KEY(id))", sql);
+                 assertEquals(
+                         "CREATE TABLE PUBLIC.test_table (id INTEGER, name VARCHAR(255), age INTEGER, PRIMARY KEY(id))",
+                         sql);
                  assertNotNull(writtenTable);
 -                assertEquals("[ID, NAME, AGE]", Arrays.toString(writtenTable.getColumnNames()));
 +                assertEquals("[ID, NAME, AGE]", Arrays.toString(writtenTable.getColumnNames().toArray()));
  
                  writtenTableRef.set(writtenTable);
              }
      public void testInterpretationOfNull() throws Exception {
          JdbcTestTemplates.interpretationOfNulls(conn);
      }
 -        assertEquals(1, schema.getRelationships().length);
+     public void testCompositeFkRelation() throws Exception {
+         try (Statement stmt = conn.createStatement()) {
+             stmt.execute(
+                     "CREATE TABLE PARENT (P1 INTEGER, P2 INTEGER, P3 INTEGER, P4 INTEGER, PRIMARY  KEY (P1,P2, P3, P4))");
+             stmt.execute(
+                     "CREATE TABLE CHILD (C1 INTEGER PRIMARY KEY, CP1 INTEGER , CP2 INTEGER, CP3 INTEGER, CP4 INTEGER, FOREIGN  KEY (CP1,CP2,CP3,CP4) REFERENCES  PARENT(P1,P2,P3,P4))");
+         }
+         final JdbcDataContext dc = new JdbcDataContext(conn);
+         final Schema schema = dc.getDefaultSchema();
 -        Relationship rel = schema.getRelationships()[0];
++        assertEquals(1, schema.getRelationships().size());
 -        assertEquals("CP1", rel.getForeignColumns()[0].getName());
 -        assertEquals("CP2", rel.getForeignColumns()[1].getName());
 -        assertEquals("CP3", rel.getForeignColumns()[2].getName());
 -        assertEquals("CP4", rel.getForeignColumns()[3].getName());
++        Relationship rel = schema.getRelationships().iterator().next();
 -        assertEquals("P1", rel.getPrimaryColumns()[0].getName());
 -        assertEquals("P2", rel.getPrimaryColumns()[1].getName());
 -        assertEquals("P3", rel.getPrimaryColumns()[2].getName());
 -        assertEquals("P4", rel.getPrimaryColumns()[3].getName());
++        assertEquals("CP1", rel.getForeignColumns().get(0).getName());
++        assertEquals("CP2", rel.getForeignColumns().get(1).getName());
++        assertEquals("CP3", rel.getForeignColumns().get(2).getName());
++        assertEquals("CP4", rel.getForeignColumns().get(3).getName());
++        assertEquals("P1", rel.getPrimaryColumns().get(0).getName());
++        assertEquals("P2", rel.getPrimaryColumns().get(1).getName());
++        assertEquals("P3", rel.getPrimaryColumns().get(2).getName());
++        assertEquals("P4", rel.getPrimaryColumns().get(3).getName());
+     }
  }
@@@ -146,8 -145,8 +145,9 @@@ public class HsqldbTest extends TestCas
          Table productsTable = schema.getTableByName("PRODUCTS");
          Table factTable = schema.getTableByName("ORDERFACT");
  
 -        Query q = new Query().from(new FromItem(JoinType.INNER, productsTable.getRelationships(factTable)[0]))
 -                .select(productsTable.getColumns()[0], factTable.getColumns()[0]);
 +        Query q = new Query().from(new FromItem(JoinType.INNER, productsTable.getRelationships(factTable).iterator().next())).select(
 +                productsTable.getColumns().get(0), factTable.getColumns().get(0));
++
          assertEquals(
                  "SELECT \"PRODUCTS\".\"PRODUCTCODE\", \"ORDERFACT\".\"ORDERNUMBER\" FROM PUBLIC.\"PRODUCTS\" INNER JOIN PUBLIC.\"ORDERFACT\" ON \"PRODUCTS\".\"PRODUCTCODE\" = \"ORDERFACT\".\"PRODUCTCODE\"",
                  q.toString());