ATLAS-4671: Basic Search : Exclude Header attributes of entities from the response master
authorPinal Shah <pinal.shah@freestoneinfotech.com>
Wed, 14 Sep 2022 10:06:39 +0000 (15:36 +0530)
committerPinal Shah <pinal.shah@freestoneinfotech.com>
Tue, 27 Sep 2022 11:11:26 +0000 (16:41 +0530)
Signed-off-by: Pinal Shah <pinal.shah@freestoneinfotech.com>
intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java
intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
webapp/src/test/resources/json/search-parameters/combination-filters.json
webapp/src/test/resources/json/search-parameters/entity-filters.json

index 79f5aae0d889495820510938a0b75a8d7f39ba07..dcad83c014c9c035cedd242a722bcd5416624787 100644 (file)
@@ -50,6 +50,7 @@ public class QuickSearchParameters implements Serializable {
     private Set<String>    attributes;
     private String         sortBy;
     private SortOrder      sortOrder;
+    private boolean        excludeHeaderAttributes;
 
     /**
      * for framework use.
@@ -158,4 +159,12 @@ public class QuickSearchParameters implements Serializable {
     public void setSortOrder(SortOrder sortOrder) {
         this.sortOrder = sortOrder;
     }
+
+    public boolean getExcludeHeaderAttributes() {
+        return excludeHeaderAttributes;
+    }
+
+    public void setExcludeHeaderAttributes(boolean excludeHeaderAttributes) {
+        this.excludeHeaderAttributes = excludeHeaderAttributes;
+    }
 }
index 78fb4a48f8165fe2df865a1ff1925f9b3c4a803a..8e68d0e82878c9ed30a68d569a801dccc0e1b1d1 100644 (file)
@@ -50,6 +50,8 @@ public class SearchParameters implements Serializable {
     private boolean includeClassificationAttributes;
     private boolean includeSubTypes                 = true;
     private boolean includeSubClassifications       = true;
+    private boolean excludeHeaderAttributes         = false;
+
     private int     limit;
     private int     offset;
     private String  marker;
@@ -258,6 +260,14 @@ public class SearchParameters implements Serializable {
         this.tagFilters = tagFilters;
     }
 
+    public boolean getExcludeHeaderAttributes() {
+        return excludeHeaderAttributes;
+    }
+
+    public void setExcludeHeaderAttributes(boolean excludeHeaderAttributes) {
+        this.excludeHeaderAttributes = excludeHeaderAttributes;
+    }
+
     /**
      * Attribute values included in the results
      * @return
@@ -307,6 +317,7 @@ public class SearchParameters implements Serializable {
                 includeClassificationAttributes == that.includeClassificationAttributes &&
                 includeSubTypes == that.includeSubTypes &&
                 includeSubClassifications == that.includeSubClassifications &&
+                excludeHeaderAttributes == that.excludeHeaderAttributes &&
                 limit == that.limit &&
                 offset == that.offset &&
                 Objects.equals(query, that.query) &&
@@ -323,7 +334,7 @@ public class SearchParameters implements Serializable {
     @Override
     public int hashCode() {
         return Objects.hash(query, typeName, classification, termName, includeSubTypes, includeSubClassifications,
-                            excludeDeletedEntities, includeClassificationAttributes, limit, offset, entityFilters,
+                            excludeDeletedEntities, includeClassificationAttributes, excludeHeaderAttributes, limit, offset, entityFilters,
                             tagFilters, attributes, sortBy, sortOrder);
     }
 
@@ -341,6 +352,7 @@ public class SearchParameters implements Serializable {
         sb.append(", includeSubClassifications='").append(includeSubClassifications).append('\'');
         sb.append(", excludeDeletedEntities=").append(excludeDeletedEntities);
         sb.append(", includeClassificationAttributes=").append(includeClassificationAttributes);
+        sb.append(", excludeHeaderAttributes=").append(excludeHeaderAttributes);
         sb.append(", limit=").append(limit);
         sb.append(", offset=").append(offset);
         sb.append(", entityFilters=").append(entityFilters);
index 8fbc22fa0e619fbe82380250207c0f9b4fa85016..4b113dbef37dd6d37761f4c12af6c4597d2e1a66 100644 (file)
@@ -468,6 +468,32 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
                 ret.setNextMarker(nextMarker);
             }
 
+            //If excludeHeaderAttributes is true, only primitive attributes requested in 'attributes' field will be sent in the response
+            Set<String> attributes = searchParameters.getAttributes();
+            if (searchContext.excludeHeaderAttributes()) {
+
+                AtlasSearchResult.AttributeSearchResult attributeSearchResult = new AtlasSearchResult.AttributeSearchResult();
+                attributeSearchResult.setName(new ArrayList<>(attributes));
+
+                Collection<List<Object>> values = new ArrayList<>();
+                for (AtlasVertex vertex : resultList) {
+                    List<Object> row = new ArrayList<>();
+
+                    for (String attrName : attributes) {
+                        AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
+                        AtlasAttribute  attribute  = entityType.getAttribute(attrName);
+                        Object value               = vertex.getProperty(attribute.getVertexPropertyName(), Object.class);
+
+                        row.add(value != null ? value : StringUtils.EMPTY);
+                    }
+                    values.add(row);
+                }
+                attributeSearchResult.setValues(new ArrayList<>(values));
+
+                ret.setAttributes(attributeSearchResult);
+                return ret;
+            }
+
             // By default any attribute that shows up in the search parameter should be sent back in the response
             // If additional values are requested then the entityAttributes will be a superset of the all search attributes
             // and the explicitly requested attribute(s)
@@ -922,6 +948,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
         searchParameters.setAttributes(quickSearchParameters.getAttributes());
         searchParameters.setSortBy(quickSearchParameters.getSortBy());
         searchParameters.setSortOrder(quickSearchParameters.getSortOrder());
+        searchParameters.setExcludeHeaderAttributes(quickSearchParameters.getExcludeHeaderAttributes());
 
         return searchParameters;
     }
index 01954d07e5e7a11c0f90e7849be31c33e55dc8e0..b8976e079cef98914975d986a83614e76357ed7e 100644 (file)
@@ -32,10 +32,13 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
 import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasBuiltInTypes;
 import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasStructType;
 import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.util.AtlasRepositoryConfiguration;
 import org.apache.commons.collections.CollectionUtils;
@@ -86,6 +89,7 @@ public class SearchContext {
     private boolean                       terminateSearch = false;
     private SearchProcessor               searchProcessor;
     private Integer                       marker;
+    private boolean                       hasRelationshipAttributes = false;
 
     public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
     public final static AtlasClassificationType MATCH_ALL_CLASSIFIED              = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
@@ -146,6 +150,9 @@ public class SearchContext {
         //remove other types if builtin type is present
         filterStructTypes();
 
+        //validate 'attributes' field
+        validateAttributes();
+
         //gather all classifications and its corresponding subtypes
         Set<String> classificationTypeAndSubTypes  = new HashSet<>();
         String classificationTypeAndSubTypesQryStr = null;
@@ -345,6 +352,37 @@ public class SearchContext {
         }
     }
 
+    private void validateAttributes() throws AtlasBaseException {
+        Set<String> attributes = searchParameters.getAttributes();
+        if (CollectionUtils.isNotEmpty(attributes) && CollectionUtils.isNotEmpty(entityTypes)) {
+
+            AtlasEntityType entityType = entityTypes.iterator().next();
+            for (String attr : attributes) {
+                AtlasAttribute attribute = entityType.getAttribute(attr);
+
+                if (attribute == null) {
+                    attribute = entityType.getRelationshipAttribute(attr, null);
+                    hasRelationshipAttributes = attribute != null;
+                }
+
+                if (attribute == null) {
+                    throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attr, entityType.getTypeName());
+                }
+            }
+        }
+    }
+
+    public boolean excludeHeaderAttributes() {
+        if (CollectionUtils.isNotEmpty(entityTypes) &&
+                searchParameters.getExcludeHeaderAttributes() &&
+                CollectionUtils.isNotEmpty(searchParameters.getAttributes()) &&
+                !hasRelationshipAttributes){
+            return true;
+        }
+
+        return false;
+    }
+
     public boolean hasAttributeFilter(FilterCriteria filterCriteria) {
         return filterCriteria != null &&
                (CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName()));
index fbc7396529fd40ecc98fadbb13dd5e92fe0200db..ecb398a44a574d0fe57161454e173e7baa907026 100644 (file)
@@ -31,6 +31,7 @@ import org.apache.atlas.model.discovery.AtlasAggregationEntry;
 import org.apache.atlas.model.instance.AtlasClassification;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.AtlasObjectId;
 import org.apache.atlas.model.instance.EntityMutationResponse;
 import org.apache.atlas.repository.graph.AtlasGraphProvider;
 import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
@@ -42,6 +43,7 @@ import org.testng.annotations.*;
 import javax.inject.Inject;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
@@ -929,6 +931,109 @@ public class AtlasDiscoveryServiceTest extends BasicTestSetup {
         Assert.assertTrue(list.get(0).getDisplayText().equalsIgnoreCase("time_id"));
     }
 
+    //test excludeHeaderAttributes
+    @Test
+    public void excludeHeaderAttributesStringAttr() throws AtlasBaseException {
+        SearchParameters params = new SearchParameters();
+        params.setTypeName(HIVE_TABLE_TYPE);
+        params.setExcludeHeaderAttributes(true);
+        SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", Operator.EQ, "Managed");
+        params.setEntityFilters(filterCriteria);
+        params.setSortBy("name");
+        params.setAttributes(new HashSet<String>() {{ add("name");}});
+        params.setLimit(1);
+
+        AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
+        AtlasSearchResult.AttributeSearchResult expected = new AtlasSearchResult.AttributeSearchResult();
+        expected.setName(Arrays.asList("name"));
+        expected.setValues(Arrays.asList(Arrays.asList("log_fact_daily_mv")));
+        assertSearchResult(searchResult,expected);
+    }
+
+    @Test
+    public void excludeHeaderAttributesRelationAttr() throws AtlasBaseException {
+        SearchParameters params = new SearchParameters();
+        params.setTypeName(HIVE_TABLE_TYPE);
+        params.setExcludeHeaderAttributes(true);
+        SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("name", Operator.EQ, "time_dim");
+        params.setEntityFilters(filterCriteria);
+        params.setAttributes(new HashSet<String>() {{ add("name"); add("db");}});
+        params.setLimit(1);
+
+        AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
+
+        assertNotNull(searchResult);
+        assertNotNull(searchResult.getEntities());
+        assertNotNull(searchResult.getReferredEntities());
+    }
+
+    @Test
+    public void excludeHeaderAttributesSystemAttr() throws AtlasBaseException {
+        SearchParameters params = new SearchParameters();
+        params.setTypeName(HIVE_TABLE_TYPE);
+        params.setExcludeHeaderAttributes(true);
+        params.setAttributes(new HashSet<String>() {{ add("name"); add("__state");}});
+        params.setLimit(1);
+        SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", Operator.EQ, "Managed");
+        params.setEntityFilters(filterCriteria);
+        params.setSortBy("name");
+
+        AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
+        AtlasSearchResult.AttributeSearchResult expected = new AtlasSearchResult.AttributeSearchResult();
+        expected.setName(Arrays.asList("name","__state"));
+        expected.setValues(Arrays.asList(Arrays.asList("log_fact_daily_mv","ACTIVE")));
+        assertSearchResult(searchResult,expected);
+    }
+
+    @Test
+    public void excludeHeaderAttributesAllEntityTypeSysAttr() throws AtlasBaseException {
+        SearchParameters params = new SearchParameters();
+        params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
+        params.setExcludeHeaderAttributes(true);
+        params.setAttributes(new HashSet<String>() {{ add("__state");}});
+        params.setLimit(2);
+
+        AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
+        AtlasSearchResult.AttributeSearchResult expected = new AtlasSearchResult.AttributeSearchResult();
+        expected.setName(Arrays.asList("__state"));
+        expected.setValues(Arrays.asList(Arrays.asList("ACTIVE"), Arrays.asList("ACTIVE")));
+        assertSearchResult(searchResult,expected);
+    }
+    @Test
+    public void excludeHeaderAttributesAllEntityTypeSysAttrs() throws AtlasBaseException {
+        SearchParameters params = new SearchParameters();
+        params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
+        params.setExcludeHeaderAttributes(true);
+        params.setAttributes(new HashSet<String>() {{ add("__state"); add("__guid");}});
+        params.setLimit(2);
+
+        AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
+        assertEquals(searchResult.getAttributes().getValues().size(), 2);
+    }
+
+    @Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute name not found for type __ENTITY_ROOT")
+    public void excludeHeaderAttributesAllEntityType() throws AtlasBaseException {
+        SearchParameters params = new SearchParameters();
+        params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
+        params.setExcludeHeaderAttributes(true);
+        params.setAttributes(new HashSet<String>() {{ add("name");}});
+        params.setLimit(1);
+
+        discoveryService.searchWithParameters(params);
+    }
+
+    @Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute name1 not found for type hive_table")
+    public void excludeHeaderAttributesInvalidAttr() throws AtlasBaseException {
+        SearchParameters params = new SearchParameters();
+        params.setTypeName(HIVE_TABLE_TYPE);
+        params.setExcludeHeaderAttributes(true);
+        params.setAttributes(new HashSet<String>() {{ add("name1");}});
+        params.setLimit(1);
+
+        discoveryService.searchWithParameters(params);
+    }
+
+
     private String gethiveTableSalesFactGuid() throws AtlasBaseException {
         if (salesFactGuid == null) {
             SearchParameters params = new SearchParameters();
@@ -958,6 +1063,18 @@ public class AtlasDiscoveryServiceTest extends BasicTestSetup {
         }
     }
 
+    private void assertSearchResult(AtlasSearchResult searchResult, AtlasSearchResult.AttributeSearchResult expected) {
+        assertNotNull(searchResult);
+        AtlasSearchResult.AttributeSearchResult result = searchResult.getAttributes();
+        assertNotNull(result);
+        assertTrue(result.getName().containsAll(expected.getName()));
+        int i = 0;
+        for (List<Object> value : result.getValues()) {
+            assertTrue(value.containsAll(expected.getValues().get(i)));
+            i++;
+        }
+    }
+
     private void assertAggregationMetrics(AtlasQuickSearchResult searchResult) {
         Map<String, List<AtlasAggregationEntry>> agg =  searchResult.getAggregationMetrics();
         Assert.assertTrue(CollectionUtils.isNotEmpty(agg.get("__typeName")));
index dc52d33d1a9d75346071fbb894744dec7be9addb..f65e2bae303493c513fa06d11f74f8c5a4dc0869 100644 (file)
@@ -10,7 +10,7 @@
       "offset": 0,
       "entityFilters": null,
       "tagFilters": null,
-      "attributes": [""]
+      "attributes": []
     },
     "expectedCount": 1
   },
@@ -25,7 +25,7 @@
       "offset": 0,
       "entityFilters": null,
       "tagFilters": null,
-      "attributes": [""]
+      "attributes": []
     },
     "expectedCount": 0
   },
         ]
       },
       "tagFilters": null,
-      "attributes": [""]
+      "attributes": []
     },
     "expectedCount": 1
   },
       "offset": 0,
       "entityFilters": null,
       "tagFilters": null,
-      "attributes": [""]
+      "attributes": []
     },
     "expectedCount": 1
   },
       "offset": 0,
       "entityFilters": null,
       "tagFilters": null,
-      "attributes": [""]
+      "attributes": []
     },
     "expectedCount": 3
   },
         "attributeValue" : "+"
       },
       "tagFilters": null,
-      "attributes": [""]
+      "attributes": []
     },
     "expectedCount": 1
   }
index 93d2d7d3649b9a52a59dc7398894d5a526eb42c5..255ed8adb69c192ec88bcb47162187c49686ea7d 100644 (file)
@@ -15,7 +15,6 @@
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3
@@ -38,7 +37,6 @@
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3
@@ -59,7 +57,6 @@
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 2
@@ -81,7 +78,6 @@
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 1
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 1
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 18
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3
       },
       "tagFilters": null,
       "attributes": [
-        ""
       ]
     },
     "expectedCount": 3