IGNITE-10205 Add to utility command - ./control.sh --cache idle_verify --dump ability...
authorvd_pyatkov <vldpyatkov@gmail.com>
Fri, 11 Jan 2019 08:58:48 +0000 (11:58 +0300)
committerIvan Rakov <irakov@apache.org>
Fri, 11 Jan 2019 09:14:34 +0000 (12:14 +0300)
Signed-off-by: Ivan Rakov <irakov@apache.org>
modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java

index fbc8819..bf596e7 100644 (file)
@@ -197,9 +197,16 @@ public class CommandHandler {
     /** */
     private static final String CMD_SKIP_ZEROS = "--skip-zeros";
 
+    /** Command exclude caches. */
+    private static final String CMD_EXCLUDE_CACHES = "--excludeCaches";
+
     /** Cache filter. */
     private static final String CACHE_FILTER = "--cache-filter";
 
+    /** One cache filter option should used message. */
+    public static final String ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG = "Should use only one of option: " +
+        CMD_EXCLUDE_CACHES + ", " + CACHE_FILTER + " or pass caches explicitly";
+
     /** */
     private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
 
@@ -859,7 +866,10 @@ public class CommandHandler {
 
         usageCache(LIST, "regexPattern", op(or("groups", "seq")), OP_NODE_ID, op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE));
         usageCache(CONTENTION, "minQueueSize", OP_NODE_ID, op("maxPrint"));
-        usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]", op(CACHE_FILTER, or(CacheFilterEnum.values())));
+        usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), op(or(CMD_EXCLUDE_CACHES + " cache1,...,cacheN",
+            op(CACHE_FILTER, or(CacheFilterEnum.ALL.toString(), CacheFilterEnum.SYSTEM.toString(),
+                CacheFilterEnum.PERSISTENT.toString(), CacheFilterEnum.NOT_PERSISTENT.toString())), "cache1,...," +
+                "cacheN")));
         usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", OP_NODE_ID, op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K")));
         usageCache(DISTRIBUTION, or(NODE_ID, NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attrName1,...,attrNameN"));
         usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN");
@@ -1031,7 +1041,7 @@ public class CommandHandler {
      */
     private void legacyCacheIdleVerify(GridClient client, CacheArguments cacheArgs) throws GridClientException {
         VisorIdleVerifyTaskResult res = executeTask(
-            client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
+            client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches()));
 
         Map<PartitionKey, List<PartitionHashRecord>> conflicts = res.getConflicts();
 
@@ -1180,7 +1190,8 @@ public class CommandHandler {
         String path = executeTask(
             client,
             VisorIdleVerifyDumpTask.class,
-            new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros(), cacheArgs.getCacheFilterEnum())
+            new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches(), cacheArgs.isSkipZeros(), cacheArgs
+                .getCacheFilterEnum())
         );
 
         log("VisorIdleVerifyDumpTask successfully written output to '" + path + "'");
@@ -1192,7 +1203,7 @@ public class CommandHandler {
      */
     private void cacheIdleVerifyV2(GridClient client, CacheArguments cacheArgs) throws GridClientException {
         IdleVerifyResultV2 res = executeTask(
-            client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
+            client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches()));
 
         res.print(System.out::print);
     }
@@ -2125,16 +2136,30 @@ public class CommandHandler {
 
                     if (CMD_DUMP.equals(nextArg))
                         cacheArgs.dump(true);
+                    else if (CMD_EXCLUDE_CACHES.equals(nextArg)) {
+                        if (cacheArgs.caches() != null || cacheArgs.getCacheFilterEnum() != CacheFilterEnum.ALL)
+                            throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
+                        parseExcludeCacheNames(nextArg("Specify caches, which will be excluded."),
+                            cacheArgs);
+                    }
                     else if (CMD_SKIP_ZEROS.equals(nextArg))
                         cacheArgs.skipZeros(true);
                     else if (CACHE_FILTER.equals(nextArg)) {
+                        if (cacheArgs.caches() != null || cacheArgs.excludeCaches() != null)
+                            throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
                         String filter = nextArg("The cache filter should be specified. The following values can be " +
                             "used: " + Arrays.toString(CacheFilterEnum.values()) + '.');
 
                         cacheArgs.setCacheFilterEnum(CacheFilterEnum.valueOf(filter.toUpperCase()));
                     }
-                    else
+                    else {
+                        if (cacheArgs.excludeCaches() != null || cacheArgs.getCacheFilterEnum() != CacheFilterEnum.ALL)
+                            throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
                         parseCacheNames(nextArg, cacheArgs);
+                    }
                 }
                 break;
 
@@ -2313,6 +2338,24 @@ public class CommandHandler {
     }
 
     /**
+     * @param cacheNames Cache names arg.
+     * @param cacheArgs Cache args.
+     */
+    private void parseExcludeCacheNames(String cacheNames, CacheArguments cacheArgs) {
+        String[] cacheNamesArr = cacheNames.split(",");
+        Set<String> cacheNamesSet = new HashSet<>();
+
+        for (String cacheName : cacheNamesArr) {
+            if (F.isEmpty(cacheName))
+                throw new IllegalArgumentException("Non-empty cache names expected.");
+
+            cacheNamesSet.add(cacheName.trim());
+        }
+
+        cacheArgs.excludeCaches(cacheNamesSet);
+    }
+
+    /**
      * Get ping param for grid client.
      *
      * @param nextArgErr Argument extraction error message.
index 9372391..fb5bc88 100644 (file)
@@ -33,6 +33,9 @@ public class CacheArguments {
     /** Caches. */
     private Set<String> caches;
 
+    /** Exclude caches or groups. */
+    private Set<String> excludeCaches;
+
     /** Partition id. */
     private int partId;
 
@@ -142,6 +145,20 @@ public class CacheArguments {
     }
 
     /**
+     * @return Exclude caches or groups.
+     */
+    public Set<String> excludeCaches() {
+        return excludeCaches;
+    }
+
+    /**
+     * @param excludeCaches Excluse caches or groups.
+     */
+    public void excludeCaches(Set<String> excludeCaches) {
+        this.excludeCaches = excludeCaches;
+    }
+
+    /**
      * @return Partition id.
      */
     public int partitionId() {
index 5e872eb..c30d37b 100644 (file)
@@ -50,6 +50,7 @@ import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheUtils;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -205,9 +206,9 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
         @Override public Map<PartitionKeyV2, PartitionHashRecordV2> execute() throws IgniteException {
             Set<Integer> grpIds = new HashSet<>();
 
-            Set<String> missingCaches = new HashSet<>();
+            if (arg.getCaches() != null && !arg.getCaches().isEmpty()) {
+                Set<String> missingCaches = new HashSet<>();
 
-            if (arg.getCaches() != null) {
                 for (String cacheName : arg.getCaches()) {
                     DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
 
@@ -229,14 +230,8 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
                         grpIds.add(desc.groupId());
                 }
             }
-            else {
-                Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
-
-                for (CacheGroupContext grp : groups) {
-                    if (!grp.systemCache() && !grp.isLocal())
-                        grpIds.add(grp.groupId());
-                }
-            }
+            else
+                grpIds = getCacheGroupIds();
 
             List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFutures = new ArrayList<>();
 
@@ -293,6 +288,45 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
         }
 
         /**
+         * Gets filtered group ids.
+         */
+        private Set<Integer> getCacheGroupIds() {
+            Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
+
+            Set<Integer> grpIds = new HashSet<>();
+
+            if (arg.excludeCaches() == null || arg.excludeCaches().isEmpty()) {
+                for (CacheGroupContext grp : groups) {
+                    if (!grp.systemCache() && !grp.isLocal())
+                        grpIds.add(grp.groupId());
+                }
+                return grpIds;
+            }
+
+            for (CacheGroupContext grp : groups) {
+                if (!grp.systemCache() && !grp.isLocal() && !isGrpExcluded(grp))
+                    grpIds.add(grp.groupId());
+            }
+
+            return grpIds;
+        }
+
+        /**
+         * @param grp Group.
+         */
+        private boolean isGrpExcluded(CacheGroupContext grp) {
+            if (arg.excludeCaches().contains(grp.name()))
+                return true;
+
+            for (GridCacheContext cacheCtx : grp.caches()) {
+                if (arg.excludeCaches().contains(cacheCtx.name()))
+                    return true;
+            }
+
+            return false;
+        }
+
+        /**
          *  Checks and throw exception if caches was missed.
          *
          * @param missingCaches Missing caches.
@@ -380,7 +414,6 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
             });
         }
 
-
         /**
          * @param grpCtx Group context.
          * @param part Local partition.
index 29dfb5b..3c836a4 100644 (file)
@@ -45,11 +45,12 @@ public class VisorIdleVerifyDumpTaskArg extends VisorIdleVerifyTaskArg {
 
     /**
      * @param caches Caches.
+     * @param excludeCaches Caches to exclude.
      * @param skipZeros Skip zeros partitions.
      * @param cacheFilterEnum Cache kind.
      */
-    public VisorIdleVerifyDumpTaskArg(Set<String> caches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) {
-        super(caches);
+    public VisorIdleVerifyDumpTaskArg(Set<String> caches, Set<String> excludeCaches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) {
+        super(caches, excludeCaches);
         this.skipZeros = skipZeros;
         this.cacheFilterEnum = cacheFilterEnum;
     }
index c82af58..d645fec 100644 (file)
@@ -35,6 +35,9 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
     /** Caches. */
     private Set<String> caches;
 
+    /** Exclude caches or groups. */
+    private Set<String> excludeCaches;
+
     /**
      * Default constructor.
      */
@@ -44,9 +47,19 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
 
     /**
      * @param caches Caches.
+     * @param excludeCaches Exclude caches or group.
+     */
+    public VisorIdleVerifyTaskArg(Set<String> caches, Set<String> excludeCaches) {
+        this.caches = caches;
+        this.excludeCaches = excludeCaches;
+    }
+
+    /**
+     * @param caches Caches.
      */
     public VisorIdleVerifyTaskArg(Set<String> caches) {
         this.caches = caches;
+        this.excludeCaches = excludeCaches;
     }
 
 
@@ -57,14 +70,30 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
         return caches;
     }
 
+    /**
+     * @return Exclude caches or groups.
+     */
+    public Set<String> excludeCaches() {
+        return excludeCaches;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeCollection(out, caches);
+        U.writeCollection(out, excludeCaches);
     }
 
     /** {@inheritDoc} */
     @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
         caches = U.readSet(in);
+
+        if (protoVer >= V2)
+            excludeCaches = U.readSet(in);
     }
 
     /** {@inheritDoc} */
index 57ae628..e6a3949 100644 (file)
@@ -1357,6 +1357,94 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
     }
 
     /**
+     * Tests that idle verify print partitions info with exclude cache group.
+     *
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testCacheIdleVerifyDumpExcludedCacheGrp() throws Exception {
+        IgniteEx ignite = (IgniteEx)startGrids(3);
+
+        ignite.cluster().active(true);
+
+        int parts = 32;
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setGroupName("shared_grp")
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME));
+
+        IgniteCache<Object, Object> secondCache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setGroupName("shared_grp")
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME + "_second"));
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--excludeCaches", "shared_grp"));
+
+        Matcher fileNameMatcher = dumpFileNameMatcher();
+
+        if (fileNameMatcher.find()) {
+            String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+            assertTrue(dumpWithConflicts.contains("idle_verify check has finished, found 0 partitions"));
+        }
+        else
+            fail("Should be found dump with conflicts");
+    }
+
+    /**
+     * Tests that idle verify print partitions info with exclude caches.
+     *
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testCacheIdleVerifyDumpExcludedCaches() throws Exception {
+        IgniteEx ignite = (IgniteEx)startGrids(3);
+
+        ignite.cluster().active(true);
+
+        int parts = 32;
+
+        ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setGroupName("shared_grp")
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME));
+
+        ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setGroupName("shared_grp")
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME + "_second"));
+
+        ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME + "_third"));
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--excludeCaches", DEFAULT_CACHE_NAME
+            + "," + DEFAULT_CACHE_NAME + "_second"));
+
+        Matcher fileNameMatcher = dumpFileNameMatcher();
+
+        if (fileNameMatcher.find()) {
+            String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+            assertTrue(dumpWithConflicts.contains("idle_verify check has finished, found 32 partitions"));
+            assertTrue(dumpWithConflicts.contains("default_third"));
+            assertTrue(!dumpWithConflicts.contains("shared_grp"));
+        }
+        else
+            fail("Should be found dump with conflicts");
+    }
+
+    /**
      * @return Build matcher for dump file name.
      */
     @NotNull private Matcher dumpFileNameMatcher() {