o changed the reclaimer threshold to 70
authorKiran Ayyagari <kayyagari@apache.org>
Wed, 13 May 2015 08:47:00 +0000 (08:47 +0000)
committerKiran Ayyagari <kayyagari@apache.org>
Wed, 13 May 2015 08:47:00 +0000 (08:47 +0000)
o updated space reclaimer to clean the BTree of BTrees
o added a multi threaded test

mavibot/src/main/java/org/apache/directory/mavibot/btree/RecordManager.java
mavibot/src/main/java/org/apache/directory/mavibot/btree/SpaceReclaimer.java
mavibot/src/test/java/org/apache/directory/mavibot/btree/SpaceReclaimerTest.java

index d960174..598d9eb 100644 (file)
@@ -162,7 +162,7 @@ public class RecordManager extends AbstractTransactionManager
     public static final boolean NORMAL_BTREE = false;
 
     /** The B-tree of B-trees */
-    private BTree<NameRevision, Long> btreeOfBtrees;
+    /* no qualifier */ BTree<NameRevision, Long> btreeOfBtrees;
 
     /** The B-tree of B-trees management btree name */
     /* no qualifier */static final String BTREE_OF_BTREES_NAME = "_btree_of_btrees_";
@@ -216,7 +216,9 @@ public class RecordManager extends AbstractTransactionManager
     private int commitCount = 0;
 
     /** the threshold at which the SpaceReclaimer will be run to free the copied pages */
-    private int spaceReclaimerThreshold = 200;
+    // FIXME the below value is derived after seeing that anything higher than that
+    // is resulting in a "This thread does not hold the transactionLock" error
+    private int spaceReclaimerThreshold = 70;
 
     public Map<Long, Integer> writeCounter = new HashMap<Long, Integer>();
 
index 43e845e..fc5c7d2 100644 (file)
@@ -72,14 +72,14 @@ public class SpaceReclaimer
             
             running = true;
             
-            rm.beginTransaction();
-            
             Set<String> managed = rm.getManagedTrees();
 
             for ( String name : managed )
             {
                 PersistedBTree tree = ( PersistedBTree ) rm.getManagedTree( name );
 
+                long latestRev = tree.getRevision();
+                
                 Set<Long> inUseRevisions = new TreeSet<Long>();
                 
                 // the tree might have been removed
@@ -94,6 +94,13 @@ public class SpaceReclaimer
 
                 List<RevisionOffset> copiedRevisions = getRevisions( name );
 
+                // the revision last removed from copiedPage BTree
+                long lastRemovedRev = -1;
+
+                // FIXME an additional txn needs to be started to safeguard the copiedPage BTree changes
+                // no clue yet on why this is needed 
+                rm.beginTransaction();
+                
                 for ( RevisionOffset ro : copiedRevisions )
                 {
                     long rv = ro.getRevision();
@@ -112,11 +119,30 @@ public class SpaceReclaimer
                     RevisionName key = new RevisionName( rv, name );
                     
                     rm.copiedPageBtree.delete( key );
+                    lastRemovedRev = rv;
+                }
+
+                rm.commit();
+                
+                // no new txn is needed for the operations on BoB
+                if ( lastRemovedRev != -1 )
+                {
+                    // we SHOULD NOT delete the latest revision from BoB
+                    NameRevision nr = new NameRevision( name, latestRev );
+                    TupleCursor<NameRevision, Long> cursor = rm.btreeOfBtrees.browseFrom( nr );
+                    
+                    while ( cursor.hasPrev() )
+                    {
+                        Tuple<NameRevision, Long> t = cursor.prev();
+                        //System.out.println( "deleting BoB rev " + t.getKey()  + " latest rev " + latestRev );
+                        rm.btreeOfBtrees.delete( t.getKey() );
+                    }
+
+                    cursor.close();
                 }
             }
 
             running = false;
-            rm.commit();
         }
         catch ( Exception e )
         {
index 3db277a..c812924 100644 (file)
 package org.apache.directory.mavibot.btree;\r
 \r
 \r
+import static org.junit.Assert.assertEquals;\r
+\r
 import java.io.File;\r
+import java.util.Map;\r
+import java.util.Random;\r
+import java.util.concurrent.ConcurrentHashMap;\r
+import java.util.concurrent.CountDownLatch;\r
 \r
 import org.apache.directory.mavibot.btree.serializer.IntSerializer;\r
 import org.apache.directory.mavibot.btree.serializer.StringSerializer;\r
@@ -29,7 +35,6 @@ import org.junit.Before;
 import org.junit.Rule;\r
 import org.junit.Test;\r
 import org.junit.rules.TemporaryFolder;\r
-import static org.junit.Assert.*;\r
 \r
 /**\r
  * Tests for free page reclaimer.\r
@@ -147,4 +152,85 @@ public class SpaceReclaimerTest
         \r
         assertEquals( count, total );\r
     }\r
+\r
+    \r
+    /**\r
+     * Test reclaimer functionality while multiple threads writing to the same BTree\r
+     * \r
+     * @throws Exception\r
+     */\r
+    @Test\r
+    public void testReclaimerWithMultiThreads() throws Exception\r
+    {\r
+        final int numEntriesPerThread = 11;\r
+        final int numThreads = 5;\r
+        \r
+        final int total = numThreads * numEntriesPerThread;\r
+        \r
+        final Map<Integer, Integer> keyMap = new ConcurrentHashMap<Integer, Integer>();\r
+        \r
+        final Random rnd = new Random();\r
+        \r
+        final CountDownLatch latch = new CountDownLatch( numThreads );\r
+        \r
+        Runnable r = new Runnable()\r
+        {\r
+            @Override\r
+            public void run()\r
+            {\r
+                for ( int i=0; i < numEntriesPerThread; i++ )\r
+                {\r
+                    try\r
+                    {\r
+                        int key = rnd.nextInt( total );\r
+                        while( true )\r
+                        {\r
+                            if( !keyMap.containsKey( key ) )\r
+                            {\r
+                                keyMap.put( key, key );\r
+                                break;\r
+                            }\r
+                            \r
+                            //System.out.println( "duplicate " + key );\r
+                            key = rnd.nextInt( total );\r
+                        }\r
+                        \r
+                        uidTree.insert( key, String.valueOf( key ) );\r
+                    }\r
+                    catch( Exception e )\r
+                    {\r
+                        throw new RuntimeException(e);\r
+                    }\r
+                }\r
+                \r
+                latch.countDown();\r
+            }\r
+        };\r
+\r
+        for ( int i=0; i<numThreads; i++ )\r
+        {\r
+            Thread t = new Thread( r );\r
+            t.start();\r
+        }\r
+        \r
+        latch.await();\r
+        \r
+        System.out.println( "Total size before closing " + dbFile.length() );\r
+        closeAndReopenRM();\r
+        System.out.println( "Total size AFTER closing " + dbFile.length() );\r
+        \r
+        int count = 0;\r
+        TupleCursor<Integer, String> cursor = uidTree.browse();\r
+        while ( cursor.hasNext() )\r
+        {\r
+            Tuple<Integer, String> t = cursor.next();\r
+            assertEquals( t.key, Integer.valueOf( count ) );\r
+            count++;\r
+        }\r
+        \r
+        cursor.close();\r
+        \r
+        assertEquals( count, total );\r
+    }\r
+    \r
 }\r