Add store/sort module (#37)
authorShouJing <guoshoujing@baidu.com>
Mon, 31 May 2021 06:51:32 +0000 (14:51 +0800)
committerGitHub <noreply@github.com>
Mon, 31 May 2021 06:51:32 +0000 (14:51 +0800)
commitdf5c2bb588549aa153be83157414d896f30638cd
tree546476bc550836d84f64d3bb765e6c3ac94149b9
parentbfb8826fca5e5f2eada4761d5f9a6ac93dc84e92
Add store/sort module (#37)

* add hgkvFile/hgkvDir module
* add EntriesUtil.java
* add SorterTest.java
* move file verification to HgkvDir
* rebuild HgkvFile readFooter and Fix HgkvDir#open
* add sort module
* implement InnerSortFlusher and OuterSortFlusher
* add KvEntry#numSubEntries
88 files changed:
computer-api/src/main/java/com/baidu/hugegraph/computer/core/util/BytesUtil.java
computer-core/src/main/java/com/baidu/hugegraph/computer/core/config/ComputerOptions.java
computer-core/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileInput.java
computer-core/src/main/java/com/baidu/hugegraph/computer/core/io/UnsafeBytesOutput.java
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/Sorter.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/SorterImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/CombinableSorterFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/CombineKvInnerSortFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/CombineKvOuterSortFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/CombineSubKvInnerSortFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/CombineSubKvOuterSortFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/InnerSortFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/KvInnerSortFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/KvOuterSortFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/OuterSortFlusher.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/PeekableIterator.java [moved from computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/Pointer.java with 82% similarity]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/flusher/PeekableIteratorAdaptor.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/merge/HgkvDirMerger.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/merge/HgkvDirMergerImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/InputSorter.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/InputsSorter.java [moved from computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/SortCombiner.java with 73% similarity]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/InputsSorterImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/JavaInputSorter.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/SubKvSorter.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/LoserTreeInputsSorting.java
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortingFactory.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortingMode.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/Sorter.java [deleted file]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/buffer/EntriesInput.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/buffer/EntriesSubKvInput.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/buffer/EntryIterator.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/AbstractKvEntry.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/CachedPointer.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/DefaultKvEntry.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/EntriesUtil.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/EntryOutput.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/EntryOutputImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/InlinePointer.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/InputToEntries.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/KvEntry.java [moved from computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/HgkvFile.java with 73% similarity]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/KvEntryWithFirstSubKv.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/KvEntryWriter.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/KvEntryWriterImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/Pointer.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/Range.java [moved from computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/Range.java with 93% similarity]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/AbstractHgkvFile.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/HgkvDir.java [moved from computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/KvEntry.java with 66% similarity]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/HgkvDirImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/HgkvFile.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/HgkvFileImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/builder/BlockBuilder.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/builder/DataBlockBuilderImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/builder/HgkvDirBuilder.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/builder/HgkvDirBuilderImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/builder/HgkvFileBuilder.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/builder/HgkvFileBuilderImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/builder/IndexBlockBuilder.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/builder/IndexBlockBuilderImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/reader/HgkvDir4SubKvReaderImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/reader/HgkvDirReader.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/reader/HgkvDirReaderImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/reader/HgkvFileReader.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/reader/HgkvFileReaderImpl.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/select/DefaultSelectedFiles.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/select/DisperseEvenlySelector.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/select/InputFilesSelector.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/file/select/SelectedFiles.java [new file with mode: 0644]
computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/IdValueUtil.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/UnitTestSuite.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileTest.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/io/StructRandomAccessOutputTest.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/io/UnsafeBytesTest.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/SorterTestUtil.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/combiner/MockIntSumCombiner.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/EntriesUtilTest.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/FlusherTest.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/SortLargeDataTest.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/SorterTest.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/sorter/SorterTestSuite.java [moved from computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/SortTestSuite.java with 83% similarity]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/InputsSortingTest.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortingTestSuite.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/TestData.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/store/EntryOutputTest.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/store/HgkvDirTest.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/store/HgkvFileTest.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/store/PointerTest.java [new file with mode: 0644]
computer-test/src/main/java/com/baidu/hugegraph/computer/core/store/StoreTestSuite.java
computer-test/src/main/java/com/baidu/hugegraph/computer/core/store/StoreTestUtil.java [new file with mode: 0644]