in hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java [446:627]
public static void initOptions(HugeConfig conf,
DBOptionsInterface<?> db,
MutableDBOptionsInterface<?> mdb,
ColumnFamilyOptionsInterface<?> cf,
MutableColumnFamilyOptionsInterface<?> mcf) {
final boolean optimize = conf.get(RocksDBOptions.OPTIMIZE_MODE);
if (db != null) {
/*
* Set true then the database will be created if it is missing.
* should we use options.setCreateMissingColumnFamilies()?
*/
db.setCreateIfMissing(true);
db.setWriteDbidToManifest(true);
db.setAvoidUnnecessaryBlockingIO(true);
// Optimize RocksDB
if (optimize) {
int processors = CoreOptions.CPUS;
db.setIncreaseParallelism(Math.max(processors / 2, 1));
db.setAllowConcurrentMemtableWrite(true);
db.setEnableWriteThreadAdaptiveYield(true);
}
db.setInfoLogLevel(InfoLogLevel.valueOf(
conf.get(RocksDBOptions.LOG_LEVEL) + "_LEVEL"));
db.setMaxSubcompactions(conf.get(RocksDBOptions.MAX_SUB_COMPACTIONS));
db.setAllowMmapWrites(conf.get(RocksDBOptions.ALLOW_MMAP_WRITES));
db.setAllowMmapReads(conf.get(RocksDBOptions.ALLOW_MMAP_READS));
db.setUseDirectReads(conf.get(RocksDBOptions.USE_DIRECT_READS));
db.setUseDirectIoForFlushAndCompaction(
conf.get(RocksDBOptions.USE_DIRECT_READS_WRITES_FC));
db.setUseFsync(conf.get(RocksDBOptions.USE_FSYNC));
db.setAtomicFlush(conf.get(RocksDBOptions.ATOMIC_FLUSH));
db.setMaxManifestFileSize(conf.get(RocksDBOptions.MAX_MANIFEST_FILE_SIZE));
db.setSkipStatsUpdateOnDbOpen(conf.get(RocksDBOptions.SKIP_STATS_UPDATE_ON_DB_OPEN));
db.setSkipCheckingSstFileSizesOnDbOpen(
conf.get(RocksDBOptions.SKIP_CHECK_SIZE_ON_DB_OPEN));
db.setMaxFileOpeningThreads(conf.get(RocksDBOptions.MAX_FILE_OPENING_THREADS));
db.setDbWriteBufferSize(conf.get(RocksDBOptions.DB_MEMTABLE_SIZE));
db.setLogReadaheadSize(conf.get(RocksDBOptions.LOG_READAHEAD_SIZE));
// A global cache for table-level rows
long cacheCapacity = conf.get(RocksDBOptions.ROW_CACHE_CAPACITY);
if (cacheCapacity > 0) {
db.setRowCache(new LRUCache(cacheCapacity));
}
}
if (mdb != null) {
/*
* Migrate to max_background_jobs option
* https://github.com/facebook/rocksdb/wiki/Thread-Pool
* https://github.com/facebook/rocksdb/pull/2205/files
*/
mdb.setMaxBackgroundJobs(conf.get(RocksDBOptions.MAX_BG_JOBS));
mdb.setDelayedWriteRate(conf.get(RocksDBOptions.DELAYED_WRITE_RATE));
mdb.setMaxOpenFiles(conf.get(RocksDBOptions.MAX_OPEN_FILES));
mdb.setMaxTotalWalSize(conf.get(RocksDBOptions.MAX_TOTAL_WAL_SIZE));
mdb.setBytesPerSync(conf.get(RocksDBOptions.BYTES_PER_SYNC));
mdb.setWalBytesPerSync(conf.get(RocksDBOptions.WAL_BYTES_PER_SYNC));
mdb.setStrictBytesPerSync(conf.get(RocksDBOptions.STRICT_BYTES_PER_SYNC));
mdb.setCompactionReadaheadSize(conf.get(RocksDBOptions.COMPACTION_READAHEAD_SIZE));
mdb.setDeleteObsoleteFilesPeriodMicros(
1000000 * conf.get(RocksDBOptions.DELETE_OBSOLETE_FILE_PERIOD));
}
if (cf != null) {
if (optimize) {
// Optimize RocksDB
cf.optimizeLevelStyleCompaction();
cf.optimizeUniversalStyleCompaction();
}
int numLevels = conf.get(RocksDBOptions.NUM_LEVELS);
List<CompressionType> compressions = conf.get(RocksDBOptions.LEVELS_COMPRESSIONS);
E.checkArgument(compressions.isEmpty() || compressions.size() == numLevels,
"Elements number of '%s' must be 0 or " +
"be the same as '%s', but got %s != %s",
RocksDBOptions.LEVELS_COMPRESSIONS.name(),
RocksDBOptions.NUM_LEVELS.name(), compressions.size(), numLevels);
cf.setNumLevels(numLevels);
cf.setCompactionStyle(conf.get(RocksDBOptions.COMPACTION_STYLE));
cf.setBottommostCompressionType(conf.get(RocksDBOptions.BOTTOMMOST_COMPRESSION));
if (!compressions.isEmpty()) {
cf.setCompressionPerLevel(compressions);
}
cf.setMinWriteBufferNumberToMerge(conf.get(RocksDBOptions.MIN_MEMTABLES_TO_MERGE));
cf.setMaxWriteBufferNumberToMaintain(
conf.get(RocksDBOptions.MAX_MEMTABLES_TO_MAINTAIN));
cf.setInplaceUpdateSupport(conf.get(RocksDBOptions.MEMTABLE_INPLACE_UPDATE_SUPPORT));
cf.setLevelCompactionDynamicLevelBytes(conf.get(RocksDBOptions.DYNAMIC_LEVEL_BYTES));
cf.setOptimizeFiltersForHits(conf.get(RocksDBOptions.BLOOM_FILTERS_SKIP_LAST_LEVEL));
cf.setTableFormatConfig(initTableConfig(conf));
// CappedPrefixExtractor uses the first N bytes
int prefixLength = conf.get(RocksDBOptions.PREFIX_EXTRACTOR_CAPPED);
if (prefixLength > 0) {
cf.useCappedPrefixExtractor(prefixLength);
}
// https://github.com/facebook/rocksdb/tree/master/utilities/merge_operators
cf.setMergeOperatorName("uint64add"); // uint64add/stringappend
}
if (mcf != null) {
mcf.setCompressionType(conf.get(RocksDBOptions.COMPRESSION));
mcf.setWriteBufferSize(conf.get(RocksDBOptions.MEMTABLE_SIZE));
mcf.setMaxWriteBufferNumber(conf.get(RocksDBOptions.MAX_MEMTABLES));
mcf.setMaxBytesForLevelBase(conf.get(RocksDBOptions.MAX_LEVEL1_BYTES));
mcf.setMaxBytesForLevelMultiplier(conf.get(RocksDBOptions.MAX_LEVEL_BYTES_MULTIPLIER));
mcf.setTargetFileSizeBase(conf.get(RocksDBOptions.TARGET_FILE_SIZE_BASE));
mcf.setTargetFileSizeMultiplier(conf.get(RocksDBOptions.TARGET_FILE_SIZE_MULTIPLIER));
mcf.setLevel0FileNumCompactionTrigger(
conf.get(RocksDBOptions.LEVEL0_COMPACTION_TRIGGER));
mcf.setLevel0SlowdownWritesTrigger(
conf.get(RocksDBOptions.LEVEL0_SLOWDOWN_WRITES_TRIGGER));
mcf.setLevel0StopWritesTrigger(conf.get(RocksDBOptions.LEVEL0_STOP_WRITES_TRIGGER));
mcf.setSoftPendingCompactionBytesLimit(
conf.get(RocksDBOptions.SOFT_PENDING_COMPACTION_LIMIT));
mcf.setHardPendingCompactionBytesLimit(
conf.get(RocksDBOptions.HARD_PENDING_COMPACTION_LIMIT));
/*
* TODO: also set memtable options:
* memtable_insert_with_hint_prefix_extractor
* The reason why use option name `memtable_bloom_size_ratio`:
* https://github.com/facebook/rocksdb/pull/9453/files
* #diff-cde52d1fcbcce2bc6aae27838f1d3e7e9e469ccad8aaf8f2695f939e279d7501R369
*/
mcf.setMemtablePrefixBloomSizeRatio(
conf.get(RocksDBOptions.MEMTABLE_BLOOM_SIZE_RATIO));
mcf.setMemtableWholeKeyFiltering(
conf.get(RocksDBOptions.MEMTABLE_BLOOM_WHOLE_KEY_FILTERING));
mcf.setMemtableHugePageSize(conf.get(RocksDBOptions.MEMTABL_BLOOM_HUGE_PAGE_SIZE));
boolean bulkload = conf.get(RocksDBOptions.BULKLOAD_MODE);
if (bulkload) {
// Disable automatic compaction
mcf.setDisableAutoCompactions(true);
int trigger = Integer.MAX_VALUE;
mcf.setLevel0FileNumCompactionTrigger(trigger);
mcf.setLevel0SlowdownWritesTrigger(trigger);
mcf.setLevel0StopWritesTrigger(trigger);
long limit = Long.MAX_VALUE;
mcf.setSoftPendingCompactionBytesLimit(limit);
mcf.setHardPendingCompactionBytesLimit(limit);
//cf.setMemTableConfig(new VectorMemTableConfig());
}
}
}