Class MVStore

java.lang.Object

org.h2.mvstore.MVStore

All Implemented Interfaces:

AutoCloseable

public class MVStore
extends Object
implements AutoCloseable

A persistent storage for maps.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	MVStore.BackgroundWriterThread	A background writer thread to automatically store changes from time to time.
static final class	MVStore.Builder	A builder for an MVStore.
private static class	MVStore.RemovedPageInfo
static final class	MVStore.TxCounter	Class TxCounter is a simple data structure to hold version of the store along with the counter of open transactions, which are still operating on this version.

Field Summary

Fields

Modifier and Type	Field	Description
private int	autoCommitDelay	The delay in milliseconds to automatically commit and write changes.
private final int	autoCommitMemory
private final int	autoCompactFillRate
private long	autoCompactLastFileOpCount
final Thread.UncaughtExceptionHandler	backgroundExceptionHandler
private final AtomicReference<MVStore.BackgroundWriterThread>	backgroundWriterThread	Reference to a background thread, which is expected to be running, if any.
(package private) static final int	BLOCK_SIZE	The block size (physical sector size) of the disk.
private ThreadPoolExecutor	bufferSaveExecutor	Single-threaded executor for saving ByteBuffer as a new Chunk
private final CacheLongKeyLIRS<Page<?,?>>	cache	The page cache.
private final ConcurrentHashMap<Integer,Chunk>	chunks	The map of chunks.
private final CacheLongKeyLIRS<long[]>	chunksToC	Cache for chunks "Table of Content" used to translate page's sequential number within containing chunk into byte position within chunk's image.
private final int	compressionLevel	The compression level for new pages (0 for disabled, 1 for fast, 2 for high).
private Compressor	compressorFast
private Compressor	compressorHigh
private long	creationTime	The time the store was created, in milliseconds since 1970.
private long	currentStoreVersion	The version of the current store operation (if any).
private MVStore.TxCounter	currentTxCounter	Counter of open transactions for the latest (current) store version
private long	currentVersion
private final Deque<Chunk>	deadChunks
private final FileStore	fileStore
private final boolean	fileStoreIsProvided
private static final int	FORMAT_READ_MAX
private static final int	FORMAT_READ_MIN
private static final int	FORMAT_WRITE_MAX
private static final int	FORMAT_WRITE_MIN
private static final String	HDR_BLOCK
private static final String	HDR_BLOCK_SIZE
private static final String	HDR_CHUNK
private static final String	HDR_CLEAN
private static final String	HDR_CREATED
private static final String	HDR_FLETCHER
private static final String	HDR_FORMAT
private static final String	HDR_FORMAT_READ
private static final String	HDR_H
private static final String	HDR_VERSION
private final int	keysPerPage
private Chunk	lastChunk	The newest chunk.
private int	lastChunkId
private long	lastCommitTime
private final AtomicInteger	lastMapId
private long	lastTimeAbsolute
private final MVMap<String,String>	layout	The layout map.
private long	leafCount
private final ConcurrentHashMap<Integer, MVMap<?,?>>	maps
private final MVMap<String,String>	meta	The metadata map.
static final String	META_ID_KEY	The key for the entry within "layout" map, which contains id of "meta" map.
private boolean	metaChanged
private long	nonLeafCount
private final AtomicLong	oldestVersionToKeep	Oldest store version in use.
private final int	pageSplitSize
private MVStoreException	panicException
private static final int	PIPE_LENGTH
private final boolean	recoveryMode
private final Queue<MVStore.RemovedPageInfo>	removedPages
private int	retentionTime	How long to retain old, persisted chunks, in milliseconds.
private boolean	reuseSpace
private final ReentrantLock	saveChunkLock
private boolean	saveNeeded
private ThreadPoolExecutor	serializationExecutor	Single-threaded executor for serialization of the store snapshot into ByteBuffer
private final ReentrantLock	serializationLock
private int	state
private static final int	STATE_CLOSED	Store is closed.
private static final int	STATE_CLOSING	Store is closing now, and any operation on it may fail.
private static final int	STATE_OPEN	Store is open.
private static final int	STATE_STOPPING	Store is about to close now, but is still operational.
private final HashMap<String,Object>	storeHeader
private final ReentrantLock	storeLock	Lock which governs access to major store operations: store(), close(), ...
private int	unsavedMemory	The estimated memory used by unsaved pages.
private long	updateAttemptCounter
private long	updateCounter
private final Deque<MVStore.TxCounter>	versions	Ordered collection of all version usage counters for all versions starting from oldestVersionToKeep and up to current.
private int	versionsToKeep
private final Queue<WriteBuffer>	writeBufferPool

Constructor Summary

Constructors

Constructor	Description
MVStore(Map<String,Object> config)	Create and open the store.

Method Summary

Modifier and Type	Method	Description
private void	acceptChunkOccupancyChanges(long time, long version)	Apply the freed space to the chunk metadata.
(package private) void	accountForRemovedPage(long pos, long version, boolean pinned, int pageNo)	Remove a page.
(package private) void	beforeWrite(MVMap<?,?> map)	This method is called before writing to a map.
(package private) void	cachePage(Page<?,?> page)	Put the page in the cache.
private int	calculatePageNo(long pos)
private static boolean	canOverwriteChunk(Chunk c, long oldestVersionToKeep)
private void	checkOpen()
private void	clearCaches()
void	close()	Close the file and the store.
void	close(int allowedCompactionTime)	Close the file and the store.
void	closeImmediately()	Close the file and the store, without writing anything.
private void	closeStore(boolean normalShutdown, int allowedCompactionTime)
private ArrayList<Page<?,?>>	collectChangedMapRoots(long version)
long	commit()	Commit the changes.
private long	commit(Predicate<MVStore> check)
boolean	compact(int targetFillRate, int write)	Try to increase the fill rate by re-writing partially full chunks.
void	compactFile(int maxCompactTime)	Compact store file, that is, compact blocks that have a low fill rate, and move chunks next to each other.
void	compactMoveChunks()	Compact by moving all chunks next to each other.
(package private) boolean	compactMoveChunks(int targetFillRate, long moveSize)	Compact the store by moving all chunks next to each other, if there is free space between chunks.
private boolean	compactMoveChunks(long moveSize)
private void	compactMoveChunks(Iterable<Chunk> move)
private int	compactRewrite(Set<Integer> set)
private Chunk	createChunk(long time, long version)
private static HashSet<Integer>	createIdSet(Iterable<Chunk> toCompact)
private static ThreadPoolExecutor	createSingleThreadExecutor(String threadName)
(package private) void	deregisterMapRoot(int mapId)	Performs final stage of map removal - delete root location info from the layout table.
void	deregisterVersionUsage(MVStore.TxCounter txCounter)	De-register (close) completed operation (transaction).
private Chunk	discoverChunk(long block)	Discover a valid chunk, searching file backwards from the given block
private void	doMaintenance(int targetFillRate)
private int	dropUnusedChunks()
private void	dropUnusedVersions()
private Iterable<Chunk>	findChunksToMove(long startBlock, long moveSize)
private boolean	findLastChunkWithCompleteValidChunkSet(Chunk[] lastChunkCandidates, Map<Long,Chunk> validChunksByLocation, Map<Integer,Chunk> validChunksById, boolean afterFullScan)
private Iterable<Chunk>	findOldChunks(int writeLimit, int targetFillRate)
private void	freeChunkSpace(Chunk chunk)
private void	freeFileSpace(long start, int length)
private long	getAfterLastBlock()	Get the index of the first block after last occupied one.
int	getAutoCommitDelay()	Get the auto-commit delay.
int	getAutoCommitMemory()	Get the maximum memory (in bytes) used for unsaved pages.
CacheLongKeyLIRS<Page<?,?>>	getCache()	Get the cache.
int	getCacheHitRatio()
private static int	getCacheHitRatio(CacheLongKeyLIRS<?> cache)
int	getCacheSize()	Get the maximum cache size, in MB.
int	getCacheSizeUsed()	Get the amount of memory used for caching, in MB.
private Chunk	getChunk(long pos)	Get the chunk for the given position.
int	getChunkCount()	Get data chunks count.
private Chunk	getChunkForVersion(long version)
int	getChunksFillRate()	Get the current fill rate (percentage of used space in the file).
private int	getChunksFillRate(boolean all)
(package private) int	getCompressionLevel()
(package private) Compressor	getCompressorFast()
(package private) Compressor	getCompressorHigh()
long	getCurrentVersion()	Get the current version of the data.
private long	getFileLengthInUse()	Get the position right after the last used byte.
FileStore	getFileStore()	Get the file store.
int	getFillRate()
int	getKeysPerPage()
MVMap<String,String>	getLayoutMap()	Get this store's layout map.
private MVMap<String,String>	getLayoutMap(long version)
int	getLeafRatio()
int	getLivePageCount()	Get live data pages count.
<K,V> MVMap<K,V>	getMap(int id)	Get map by id.
private int	getMapId(String name)
String	getMapName(int id)	Get the name of the given map.
Set<String>	getMapNames()	Get the set of all map names.
long	getMaxPageSize()
MVMap<String,String>	getMetaMap()	Get the metadata map.
private int	getMovePriority(Chunk chunk)
(package private) long	getOldestVersionToKeep()	Get the oldest version to retain.
int	getPageCount()	Get data pages count.
int	getPageSplitSize()
MVStoreException	getPanicException()
private int	getProjectedFillRate(int thresholdChunkFillRate)
int	getRetentionTime()
boolean	getReuseSpace()
int	getRewritableChunksFillRate()
private long	getRootPos(int mapId)
Map<String,Object>	getStoreHeader()	Get the store header.
int	getStoreVersion()	Get the store version.
private int	getTargetFillRate()
private long	getTimeAbsolute()
private long	getTimeSinceCreation()
private long[]	getToC(Chunk chunk)
int	getTocCacheHitRatio()
int	getUnsavedMemory()	Get the estimated memory (in bytes) of unsaved data.
private MVStoreException	getUnsupportedWriteFormatException(long format, int expectedFormat, String s)
double	getUpdateFailureRatio()
long	getVersionsToKeep()	Get the oldest version to retain in memory (for in-memory stores).
private WriteBuffer	getWriteBuffer()	Get a buffer for writing.
private void	handleException(Throwable ex)
boolean	hasData(String name)	Check whether a given map exists and has data.
boolean	hasMap(String name)	Check whether a given map exists.
boolean	hasUnsavedChanges()	Check whether there are any unsaved changes.
boolean	isBackgroundThread()
boolean	isClosed()	Determine that store is open, or wait for it to be closed (by other thread)
private boolean	isIdle()
private boolean	isKnownVersion(long version)	Check whether all data can be read from this version.
private boolean	isOpen()
private boolean	isOpenOrStopping()
boolean	isReadOnly()	Whether the store is read-only.
private boolean	isRewritable(Chunk chunk, long time)
(package private) boolean	isSaveNeeded()
private boolean	isSeasonedChunk(Chunk chunk, long time)
private boolean	isWriteStoreHeader(Chunk c, boolean storeAtEndOfFile)
private long	lastChunkVersion()
private void	markMetaChanged()
private long	measureFileLengthInUse()
private boolean	moveChunk(Chunk chunk, long reservedAreaLow, long reservedAreaHigh)	Move specified chunk into free area of the file.
private boolean	moveChunkInside(Chunk chunkToMove, long boundary)
private boolean	needStore()
private void	onVersionChange(long version)
static MVStore	open(String fileName)	Open a store in exclusive mode.
<M extends MVMap<K,V>, K, V> M	openMap(int id, MVMap.MapBuilder<M,K,V> builder)	Open an existing map with the given builder.
<K,V> MVMap<K,V>	openMap(String name)	Open a map with the default settings.
<M extends MVMap<K,V>, K, V> M	openMap(String name, MVMap.MapBuilder<M,K,V> builder)	Open a map with the given builder.
private MVMap<String,String>	openMetaMap()
private void	panic(MVStoreException e)
private Chunk	readChunkFooter(long block)	Try to read a chunk footer.
private Chunk	readChunkHeader(long block)
private Chunk	readChunkHeaderAndFooter(long block, int expectedId)	Read a chunk header and footer, and verify the stored data is consistent.
private Chunk	readChunkHeaderOptionally(long block)
private Chunk	readChunkHeaderOptionally(long block, int expectedId)
(package private) <K,V> Page<K,V>	readPage(MVMap<K,V> map, long pos)	Read a page.
private <K,V> Page<K,V>	readPageFromCache(long pos)
private void	readStoreHeader()
void	registerUnsavedMemory(int memory)	Adjust amount of "unsaved memory" meaning amount of RAM occupied by pages not saved yet to the file.
MVStore.TxCounter	registerVersionUsage()	Register opened operation (transaction).
private void	releaseWriteBuffer(WriteBuffer buff)	Release a buffer for writing.
void	removeMap(String name)	Remove map by name.
void	removeMap(MVMap<?,?> map)	Remove a map from the current version of the store.
void	renameMap(MVMap<?,?> map, String newName)	Rename a map.
private boolean	requireStore()
private boolean	rewriteChunks(int writeLimit, int targetFillRate)
private int	rewriteChunks(Set<Integer> set, boolean secondPass)
void	rollback()	Revert to the beginning of the current version, reverting all uncommitted changes.
void	rollbackTo(long version)	Revert to the beginning of the given version.
private void	scrubLayoutMap()
private void	scrubMetaMap()
private void	serializeAndStore(boolean syncRun, long reservedLow, Supplier<Long> reservedHighSupplier, ArrayList<Page<?,?>> changed, long time, long version)
private void	serializeToBuffer(WriteBuffer buff, ArrayList<Page<?,?>> changed, Chunk c, long reservedLow, Supplier<Long> reservedHighSupplier)
void	setAutoCommitDelay(int millis)	Set the maximum delay in milliseconds to auto-commit changes.
void	setCacheSize(int mb)	Set the read cache size in MB.
private void	setLastChunk(Chunk last)
private void	setOldestVersionToKeep(long oldestVersionToKeep)
void	setRetentionTime(int ms)	How long to retain old, persisted chunks, in milliseconds.
void	setReuseSpace(boolean reuseSpace)	Whether empty space in the file should be re-used.
void	setStoreVersion(int version)	Update the store version.
void	setVersionsToKeep(int count)	How many versions to retain for in-memory stores.
private void	setWriteVersion(long version)
private void	shrinkFileIfPossible(int minPercent)	Shrink the file if possible, and if at least a given percentage can be saved.
private static void	shutdownExecutor(ThreadPoolExecutor executor)
private void	stopBackgroundThread(boolean waitForIt)
private void	store(boolean syncWrite)
private void	store(long reservedLow, long reservedHigh)
private void	storeBuffer(Chunk c, WriteBuffer buff, ArrayList<Page<?,?>> changed)
private void	storeNow(boolean syncWrite, long reservedLow, Supplier<Long> reservedHighSupplier)
private static void	submitOrRun(ThreadPoolExecutor executor, Runnable action, boolean syncRun)
void	sync()	Force all stored changes to be written to the storage.
long	tryCommit()	Unlike regular commit this method returns immediately if there is commit in progress on another thread, otherwise it acts as regular commit.
private long	tryCommit(Predicate<MVStore> check)
private void	unlockAndCheckPanicCondition()
private boolean	validateFileLength(String msg)
private void	write(long pos, ByteBuffer buffer)
(package private) void	writeInBackground()	Commit and save all changes, if there are any, and compact the store if needed.
private void	writeStoreHeader()

Methods inherited from class Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

HDR_H

private static final String HDR_H

See Also:

• Constant Field Values

HDR_BLOCK_SIZE

private static final String HDR_BLOCK_SIZE

See Also:

• Constant Field Values

HDR_FORMAT

private static final String HDR_FORMAT

See Also:

• Constant Field Values

HDR_CREATED

private static final String HDR_CREATED

See Also:

• Constant Field Values

HDR_FORMAT_READ

private static final String HDR_FORMAT_READ

See Also:

• Constant Field Values

HDR_CHUNK

private static final String HDR_CHUNK

See Also:

• Constant Field Values

HDR_BLOCK

private static final String HDR_BLOCK

See Also:

• Constant Field Values

HDR_VERSION

private static final String HDR_VERSION

See Also:

• Constant Field Values

HDR_CLEAN

private static final String HDR_CLEAN

See Also:

• Constant Field Values

HDR_FLETCHER

private static final String HDR_FLETCHER

See Also:

• Constant Field Values

META_ID_KEY

public static final String META_ID_KEY

The key for the entry within "layout" map, which contains id of "meta" map. Entry value (hex encoded) is usually equal to 1, unless it's a legacy (upgraded) database and id 1 has been taken already by another map.

See Also:

• Constant Field Values

BLOCK_SIZE

static final int BLOCK_SIZE

The block size (physical sector size) of the disk. The store header is written twice, one copy in each block, to ensure it survives a crash.

See Also:

• Constant Field Values

FORMAT_WRITE_MIN

private static final int FORMAT_WRITE_MIN

See Also:

• Constant Field Values

FORMAT_WRITE_MAX

private static final int FORMAT_WRITE_MAX

See Also:

• Constant Field Values

FORMAT_READ_MIN

private static final int FORMAT_READ_MIN

See Also:

• Constant Field Values

FORMAT_READ_MAX

private static final int FORMAT_READ_MAX

See Also:

• Constant Field Values

STATE_OPEN

private static final int STATE_OPEN

Store is open.

See Also:

• Constant Field Values

STATE_STOPPING

private static final int STATE_STOPPING

Store is about to close now, but is still operational. Outstanding store operation by background writer or other thread may be in progress. New updates must not be initiated, unless they are part of a closing procedure itself.

See Also:

• Constant Field Values

STATE_CLOSING

private static final int STATE_CLOSING

Store is closing now, and any operation on it may fail.

See Also:

• Constant Field Values

STATE_CLOSED

private static final int STATE_CLOSED

Store is closed.

See Also:

• Constant Field Values

PIPE_LENGTH

private static final int PIPE_LENGTH

See Also:

• Constant Field Values

storeLock

private final ReentrantLock storeLock

Lock which governs access to major store operations: store(), close(), ... It serves as a replacement for synchronized(this), except it allows for non-blocking lock attempts.

serializationLock

private final ReentrantLock serializationLock

saveChunkLock

private final ReentrantLock saveChunkLock

backgroundWriterThread

private final AtomicReference<MVStore.BackgroundWriterThread> backgroundWriterThread

Reference to a background thread, which is expected to be running, if any.

serializationExecutor

private ThreadPoolExecutor serializationExecutor

Single-threaded executor for serialization of the store snapshot into ByteBuffer

bufferSaveExecutor

private ThreadPoolExecutor bufferSaveExecutor

Single-threaded executor for saving ByteBuffer as a new Chunk

reuseSpace

private volatile boolean reuseSpace

state

private volatile int state

fileStore

private final FileStore fileStore

fileStoreIsProvided

private final boolean fileStoreIsProvided

pageSplitSize

private final int pageSplitSize

keysPerPage

private final int keysPerPage

cache

private final CacheLongKeyLIRS<Page<?,?>> cache

The page cache. The default size is 16 MB, and the average size is 2 KB. It is split in 16 segments. The stack move distance is 2% of the expected number of entries.

chunksToC

private final CacheLongKeyLIRS<long[]> chunksToC

Cache for chunks "Table of Content" used to translate page's sequential number within containing chunk into byte position within chunk's image. Cache keyed by chunk id.

lastChunk

private volatile Chunk lastChunk

The newest chunk. If nothing was stored yet, this field is not set.

chunks

private final ConcurrentHashMap<Integer,Chunk> chunks

The map of chunks.

removedPages

private final Queue<MVStore.RemovedPageInfo> removedPages

deadChunks

private final Deque<Chunk> deadChunks

updateCounter

private long updateCounter

updateAttemptCounter

private long updateAttemptCounter

layout

private final MVMap<String,String> layout

The layout map. Contains chunks metadata and root locations for all maps. This is relatively fast changing part of metadata

meta

private final MVMap<String,String> meta

The metadata map. Holds name -> id and id -> name and id -> metadata mapping for all maps. This is relatively slow changing part of metadata

maps

private final ConcurrentHashMap<Integer, MVMap<?,?>> maps

storeHeader

private final HashMap<String,Object> storeHeader

writeBufferPool

private final Queue<WriteBuffer> writeBufferPool

lastMapId

private final AtomicInteger lastMapId

lastChunkId

private int lastChunkId

versionsToKeep

private int versionsToKeep

compressionLevel

private final int compressionLevel

The compression level for new pages (0 for disabled, 1 for fast, 2 for high). Even if disabled, the store may contain (old) compressed pages.

compressorFast

private Compressor compressorFast

compressorHigh

private Compressor compressorHigh

recoveryMode

private final boolean recoveryMode

backgroundExceptionHandler

public final Thread.UncaughtExceptionHandler backgroundExceptionHandler

currentVersion

private volatile long currentVersion

oldestVersionToKeep

private final AtomicLong oldestVersionToKeep

Oldest store version in use. All version beyond this can be safely dropped

versions

private final Deque<MVStore.TxCounter> versions

Ordered collection of all version usage counters for all versions starting from oldestVersionToKeep and up to current.

currentTxCounter

private volatile MVStore.TxCounter currentTxCounter

Counter of open transactions for the latest (current) store version

unsavedMemory

private int unsavedMemory

The estimated memory used by unsaved pages. This number is not accurate, also because it may be changed concurrently, and because temporary pages are counted.

autoCommitMemory

private final int autoCommitMemory

saveNeeded

private volatile boolean saveNeeded

creationTime

private long creationTime

The time the store was created, in milliseconds since 1970.

retentionTime

private int retentionTime

How long to retain old, persisted chunks, in milliseconds. For larger or equal to zero, a chunk is never directly overwritten if unused, but instead, the unused field is set. If smaller zero, chunks are directly overwritten if unused.

lastCommitTime

private long lastCommitTime

currentStoreVersion

private volatile long currentStoreVersion

The version of the current store operation (if any).

metaChanged

private volatile boolean metaChanged

autoCommitDelay

private int autoCommitDelay

The delay in milliseconds to automatically commit and write changes.

autoCompactFillRate

private final int autoCompactFillRate

autoCompactLastFileOpCount

private long autoCompactLastFileOpCount

panicException

private volatile MVStoreException panicException

lastTimeAbsolute

private long lastTimeAbsolute

leafCount

private long leafCount

nonLeafCount

private long nonLeafCount

Constructor Details

MVStore

MVStore(Map<String,Object> config)

Create and open the store.

Parameters:

config - the configuration to use

Throws:

MVStoreException - if the file is corrupt, or an exception occurred while opening

IllegalArgumentException - if the directory does not exist

Method Details

openMetaMap

private MVMap<String,String> openMetaMap()

scrubLayoutMap

private void scrubLayoutMap()

scrubMetaMap

private void scrubMetaMap()

unlockAndCheckPanicCondition

private void unlockAndCheckPanicCondition()

panic

private void panic(MVStoreException e)

getPanicException

public MVStoreException getPanicException()

open

public static MVStore open(String fileName)

Open a store in exclusive mode. For a file-based store, the parent directory must already exist.

Parameters:

fileName - the file name (null for in-memory)

Returns:

the store

openMap

public <K,V> MVMap<K,V> openMap(String name)

Open a map with the default settings. The map is automatically create if it does not yet exist. If a map with this name is already open, this map is returned.

Type Parameters:

K - the key type

V - the value type

Parameters:

name - the name of the map

Returns:

the map

openMap

public <M extends MVMap<K,V>, K, V> M openMap(String name,
 MVMap.MapBuilder<M,K,V> builder)

Open a map with the given builder. The map is automatically create if it does not yet exist. If a map with this name is already open, this map is returned.

Type Parameters:

M - the map type

K - the key type

V - the value type

Parameters:

name - the name of the map

builder - the map builder

Returns:

the map

openMap

public <M extends MVMap<K,V>, K, V> M openMap(int id,
 MVMap.MapBuilder<M,K,V> builder)

Open an existing map with the given builder.

Type Parameters:

M - the map type

K - the key type

V - the value type

Parameters:

id - the map id

builder - the map builder

Returns:

the map

getMap

public <K,V> MVMap<K,V> getMap(int id)

Get map by id.

Type Parameters:

K - the key type

V - the value type

Parameters:

id - map id

Returns:

Map

getMapNames

public Set<String> getMapNames()

Get the set of all map names.

Returns:

the set of names

getLayoutMap

public MVMap<String,String> getLayoutMap()

Get this store's layout map. This data is for informational purposes only. The data is subject to change in future versions.

The data in this map should not be modified (changing system data may corrupt the store).

The layout map contains the following entries:

chunk.{chunkId} = {chunk metadata}
root.{mapId} = {root position}

Returns:

the metadata map

getMetaMap

public MVMap<String,String> getMetaMap()

Get the metadata map. This data is for informational purposes only. The data is subject to change in future versions.

The data in this map should not be modified (changing system data may corrupt the store).

The metadata map contains the following entries:

name.{name} = {mapId}
map.{mapId} = {map metadata}
setting.storeVersion = {version}

Returns:

the metadata map

getLayoutMap

private MVMap<String,String> getLayoutMap(long version)

getChunkForVersion

private Chunk getChunkForVersion(long version)

hasMap

public boolean hasMap(String name)

Check whether a given map exists.

Parameters:

name - the map name

Returns:

true if it exists

hasData

public boolean hasData(String name)

Check whether a given map exists and has data.

Parameters:

name - the map name

Returns:

true if it exists and has data.

markMetaChanged

private void markMetaChanged()

readStoreHeader

private void readStoreHeader()

getUnsupportedWriteFormatException

private MVStoreException getUnsupportedWriteFormatException(long format,
 int expectedFormat,
 String s)

findLastChunkWithCompleteValidChunkSet

private boolean findLastChunkWithCompleteValidChunkSet(Chunk[] lastChunkCandidates,
 Map<Long,Chunk> validChunksByLocation,
 Map<Integer,Chunk> validChunksById,
 boolean afterFullScan)

setLastChunk

private void setLastChunk(Chunk last)

discoverChunk

private Chunk discoverChunk(long block)

Discover a valid chunk, searching file backwards from the given block

Parameters:

block - to start search from (found chunk footer should be no further than block-1)

Returns:

valid chunk or null if none found

readChunkHeaderAndFooter

private Chunk readChunkHeaderAndFooter(long block,
 int expectedId)

Read a chunk header and footer, and verify the stored data is consistent.

Parameters:

block - the block

expectedId - of the chunk

Returns:

the chunk, or null if the header or footer don't match or are not consistent

readChunkFooter

private Chunk readChunkFooter(long block)

Try to read a chunk footer.

Parameters:

block - the index of the next block after the chunk

Returns:

the chunk, or null if not successful

writeStoreHeader

private void writeStoreHeader()

write

private void write(long pos,
 ByteBuffer buffer)

close

public void close()

Close the file and the store. Unsaved changes are written to disk first.

Specified by:

close in interface AutoCloseable

close

public void close(int allowedCompactionTime)

Close the file and the store. Unsaved changes are written to disk first, and compaction (up to a specified number of milliseconds) is attempted.

Parameters:

allowedCompactionTime - the allowed time for compaction (in milliseconds)

closeImmediately

public void closeImmediately()

Close the file and the store, without writing anything. This will try to stop the background thread (without waiting for it). This method ignores all errors.

closeStore

private void closeStore(boolean normalShutdown,
 int allowedCompactionTime)

shutdownExecutor

private static void shutdownExecutor(ThreadPoolExecutor executor)

getChunk

private Chunk getChunk(long pos)

Get the chunk for the given position.

Parameters:

pos - the position

Returns:

the chunk

setWriteVersion

private void setWriteVersion(long version)

tryCommit

public long tryCommit()

Unlike regular commit this method returns immediately if there is commit in progress on another thread, otherwise it acts as regular commit. This method may return BEFORE this thread changes are actually persisted!

Returns:

the new version (incremented if there were changes)

tryCommit

private long tryCommit(Predicate<MVStore> check)

commit

public long commit()

Commit the changes.

This method does nothing if there are no unsaved changes, otherwise it increments the current version and stores the data (for file based stores).

It is not necessary to call this method when auto-commit is enabled (the default setting), as in this case it is automatically called from time to time or when enough changes have accumulated. However, it may still be called to flush all changes to disk.

At most one store operation may run at any time.

Returns:

the new version (incremented if there were changes)

commit

private long commit(Predicate<MVStore> check)

store

private void store(boolean syncWrite)

storeNow

private void storeNow(boolean syncWrite,
 long reservedLow,
 Supplier<Long> reservedHighSupplier)

submitOrRun

private static void submitOrRun(ThreadPoolExecutor executor,
 Runnable action,
 boolean syncRun)
                         throws ExecutionException

Throws:

ExecutionException

collectChangedMapRoots

private ArrayList<Page<?,?>> collectChangedMapRoots(long version)

serializeAndStore

private void serializeAndStore(boolean syncRun,
 long reservedLow,
 Supplier<Long> reservedHighSupplier,
 ArrayList<Page<?,?>> changed,
 long time,
 long version)

createChunk

private Chunk createChunk(long time,
 long version)

serializeToBuffer

private void serializeToBuffer(WriteBuffer buff,
 ArrayList<Page<?,?>> changed,
 Chunk c,
 long reservedLow,
 Supplier<Long> reservedHighSupplier)

storeBuffer

private void storeBuffer(Chunk c,
 WriteBuffer buff,
 ArrayList<Page<?,?>> changed)

isWriteStoreHeader

private boolean isWriteStoreHeader(Chunk c,
 boolean storeAtEndOfFile)

getWriteBuffer

private WriteBuffer getWriteBuffer()

Get a buffer for writing. This caller must synchronize on the store before calling the method and until after using the buffer.

Returns:

the buffer

releaseWriteBuffer

private void releaseWriteBuffer(WriteBuffer buff)

Release a buffer for writing. This caller must synchronize on the store before calling the method and until after using the buffer.

Parameters:

buff - the buffer than can be re-used

canOverwriteChunk

private static boolean canOverwriteChunk(Chunk c,
 long oldestVersionToKeep)

isSeasonedChunk

private boolean isSeasonedChunk(Chunk chunk,
 long time)

getTimeSinceCreation

private long getTimeSinceCreation()

getTimeAbsolute

private long getTimeAbsolute()

acceptChunkOccupancyChanges

private void acceptChunkOccupancyChanges(long time,
 long version)

Apply the freed space to the chunk metadata. The metadata is updated, but completely free chunks are not removed from the set of chunks, and the disk space is not yet marked as free. They are queued instead and wait until their usage is over.

shrinkFileIfPossible

private void shrinkFileIfPossible(int minPercent)

Shrink the file if possible, and if at least a given percentage can be saved.

Parameters:

minPercent - the minimum percentage to save

getFileLengthInUse

private long getFileLengthInUse()

Get the position right after the last used byte.

Returns:

the position

getAfterLastBlock

private long getAfterLastBlock()

Get the index of the first block after last occupied one. It marks the beginning of the last (infinite) free space.

Returns:

block index

measureFileLengthInUse

private long measureFileLengthInUse()

hasUnsavedChanges

public boolean hasUnsavedChanges()

Check whether there are any unsaved changes.

Returns:

if there are any changes

readChunkHeader

private Chunk readChunkHeader(long block)

readChunkHeaderOptionally

private Chunk readChunkHeaderOptionally(long block)

readChunkHeaderOptionally

private Chunk readChunkHeaderOptionally(long block,
 int expectedId)

compactMoveChunks

public void compactMoveChunks()

Compact by moving all chunks next to each other.

compactMoveChunks

boolean compactMoveChunks(int targetFillRate,
 long moveSize)

Compact the store by moving all chunks next to each other, if there is free space between chunks. This might temporarily increase the file size. Chunks are overwritten irrespective of the current retention time. Before overwriting chunks and before resizing the file, syncFile() is called.

Parameters:

targetFillRate - do nothing if the file store fill rate is higher than this

moveSize - the number of bytes to move

Returns:

true if any chunks were moved as result of this operation, false otherwise

compactMoveChunks

private boolean compactMoveChunks(long moveSize)

findChunksToMove

private Iterable<Chunk> findChunksToMove(long startBlock,
 long moveSize)

getMovePriority

private int getMovePriority(Chunk chunk)

compactMoveChunks

private void compactMoveChunks(Iterable<Chunk> move)

store

private void store(long reservedLow,
 long reservedHigh)

moveChunkInside

private boolean moveChunkInside(Chunk chunkToMove,
 long boundary)

moveChunk

private boolean moveChunk(Chunk chunk,
 long reservedAreaLow,
 long reservedAreaHigh)

Move specified chunk into free area of the file. "Reserved" area specifies file interval to be avoided, when un-allocated space will be chosen for a new chunk's location.

Parameters:

chunk - to move

reservedAreaLow - low boundary of reserved area, inclusive

reservedAreaHigh - high boundary of reserved area, exclusive

Returns:

true if block was moved, false otherwise

sync

public void sync()

Force all stored changes to be written to the storage. The default implementation calls FileChannel.force(true).

compactFile

public void compactFile(int maxCompactTime)

Compact store file, that is, compact blocks that have a low fill rate, and move chunks next to each other. This will typically shrink the file. Changes are flushed to the file, and old chunks are overwritten.

Parameters:

maxCompactTime - the maximum time in milliseconds to compact

compact

public boolean compact(int targetFillRate,
 int write)

Try to increase the fill rate by re-writing partially full chunks. Chunks with a low number of live items are re-written.

If the current fill rate is higher than the target fill rate, nothing is done.

Please note this method will not necessarily reduce the file size, as empty chunks are not overwritten.

Only data of open maps can be moved. For maps that are not open, the old chunk is still referenced. Therefore, it is recommended to open all maps before calling this method.

Parameters:

targetFillRate - the minimum percentage of live entries

write - the minimum number of bytes to write

Returns:

if a chunk was re-written

rewriteChunks

private boolean rewriteChunks(int writeLimit,
 int targetFillRate)

getChunksFillRate

public int getChunksFillRate()

Get the current fill rate (percentage of used space in the file). Unlike the fill rate of the store, here we only account for chunk data; the fill rate here is how much of the chunk data is live (still referenced). Young chunks are considered live.

Returns:

the fill rate, in percent (100 is completely full)

getRewritableChunksFillRate

public int getRewritableChunksFillRate()

getChunksFillRate

private int getChunksFillRate(boolean all)

getChunkCount

public int getChunkCount()

Get data chunks count.

Returns:

number of existing chunks in store.

getPageCount

public int getPageCount()

Get data pages count.

Returns:

number of existing pages in store.

getLivePageCount

public int getLivePageCount()

Get live data pages count.

Returns:

number of existing live pages in store.

getProjectedFillRate

private int getProjectedFillRate(int thresholdChunkFillRate)

getFillRate

public int getFillRate()

findOldChunks

private Iterable<Chunk> findOldChunks(int writeLimit,
 int targetFillRate)

isRewritable

private boolean isRewritable(Chunk chunk,
 long time)

compactRewrite

private int compactRewrite(Set<Integer> set)

rewriteChunks

private int rewriteChunks(Set<Integer> set,
 boolean secondPass)

createIdSet

private static HashSet<Integer> createIdSet(Iterable<Chunk> toCompact)

readPage

<K,V> Page<K,V> readPage(MVMap<K,V> map,
 long pos)

Read a page.

Type Parameters:

K - key type

V - value type

Parameters:

map - the map

pos - the page position

Returns:

the page

getToC

private long[] getToC(Chunk chunk)

readPageFromCache

private <K,V> Page<K,V> readPageFromCache(long pos)

accountForRemovedPage

void accountForRemovedPage(long pos,
 long version,
 boolean pinned,
 int pageNo)

Remove a page.

Parameters:

pos - the position of the page

version - at which page was removed

pinned - whether page is considered pinned

pageNo - sequential page number within chunk

calculatePageNo

private int calculatePageNo(long pos)

getCompressorFast

Compressor getCompressorFast()

getCompressorHigh

Compressor getCompressorHigh()

getCompressionLevel

int getCompressionLevel()

getPageSplitSize

public int getPageSplitSize()

getKeysPerPage

public int getKeysPerPage()

getMaxPageSize

public long getMaxPageSize()

getReuseSpace

public boolean getReuseSpace()

setReuseSpace

public void setReuseSpace(boolean reuseSpace)

Whether empty space in the file should be re-used. If enabled, old data is overwritten (default). If disabled, writes are appended at the end of the file.

This setting is specially useful for online backup. To create an online backup, disable this setting, then copy the file (starting at the beginning of the file). In this case, concurrent backup and write operations are possible (obviously the backup process needs to be faster than the write operations).

Parameters:

reuseSpace - the new value

getRetentionTime

public int getRetentionTime()

setRetentionTime

public void setRetentionTime(int ms)

How long to retain old, persisted chunks, in milliseconds. Chunks that are older may be overwritten once they contain no live data.

The default value is 45000 (45 seconds) when using the default file store. It is assumed that a file system and hard disk will flush all write buffers within this time. Using a lower value might be dangerous, unless the file system and hard disk flush the buffers earlier. To manually flush the buffers, use MVStore.getFile().force(true), however please note that according to various tests this does not always work as expected depending on the operating system and hardware.

The retention time needs to be long enough to allow reading old chunks while traversing over the entries of a map.

This setting is not persisted.

Parameters:

ms - how many milliseconds to retain old chunks (0 to overwrite them as early as possible)

setVersionsToKeep

public void setVersionsToKeep(int count)

How many versions to retain for in-memory stores. If not set, 5 old versions are retained.

Parameters:

count - the number of versions to keep

getVersionsToKeep

public long getVersionsToKeep()

Get the oldest version to retain in memory (for in-memory stores).

Returns:

the version

getOldestVersionToKeep

long getOldestVersionToKeep()

Get the oldest version to retain. We keep at least number of previous versions specified by "versionsToKeep" configuration parameter (default 5). Previously it was used only in case of non-persistent MVStore. Now it's honored in all cases (although H2 always sets it to zero). Oldest version determination also takes into account calls (de)registerVersionUsage(), an will not release the version, while version is still in use.

Returns:

the version

setOldestVersionToKeep

private void setOldestVersionToKeep(long oldestVersionToKeep)

lastChunkVersion

private long lastChunkVersion()

isKnownVersion

private boolean isKnownVersion(long version)

Check whether all data can be read from this version. This requires that all chunks referenced by this version are still available (not overwritten).

Parameters:

version - the version

Returns:

true if all data can be read

registerUnsavedMemory

public void registerUnsavedMemory(int memory)

Adjust amount of "unsaved memory" meaning amount of RAM occupied by pages not saved yet to the file. This is the amount which triggers auto-commit.

Parameters:

memory - adjustment

isSaveNeeded

boolean isSaveNeeded()

beforeWrite

void beforeWrite(MVMap<?,?> map)

This method is called before writing to a map.

Parameters:

map - the map

requireStore

private boolean requireStore()

needStore

private boolean needStore()

getStoreVersion

public int getStoreVersion()

Get the store version. The store version is usually used to upgrade the structure of the store after upgrading the application. Initially the store version is 0, until it is changed.

Returns:

the store version

setStoreVersion

public void setStoreVersion(int version)

Update the store version.

Parameters:

version - the new store version

rollback

public void rollback()

Revert to the beginning of the current version, reverting all uncommitted changes.

rollbackTo

public void rollbackTo(long version)

Revert to the beginning of the given version. All later changes (stored or not) are forgotten. All maps that were created later are closed. A rollback to a version before the last stored version is immediately persisted. Rollback to version 0 means all data is removed.

Parameters:

version - the version to revert to

clearCaches

private void clearCaches()

getRootPos

private long getRootPos(int mapId)

getCurrentVersion

public long getCurrentVersion()

Get the current version of the data. When a new store is created, the version is 0.

Returns:

the version

getFileStore

public FileStore getFileStore()

Get the file store.

Returns:

the file store

getStoreHeader

public Map<String,Object> getStoreHeader()

Get the store header. This data is for informational purposes only. The data is subject to change in future versions. The data should not be modified (doing so may corrupt the store).

Returns:

the store header

checkOpen

private void checkOpen()

renameMap

public void renameMap(MVMap<?,?> map,
 String newName)

Rename a map.

Parameters:

map - the map

newName - the new name

removeMap

public void removeMap(MVMap<?,?> map)

Remove a map from the current version of the store.

Parameters:

map - the map to remove

deregisterMapRoot

void deregisterMapRoot(int mapId)

Performs final stage of map removal - delete root location info from the layout table. Map is supposedly closed and anonymous and has no outstanding usage by now.

Parameters:

mapId - to deregister

removeMap

public void removeMap(String name)

Remove map by name.

Parameters:

name - the map name

getMapName

public String getMapName(int id)

Get the name of the given map.

Parameters:

id - the map id

Returns:

the name, or null if not found

getMapId

private int getMapId(String name)

writeInBackground

void writeInBackground()

Commit and save all changes, if there are any, and compact the store if needed.

doMaintenance

private void doMaintenance(int targetFillRate)

getTargetFillRate

private int getTargetFillRate()

isIdle

private boolean isIdle()

handleException

private void handleException(Throwable ex)

setCacheSize

public void setCacheSize(int mb)

Set the read cache size in MB.

Parameters:

mb - the cache size in MB.

isOpen

private boolean isOpen()

isClosed

public boolean isClosed()

Determine that store is open, or wait for it to be closed (by other thread)

Returns:

true if store is open, false otherwise

isOpenOrStopping

private boolean isOpenOrStopping()

stopBackgroundThread

private void stopBackgroundThread(boolean waitForIt)

setAutoCommitDelay

public void setAutoCommitDelay(int millis)

Set the maximum delay in milliseconds to auto-commit changes.

To disable auto-commit, set the value to 0. In this case, changes are only committed when explicitly calling commit.

The default is 1000, meaning all changes are committed after at most one second.

Parameters:

millis - the maximum delay

createSingleThreadExecutor

private static ThreadPoolExecutor createSingleThreadExecutor(String threadName)

isBackgroundThread

public boolean isBackgroundThread()

getAutoCommitDelay

public int getAutoCommitDelay()

Get the auto-commit delay.

Returns:

the delay in milliseconds, or 0 if auto-commit is disabled.

getAutoCommitMemory

public int getAutoCommitMemory()

Get the maximum memory (in bytes) used for unsaved pages. If this number is exceeded, unsaved changes are stored to disk.

Returns:

the memory in bytes

getUnsavedMemory

public int getUnsavedMemory()

Get the estimated memory (in bytes) of unsaved data. If the value exceeds the auto-commit memory, the changes are committed.

The returned value is an estimation only.

Returns:

the memory in bytes

cachePage

void cachePage(Page<?,?> page)

Put the page in the cache.

Parameters:

page - the page

getCacheSizeUsed

public int getCacheSizeUsed()

Get the amount of memory used for caching, in MB. Note that this does not include the page chunk references cache, which is 25% of the size of the page cache.

Returns:

the amount of memory used for caching

getCacheSize

public int getCacheSize()

Get the maximum cache size, in MB. Note that this does not include the page chunk references cache, which is 25% of the size of the page cache.

Returns:

the cache size

getCache

public CacheLongKeyLIRS<Page<?,?>> getCache()

Get the cache.

Returns:

the cache

isReadOnly

public boolean isReadOnly()

Whether the store is read-only.

Returns:

true if it is

getCacheHitRatio

public int getCacheHitRatio()

getTocCacheHitRatio

public int getTocCacheHitRatio()

getCacheHitRatio

private static int getCacheHitRatio(CacheLongKeyLIRS<?> cache)

getLeafRatio

public int getLeafRatio()

getUpdateFailureRatio

public double getUpdateFailureRatio()

registerVersionUsage

public MVStore.TxCounter registerVersionUsage()

Register opened operation (transaction). This would increment usage counter for the current version. This version (and all after it) should not be dropped until all transactions involved are closed and usage counter goes to zero.

Returns:

TxCounter to be decremented when operation finishes (transaction closed).

deregisterVersionUsage

public void deregisterVersionUsage(MVStore.TxCounter txCounter)

De-register (close) completed operation (transaction). This will decrement usage counter for the corresponding version. If counter reaches zero, that version (and all unused after it) can be dropped immediately.

Parameters:

txCounter - to be decremented, obtained from registerVersionUsage()

onVersionChange

private void onVersionChange(long version)

dropUnusedVersions

private void dropUnusedVersions()

dropUnusedChunks

private int dropUnusedChunks()

freeChunkSpace

private void freeChunkSpace(Chunk chunk)

freeFileSpace

private void freeFileSpace(long start,
 int length)

validateFileLength

private boolean validateFileLength(String msg)