in entity-store/src/main/java/jetbrains/exodus/entitystore/util/BackupUtil.java [91:444]
public static InputStream reEncryptBackup(final ArchiveInputStream archiveStream,
final byte[] firstKey, final long firstIv,
final byte[] secondKey, final long secondIv,
final StreamCipherProvider cipherProvider,
final boolean ignorePageSizeCheck,
final int pageSize) throws IOException {
var pipedInputStream = new PipedInputStream(1024 * 1024);
var pipedOutputStream = new PipedOutputStream(pipedInputStream);
var errorAwareInputStream = new ErrorAwareInputStream(pipedInputStream);
var bufferedOutputStream = new BufferedOutputStream(pipedOutputStream, 1024 * 1024);
final TarArchiveOutputStream tarArchiveOutputStream;
if (secondKey == null) {
tarArchiveOutputStream = new TarArchiveOutputStream(new GzipCompressorOutputStream(bufferedOutputStream));
} else {
var gzipParameters = new GzipParameters();
gzipParameters.setCompressionLevel(Deflater.NO_COMPRESSION);
tarArchiveOutputStream =
new TarArchiveOutputStream(new GzipCompressorOutputStream(bufferedOutputStream, gzipParameters));
}
var executor = Executors.newSingleThreadExecutor((runnable) -> {
var thread = new Thread(runnable);
thread.setName("Backup re-encryption thread");
thread.setDaemon(true);
thread.setUncaughtExceptionHandler((t, e) -> logger.error("Uncaught exception in thread " + t.getName(), e));
return thread;
});
executor.submit(() -> {
try {
if (ignorePageSizeCheck && pageSize <= 0) {
throw new IllegalStateException("Page size should be specified if ignorePageSizeCheck is set");
}
if (ignorePageSizeCheck) {
logger.warn("Because flag ignorePageSizeCheck is set, database will be marked is incorrectly closed " +
"and will be recovered on next startup. This is not a problem if you are going to see " +
"database restore routine.");
}
final byte[] readBuffer = new byte[1024 * 1024];
final int pageStep = 4 * 1024;
final int pageMaxSize = 256 * 1024;
var metadataMap = new HashMap<String, DbMetadata>();
var inputEntry = archiveStream.getNextEntry();
while (inputEntry != null) {
//skip directories we are interested only in files
if (inputEntry.isDirectory()) {
inputEntry = archiveStream.getNextEntry();
continue;
}
var name = inputEntry.getName();
var outputArchiveEntry = new TarArchiveEntry(name);
var entrySize = inputEntry.getSize();
if (entrySize < 0) {
throw new IllegalStateException("Archive entries with unknown size are not" +
" supported. Entry " + name + " should provide size to be re-encrypted");
}
final Path namePath = Path.of(name);
if (ignorePageSizeCheck && name.endsWith(LogUtil.LOG_FILE_EXTENSION)) {
if ((entrySize & (pageSize - 1)) != 0) {
//rounding file size to the page size
var newEntrySize = entrySize & -pageSize;
logger.error("File {} size {} is not multiple of page size {}. Rounding to {} because flag " +
"ignorePageSizeCheck is set to true. ",
name, entrySize, pageSize, newEntrySize);
entrySize = newEntrySize;
}
}
outputArchiveEntry.setSize(entrySize);
outputArchiveEntry.setModTime(inputEntry.getLastModifiedDate());
tarArchiveOutputStream.putArchiveEntry(outputArchiveEntry);
if (name.endsWith(LogUtil.LOG_FILE_EXTENSION)) {
int processed = 0;
int readBufferOffset = 0;
final String rootName = extractRootName(namePath);
DbMetadata dbMetadata = metadataMap.get(rootName);
final long fileAddress = LogUtil.getAddress(namePath.getFileName().toString());
while (processed < entrySize) {
if (dbMetadata != null &&
dbMetadata.binaryFormatVersion == EnvironmentImpl.CURRENT_FORMAT_VERSION) {
if (dbMetadata.backupMetadata != null) {
var backupMetadata = dbMetadata.backupMetadata;
var rootAddress = backupMetadata.getRootAddress();
if (fileAddress + processed > rootAddress && (
(dbMetadata.fileLengthBound >= 0 && entrySize > dbMetadata.fileLengthBound)
|| ((entrySize & (dbMetadata.pageSize - 1)) != 0))) {
break;
}
}
if (dbMetadata.fileLengthBound >= 0 && entrySize > dbMetadata.fileLengthBound) {
throw new IllegalStateException("Backup is broken, size of the file " + name +
" should not exceed " + dbMetadata.fileLengthBound);
}
if ((entrySize & (dbMetadata.pageSize - 1)) != 0) {
throw new IllegalStateException("Backup is broken, size of the file " + name +
" should be quantified by " + dbMetadata.pageSize + " size of the file is "
+ entrySize);
}
}
final int readSize = (int) Math.min(
entrySize - processed, readBuffer.length - readBufferOffset);
assert dbMetadata == null ||
dbMetadata.binaryFormatVersion < EnvironmentImpl.CURRENT_FORMAT_VERSION ||
((readSize + readBufferOffset) & (dbMetadata.pageSize - 1)) == 0;
IOUtils.readFully(archiveStream, readBuffer, readBufferOffset, readSize);
final int bufferSize = readSize + readBufferOffset;
readBufferOffset = 0;
if (dbMetadata == null) {
var versionInformation = detectFormatVersion(
readBuffer, bufferSize, pageStep, pageMaxSize);
dbMetadata = new DbMetadata();
dbMetadata.binaryFormatVersion = versionInformation[0];
dbMetadata.pageSize = versionInformation[1];
metadataMap.put(rootName, dbMetadata);
assert dbMetadata.binaryFormatVersion >= 0;
}
byte[] dataToWrite;
final int dataToWriteLen;
if (dbMetadata.binaryFormatVersion < EnvironmentImpl.CURRENT_FORMAT_VERSION) {
dataToWrite = readBuffer;
dataToWriteLen = bufferSize;
if (firstKey != null) {
assert cipherProvider != null;
EnvKryptKt.cryptBlocksMutable(cipherProvider, firstKey, firstIv,
fileAddress + processed, readBuffer, 0, bufferSize,
LogUtil.LOG_BLOCK_ALIGNMENT);
}
if (secondKey != null) {
assert cipherProvider != null;
EnvKryptKt.cryptBlocksMutable(cipherProvider, secondKey, secondIv,
fileAddress + processed, readBuffer, 0, bufferSize,
LogUtil.LOG_BLOCK_ALIGNMENT);
}
} else {
var pages = bufferSize / dbMetadata.pageSize;
dataToWriteLen = pages * dbMetadata.pageSize;
validateBackupContent(readBuffer, dbMetadata.pageSize, name, processed, dataToWriteLen);
if (firstKey != null) {
assert cipherProvider != null;
dataToWrite = new byte[dataToWriteLen];
encryptV2FormatPages(firstKey, firstIv, cipherProvider,
readBuffer, dbMetadata.pageSize,
processed, fileAddress, dataToWrite, dataToWriteLen);
} else {
dataToWrite = readBuffer;
}
assert validateBackupContent(dataToWrite,
dbMetadata.pageSize, name, processed, dataToWriteLen);
if (secondKey != null) {
assert cipherProvider != null;
encryptV2FormatPages(secondKey, secondIv, cipherProvider,
dataToWrite,
dbMetadata.pageSize, processed, fileAddress, dataToWrite, dataToWriteLen);
}
assert validateBackupContent(dataToWrite,
dbMetadata.pageSize, name, processed, dataToWriteLen);
readBufferOffset = bufferSize - dataToWriteLen;
}
tarArchiveOutputStream.write(dataToWrite, 0, dataToWriteLen);
processed += dataToWriteLen;
if (readBufferOffset > 0) {
System.arraycopy(readBuffer, bufferSize - readBufferOffset, readBuffer,
0, readBufferOffset);
}
}
} else if (name.endsWith(PersistentEntityStoreImpl.BLOBS_EXTENSION)) {
final InputStream entryInputStream;
final OutputStream entryOutputStream;
final File blobFile = new File(name);
final File vault = findBlobsVault(blobFile);
long blobHandle = FileSystemBlobVault.getBlobHandleByFile(blobFile,
PersistentEntityStoreImpl.BLOBS_EXTENSION, vault);
if (firstKey != null) {
assert cipherProvider != null;
var cipher = EncryptedBlobVault.newCipher(cipherProvider, blobHandle, firstKey, firstIv);
entryInputStream = new StreamCipherInputStream(archiveStream, () -> cipher);
} else {
entryInputStream = archiveStream;
}
if (secondKey != null) {
assert cipherProvider != null;
var cipher = EncryptedBlobVault.newCipher(cipherProvider, blobHandle, secondKey, secondIv);
entryOutputStream = new StreamCipherOutputStream(tarArchiveOutputStream, cipher);
} else {
entryOutputStream = tarArchiveOutputStream;
}
IOUtils.copyLarge(entryInputStream, entryOutputStream, readBuffer);
} else if (name.equals(StartupMetadata.ZERO_FILE_NAME) ||
name.endsWith("/" + StartupMetadata.ZERO_FILE_NAME) ||
name.equals(StartupMetadata.FIRST_FILE_NAME) ||
name.endsWith("/" + StartupMetadata.FIRST_FILE_NAME)) {
final String rootName = extractRootName(namePath);
DbMetadata dbMetadata = metadataMap.get(rootName);
if (dbMetadata != null && dbMetadata.binaryFormatVersion < EnvironmentImpl.CURRENT_FORMAT_VERSION) {
throw new IllegalStateException("Backup is broken please fix backup consistency by " +
"opening it in database and correctly closing database. " +
"Database will restore data automatically.");
}
IOUtils.readFully(archiveStream, readBuffer, 0, StartupMetadata.FILE_SIZE);
var buffer = ByteBuffer.wrap(readBuffer);
buffer.limit(StartupMetadata.FILE_SIZE);
StartupMetadata startupMetadata = null;
var fileVersion = StartupMetadata.getFileVersion(buffer);
if (fileVersion >= 0) {
startupMetadata = StartupMetadata.deserialize(buffer, 0, false);
if (dbMetadata == null) {
dbMetadata = new DbMetadata();
dbMetadata.binaryFormatVersion = EnvironmentImpl.CURRENT_FORMAT_VERSION;
dbMetadata.pageSize = startupMetadata.getPageSize();
dbMetadata.fileLengthBound = startupMetadata.getFileLengthBoundary();
metadataMap.put(rootName, dbMetadata);
}
if (dbMetadata.pageSize != startupMetadata.getPageSize()) {
throw new IllegalStateException("Backup is broken please fix backup consistency by " +
"opening it in database and correctly closing database. " +
"Database will restore data automatically.");
}
}
if (ignorePageSizeCheck && startupMetadata != null) {
var serializedBuffer = StartupMetadata.serialize(fileVersion,
EnvironmentImpl.CURRENT_FORMAT_VERSION, startupMetadata.getRootAddress(),
startupMetadata.getPageSize(), startupMetadata.getFileLengthBoundary(),
false);
tarArchiveOutputStream.write(serializedBuffer.array(), 0, StartupMetadata.FILE_SIZE);
} else {
tarArchiveOutputStream.write(readBuffer, 0, StartupMetadata.FILE_SIZE);
}
} else if (name.equals(BackupMetadata.BACKUP_METADATA_FILE_NAME)) {
final String rootName = extractRootName(namePath);
DbMetadata dbMetadata = metadataMap.get(rootName);
if (dbMetadata != null && dbMetadata.binaryFormatVersion < EnvironmentImpl.CURRENT_FORMAT_VERSION) {
throw new IllegalStateException("Backup is broken please fix backup consistency by " +
"opening it in database and correctly closing database. " +
"Database will restore data automatically.");
}
IOUtils.readFully(archiveStream, readBuffer, 0, BackupMetadata.FILE_SIZE);
var buffer = ByteBuffer.wrap(readBuffer);
buffer.limit(BackupMetadata.FILE_SIZE);
var backupMetadata = BackupMetadata.deserialize(buffer, 0, false);
if (backupMetadata != null) {
if (dbMetadata == null) {
dbMetadata = new DbMetadata();
dbMetadata.binaryFormatVersion = EnvironmentImpl.CURRENT_FORMAT_VERSION;
dbMetadata.pageSize = backupMetadata.getPageSize();
dbMetadata.fileLengthBound = backupMetadata.getFileLengthBoundary();
metadataMap.put(rootName, dbMetadata);
} else {
if (dbMetadata.pageSize != backupMetadata.getPageSize() ||
dbMetadata.fileLengthBound != backupMetadata.getFileLengthBoundary() ||
dbMetadata.backupMetadata != null) {
throw new IllegalStateException("Backup is broken please fix backup consistency by " +
"opening it in database and correctly closing database. " +
"Database will restore data automatically.");
}
}
dbMetadata.backupMetadata = backupMetadata;
}
tarArchiveOutputStream.write(readBuffer, 0, BackupMetadata.FILE_SIZE);
} else {
IOUtils.copyLarge(archiveStream, tarArchiveOutputStream, readBuffer);
}
tarArchiveOutputStream.closeArchiveEntry();
inputEntry = archiveStream.getNextEntry();
}
} catch (final Throwable e) {
errorAwareInputStream.error = e;
logger.error("Error during backup re-encryption", e);
} finally {
try {
archiveStream.close();
} catch (IOException e) {
errorAwareInputStream.error = e;
logger.error("Error during backup re-encryption", e);
}
try {
tarArchiveOutputStream.close();
} catch (IOException e) {
errorAwareInputStream.error = e;
logger.error("Error during backup re-encryption", e);
}
}
});
return errorAwareInputStream;
}