in server/src/main/java/org/opensearch/snapshots/RestoreService.java [203:721]
public void restoreSnapshot(final RestoreSnapshotRequest request, final ActionListener<RestoreCompletionResponse> listener) {
try {
// Read snapshot info and metadata from the repository
final String repositoryName = request.repository();
Repository repository = repositoriesService.repository(repositoryName);
final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
repository.getRepositoryData(repositoryDataListener);
repositoryDataListener.whenComplete(repositoryData -> {
final String snapshotName = request.snapshot();
final Optional<SnapshotId> matchingSnapshotId = repositoryData.getSnapshotIds()
.stream()
.filter(s -> snapshotName.equals(s.getName()))
.findFirst();
if (matchingSnapshotId.isPresent() == false) {
throw new SnapshotRestoreException(repositoryName, snapshotName, "snapshot does not exist");
}
final SnapshotId snapshotId = matchingSnapshotId.get();
if (request.snapshotUuid() != null && request.snapshotUuid().equals(snapshotId.getUUID()) == false) {
throw new SnapshotRestoreException(
repositoryName,
snapshotName,
"snapshot UUID mismatch: expected [" + request.snapshotUuid() + "] but got [" + snapshotId.getUUID() + "]"
);
}
final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
final Snapshot snapshot = new Snapshot(repositoryName, snapshotId);
// Make sure that we can restore from this snapshot
validateSnapshotRestorable(repositoryName, snapshotInfo);
Metadata globalMetadata = null;
// Resolve the indices from the snapshot that need to be restored
Map<String, DataStream> dataStreams;
List<String> requestIndices = new ArrayList<>(Arrays.asList(request.indices()));
List<String> requestedDataStreams = filterIndices(
snapshotInfo.dataStreams(),
requestIndices.toArray(new String[0]),
IndicesOptions.fromOptions(true, true, true, true)
);
if (requestedDataStreams.isEmpty()) {
dataStreams = new HashMap<>();
} else {
globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId);
final Map<String, DataStream> dataStreamsInSnapshot = globalMetadata.dataStreams();
dataStreams = new HashMap<>(requestedDataStreams.size());
for (String requestedDataStream : requestedDataStreams) {
final DataStream dataStreamInSnapshot = dataStreamsInSnapshot.get(requestedDataStream);
assert dataStreamInSnapshot != null : "DataStream [" + requestedDataStream + "] not found in snapshot";
dataStreams.put(requestedDataStream, dataStreamInSnapshot);
}
}
requestIndices.removeAll(dataStreams.keySet());
Set<String> dataStreamIndices = dataStreams.values()
.stream()
.flatMap(ds -> ds.getIndices().stream())
.map(Index::getName)
.collect(Collectors.toSet());
requestIndices.addAll(dataStreamIndices);
final List<String> indicesInSnapshot = filterIndices(
snapshotInfo.indices(),
requestIndices.toArray(new String[0]),
request.indicesOptions()
);
final Metadata.Builder metadataBuilder;
if (request.includeGlobalState()) {
if (globalMetadata == null) {
globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId);
}
metadataBuilder = Metadata.builder(globalMetadata);
} else {
metadataBuilder = Metadata.builder();
}
final List<IndexId> indexIdsInSnapshot = repositoryData.resolveIndices(indicesInSnapshot);
for (IndexId indexId : indexIdsInSnapshot) {
metadataBuilder.put(repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId), false);
}
final Metadata metadata = metadataBuilder.build();
// Apply renaming on index names, returning a map of names where
// the key is the renamed index and the value is the original name
final Map<String, String> indices = renamedIndices(request, indicesInSnapshot, dataStreamIndices);
// Now we can start the actual restore process by adding shards to be recovered in the cluster state
// and updating cluster metadata (global and index) as needed
clusterService.submitStateUpdateTask("restore_snapshot[" + snapshotName + ']', new ClusterStateUpdateTask() {
final String restoreUUID = UUIDs.randomBase64UUID();
RestoreInfo restoreInfo = null;
@Override
public ClusterState execute(ClusterState currentState) {
RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY);
if (currentState.getNodes().getMinNodeVersion().before(LegacyESVersion.V_7_0_0)) {
// Check if another restore process is already running - cannot run two restore processes at the
// same time in versions prior to 7.0
if (restoreInProgress.isEmpty() == false) {
throw new ConcurrentSnapshotExecutionException(
snapshot,
"Restore process is already running in this cluster"
);
}
}
// Check if the snapshot to restore is currently being deleted
SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(
SnapshotDeletionsInProgress.TYPE,
SnapshotDeletionsInProgress.EMPTY
);
if (deletionsInProgress.getEntries().stream().anyMatch(entry -> entry.getSnapshots().contains(snapshotId))) {
throw new ConcurrentSnapshotExecutionException(
snapshot,
"cannot restore a snapshot while a snapshot deletion is in-progress ["
+ deletionsInProgress.getEntries().get(0)
+ "]"
);
}
// Updating cluster state
ClusterState.Builder builder = ClusterState.builder(currentState);
Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata());
ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards;
Set<String> aliases = new HashSet<>();
if (indices.isEmpty() == false) {
// We have some indices to restore
ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shardsBuilder = ImmutableOpenMap
.builder();
final Version minIndexCompatibilityVersion = currentState.getNodes()
.getMaxNodeVersion()
.minimumIndexCompatibilityVersion();
for (Map.Entry<String, String> indexEntry : indices.entrySet()) {
String index = indexEntry.getValue();
boolean partial = checkPartial(index);
SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(
restoreUUID,
snapshot,
snapshotInfo.version(),
repositoryData.resolveIndexId(index)
);
String renamedIndexName = indexEntry.getKey();
IndexMetadata snapshotIndexMetadata = metadata.index(index);
snapshotIndexMetadata = updateIndexSettings(
snapshotIndexMetadata,
request.indexSettings(),
request.ignoreIndexSettings()
);
try {
snapshotIndexMetadata = metadataIndexUpgradeService.upgradeIndexMetadata(
snapshotIndexMetadata,
minIndexCompatibilityVersion
);
} catch (Exception ex) {
throw new SnapshotRestoreException(
snapshot,
"cannot restore index [" + index + "] because it cannot be upgraded",
ex
);
}
// Check that the index is closed or doesn't exist
IndexMetadata currentIndexMetadata = currentState.metadata().index(renamedIndexName);
IntSet ignoreShards = new IntHashSet();
final Index renamedIndex;
if (currentIndexMetadata == null) {
// Index doesn't exist - create it and start recovery
// Make sure that the index we are about to create has a validate name
boolean isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(snapshotIndexMetadata.getSettings());
createIndexService.validateIndexName(renamedIndexName, currentState);
createIndexService.validateDotIndex(renamedIndexName, isHidden);
createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetadata.getSettings(), false);
IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata)
.state(IndexMetadata.State.OPEN)
.index(renamedIndexName);
indexMdBuilder.settings(
Settings.builder()
.put(snapshotIndexMetadata.getSettings())
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
);
shardLimitValidator.validateShardLimit(snapshotIndexMetadata.getSettings(), currentState);
if (!request.includeAliases() && !snapshotIndexMetadata.getAliases().isEmpty()) {
// Remove all aliases - they shouldn't be restored
indexMdBuilder.removeAllAliases();
} else {
for (ObjectCursor<String> alias : snapshotIndexMetadata.getAliases().keys()) {
aliases.add(alias.value);
}
}
IndexMetadata updatedIndexMetadata = indexMdBuilder.build();
if (partial) {
populateIgnoredShards(index, ignoreShards);
}
rtBuilder.addAsNewRestore(updatedIndexMetadata, recoverySource, ignoreShards);
blocks.addBlocks(updatedIndexMetadata);
mdBuilder.put(updatedIndexMetadata, true);
renamedIndex = updatedIndexMetadata.getIndex();
} else {
validateExistingIndex(currentIndexMetadata, snapshotIndexMetadata, renamedIndexName, partial);
// Index exists and it's closed - open it in metadata and start recovery
IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata)
.state(IndexMetadata.State.OPEN);
indexMdBuilder.version(
Math.max(snapshotIndexMetadata.getVersion(), 1 + currentIndexMetadata.getVersion())
);
indexMdBuilder.mappingVersion(
Math.max(snapshotIndexMetadata.getMappingVersion(), 1 + currentIndexMetadata.getMappingVersion())
);
indexMdBuilder.settingsVersion(
Math.max(snapshotIndexMetadata.getSettingsVersion(), 1 + currentIndexMetadata.getSettingsVersion())
);
indexMdBuilder.aliasesVersion(
Math.max(snapshotIndexMetadata.getAliasesVersion(), 1 + currentIndexMetadata.getAliasesVersion())
);
for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) {
indexMdBuilder.primaryTerm(
shard,
Math.max(snapshotIndexMetadata.primaryTerm(shard), currentIndexMetadata.primaryTerm(shard))
);
}
if (!request.includeAliases()) {
// Remove all snapshot aliases
if (!snapshotIndexMetadata.getAliases().isEmpty()) {
indexMdBuilder.removeAllAliases();
}
/// Add existing aliases
for (ObjectCursor<AliasMetadata> alias : currentIndexMetadata.getAliases().values()) {
indexMdBuilder.putAlias(alias.value);
}
} else {
for (ObjectCursor<String> alias : snapshotIndexMetadata.getAliases().keys()) {
aliases.add(alias.value);
}
}
final Settings.Builder indexSettingsBuilder = Settings.builder()
.put(snapshotIndexMetadata.getSettings())
.put(IndexMetadata.SETTING_INDEX_UUID, currentIndexMetadata.getIndexUUID());
// Only add a restore uuid if either all nodes in the cluster support it (version >= 7.9) or if the
// index itself was created after 7.9 and thus won't be restored to a node that doesn't support the
// setting anyway
if (snapshotIndexMetadata.getCreationVersion().onOrAfter(LegacyESVersion.V_7_9_0)
|| currentState.nodes().getMinNodeVersion().onOrAfter(LegacyESVersion.V_7_9_0)) {
indexSettingsBuilder.put(SETTING_HISTORY_UUID, UUIDs.randomBase64UUID());
}
indexMdBuilder.settings(indexSettingsBuilder);
IndexMetadata updatedIndexMetadata = indexMdBuilder.index(renamedIndexName).build();
rtBuilder.addAsRestore(updatedIndexMetadata, recoverySource);
blocks.updateBlocks(updatedIndexMetadata);
mdBuilder.put(updatedIndexMetadata, true);
renamedIndex = updatedIndexMetadata.getIndex();
}
for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) {
if (!ignoreShards.contains(shard)) {
shardsBuilder.put(
new ShardId(renamedIndex, shard),
new RestoreInProgress.ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId())
);
} else {
shardsBuilder.put(
new ShardId(renamedIndex, shard),
new RestoreInProgress.ShardRestoreStatus(
clusterService.state().nodes().getLocalNodeId(),
RestoreInProgress.State.FAILURE
)
);
}
}
}
shards = shardsBuilder.build();
RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(
restoreUUID,
snapshot,
overallState(RestoreInProgress.State.INIT, shards),
Collections.unmodifiableList(new ArrayList<>(indices.keySet())),
shards
);
builder.putCustom(
RestoreInProgress.TYPE,
new RestoreInProgress.Builder(currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)).add(
restoreEntry
).build()
);
} else {
shards = ImmutableOpenMap.of();
}
checkAliasNameConflicts(indices, aliases);
Map<String, DataStream> updatedDataStreams = new HashMap<>(currentState.metadata().dataStreams());
updatedDataStreams.putAll(
dataStreams.values()
.stream()
.map(ds -> updateDataStream(ds, mdBuilder, request))
.collect(Collectors.toMap(DataStream::getName, Function.identity()))
);
mdBuilder.dataStreams(updatedDataStreams);
// Restore global state if needed
if (request.includeGlobalState()) {
if (metadata.persistentSettings() != null) {
Settings settings = metadata.persistentSettings();
clusterSettings.validateUpdate(settings);
mdBuilder.persistentSettings(settings);
}
if (metadata.templates() != null) {
// TODO: Should all existing templates be deleted first?
for (ObjectCursor<IndexTemplateMetadata> cursor : metadata.templates().values()) {
mdBuilder.put(cursor.value);
}
}
if (metadata.customs() != null) {
for (ObjectObjectCursor<String, Metadata.Custom> cursor : metadata.customs()) {
if (RepositoriesMetadata.TYPE.equals(cursor.key) == false
&& DataStreamMetadata.TYPE.equals(cursor.key) == false) {
// Don't restore repositories while we are working with them
// TODO: Should we restore them at the end?
// Also, don't restore data streams here, we already added them to the metadata builder above
mdBuilder.putCustom(cursor.key, cursor.value);
}
}
}
}
if (completed(shards)) {
// We don't have any indices to restore - we are done
restoreInfo = new RestoreInfo(
snapshotId.getName(),
Collections.unmodifiableList(new ArrayList<>(indices.keySet())),
shards.size(),
shards.size() - failedShards(shards)
);
}
RoutingTable rt = rtBuilder.build();
ClusterState updatedState = builder.metadata(mdBuilder).blocks(blocks).routingTable(rt).build();
return allocationService.reroute(updatedState, "restored snapshot [" + snapshot + "]");
}
private void checkAliasNameConflicts(Map<String, String> renamedIndices, Set<String> aliases) {
for (Map.Entry<String, String> renamedIndex : renamedIndices.entrySet()) {
if (aliases.contains(renamedIndex.getKey())) {
throw new SnapshotRestoreException(
snapshot,
"cannot rename index ["
+ renamedIndex.getValue()
+ "] into ["
+ renamedIndex.getKey()
+ "] because of conflict with an alias with the same name"
);
}
}
}
private void populateIgnoredShards(String index, IntSet ignoreShards) {
for (SnapshotShardFailure failure : snapshotInfo.shardFailures()) {
if (index.equals(failure.index())) {
ignoreShards.add(failure.shardId());
}
}
}
private boolean checkPartial(String index) {
// Make sure that index was fully snapshotted
if (failed(snapshotInfo, index)) {
if (request.partial()) {
return true;
} else {
throw new SnapshotRestoreException(
snapshot,
"index [" + index + "] wasn't fully snapshotted - cannot " + "restore"
);
}
} else {
return false;
}
}
private void validateExistingIndex(
IndexMetadata currentIndexMetadata,
IndexMetadata snapshotIndexMetadata,
String renamedIndex,
boolean partial
) {
// Index exist - checking that it's closed
if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) {
// TODO: Enable restore for open indices
throw new SnapshotRestoreException(
snapshot,
"cannot restore index ["
+ renamedIndex
+ "] because an open index "
+ "with same name already exists in the cluster. Either close or delete the existing index or restore the "
+ "index under a different name by providing a rename pattern and replacement name"
);
}
// Index exist - checking if it's partial restore
if (partial) {
throw new SnapshotRestoreException(
snapshot,
"cannot restore partial index [" + renamedIndex + "] because such index already exists"
);
}
// Make sure that the number of shards is the same. That's the only thing that we cannot change
if (currentIndexMetadata.getNumberOfShards() != snapshotIndexMetadata.getNumberOfShards()) {
throw new SnapshotRestoreException(
snapshot,
"cannot restore index ["
+ renamedIndex
+ "] with ["
+ currentIndexMetadata.getNumberOfShards()
+ "] shards from a snapshot of index ["
+ snapshotIndexMetadata.getIndex().getName()
+ "] with ["
+ snapshotIndexMetadata.getNumberOfShards()
+ "] shards"
);
}
}
/**
* Optionally updates index settings in indexMetadata by removing settings listed in ignoreSettings and
* merging them with settings in changeSettings.
*/
private IndexMetadata updateIndexSettings(
IndexMetadata indexMetadata,
Settings changeSettings,
String[] ignoreSettings
) {
Settings normalizedChangeSettings = Settings.builder()
.put(changeSettings)
.normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX)
.build();
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetadata.getSettings())
&& IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(changeSettings)
&& IndexSettings.INDEX_SOFT_DELETES_SETTING.get(changeSettings) == false) {
throw new SnapshotRestoreException(
snapshot,
"cannot disable setting [" + IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey() + "] on restore"
);
}
IndexMetadata.Builder builder = IndexMetadata.builder(indexMetadata);
Settings settings = indexMetadata.getSettings();
Set<String> keyFilters = new HashSet<>();
List<String> simpleMatchPatterns = new ArrayList<>();
for (String ignoredSetting : ignoreSettings) {
if (!Regex.isSimpleMatchPattern(ignoredSetting)) {
if (UNREMOVABLE_SETTINGS.contains(ignoredSetting)) {
throw new SnapshotRestoreException(
snapshot,
"cannot remove setting [" + ignoredSetting + "] on restore"
);
} else {
keyFilters.add(ignoredSetting);
}
} else {
simpleMatchPatterns.add(ignoredSetting);
}
}
Predicate<String> settingsFilter = k -> {
if (UNREMOVABLE_SETTINGS.contains(k) == false) {
for (String filterKey : keyFilters) {
if (k.equals(filterKey)) {
return false;
}
}
for (String pattern : simpleMatchPatterns) {
if (Regex.simpleMatch(pattern, k)) {
return false;
}
}
}
return true;
};
Settings.Builder settingsBuilder = Settings.builder()
.put(settings.filter(settingsFilter))
.put(normalizedChangeSettings.filter(k -> {
if (UNMODIFIABLE_SETTINGS.contains(k)) {
throw new SnapshotRestoreException(snapshot, "cannot modify setting [" + k + "] on restore");
} else {
return true;
}
}));
settingsBuilder.remove(MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey());
return builder.settings(settingsBuilder).build();
}
@Override
public void onFailure(String source, Exception e) {
logger.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", snapshotId), e);
listener.onFailure(e);
}
@Override
public TimeValue timeout() {
return request.masterNodeTimeout();
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
listener.onResponse(new RestoreCompletionResponse(restoreUUID, snapshot, restoreInfo));
}
});
}, listener::onFailure);
} catch (Exception e) {
logger.warn(
() -> new ParameterizedMessage("[{}] failed to restore snapshot", request.repository() + ":" + request.snapshot()),
e
);
listener.onFailure(e);
}
}