in x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java [83:413]
public void testCreateAndRestorePartialSearchableSnapshot() throws Exception {
final String fsRepoName = randomAlphaOfLength(10);
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
final String aliasName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
createRepository(
fsRepoName,
"fs",
Settings.builder().put("location", randomRepoPath()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
);
// Peer recovery always copies .liv files but we do not permit writing to searchable snapshot directories so this doesn't work, but
// we can bypass this by forcing soft deletes to be used. TODO this restriction can be lifted when #55142 is resolved.
final Settings.Builder originalIndexSettings = Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true);
if (randomBoolean()) {
// INDEX_CHECK_ON_STARTUP requires expensive processing due to verification the integrity of many important files during
// a shard recovery or relocation. Therefore, it takes lots of time for the files to clean up and the assertShardFolder
// check may not complete in 30s.
originalIndexSettings.put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), "false");
}
assertAcked(prepareCreate(indexName, originalIndexSettings));
assertAcked(indicesAdmin().prepareAliases(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).addAlias(indexName, aliasName));
populateIndex(indexName, 10_000);
final TotalHits originalAllHits;
var originalResponse = internalCluster().client().prepareSearch(indexName).setTrackTotalHits(true).get();
try {
originalAllHits = originalResponse.getHits().getTotalHits();
} finally {
originalResponse.decRef();
}
final TotalHits originalBarHits;
var barResponse = internalCluster().client()
.prepareSearch(indexName)
.setTrackTotalHits(true)
.setQuery(matchQuery("foo", "bar"))
.get();
try {
originalBarHits = barResponse.getHits().getTotalHits();
} finally {
barResponse.decRef();
}
logger.info("--> [{}] in total, of which [{}] match the query", originalAllHits, originalBarHits);
expectThrows(
ResourceNotFoundException.class,
"Searchable snapshot stats on a non snapshot searchable index should fail",
() -> client().execute(SearchableSnapshotsStatsAction.INSTANCE, new SearchableSnapshotsStatsRequest()).actionGet()
);
final SnapshotInfo snapshotInfo = createFullSnapshot(fsRepoName, snapshotName);
ensureGreen(indexName);
assertShardFolders(indexName, false);
assertThat(
clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT)
.clear()
.setMetadata(true)
.setIndices(indexName)
.get()
.getState()
.metadata()
.getProject()
.index(indexName)
.getTimestampRange(),
sameInstance(IndexLongFieldRange.UNKNOWN)
);
final boolean deletedBeforeMount = randomBoolean();
if (deletedBeforeMount) {
assertAcked(indicesAdmin().prepareDelete(indexName));
} else {
assertAcked(indicesAdmin().prepareClose(indexName));
}
logger.info("--> restoring partial index [{}] with cache enabled", restoredIndexName);
Settings.Builder indexSettingsBuilder = Settings.builder().put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true);
if (randomBoolean()) {
var nonCachedExtensions = randomSubsetOf(Arrays.asList("fdt", "fdx", "nvd", "dvd", "tip", "cfs", "dim"));
indexSettingsBuilder.putList(SearchableSnapshots.SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING.getKey(), nonCachedExtensions);
}
if (randomBoolean()) {
indexSettingsBuilder.put(
SearchableSnapshots.SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING.getKey(),
ByteSizeValue.ofBytes(randomLongBetween(10, 100_000))
);
}
final int expectedReplicas;
if (randomBoolean()) {
expectedReplicas = numberOfReplicas();
indexSettingsBuilder.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, expectedReplicas);
} else {
expectedReplicas = 0;
}
final String indexCheckOnStartup;
if (randomBoolean()) {
indexCheckOnStartup = randomFrom("false", "true", "checksum");
indexSettingsBuilder.put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), indexCheckOnStartup);
} else {
indexCheckOnStartup = "false";
}
final String expectedDataTiersPreference;
expectedDataTiersPreference = MountSearchableSnapshotRequest.Storage.SHARED_CACHE.defaultDataTiersPreference();
indexSettingsBuilder.put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.ZERO);
final AtomicBoolean statsWatcherRunning = new AtomicBoolean(true);
final Thread statsWatcher = new Thread(() -> {
while (statsWatcherRunning.get()) {
final IndicesStatsResponse indicesStatsResponse;
try {
indicesStatsResponse = indicesAdmin().prepareStats(restoredIndexName).clear().setStore(true).get();
} catch (IndexNotFoundException | IndexClosedException e) {
continue;
// ok
}
for (ShardStats shardStats : indicesStatsResponse.getShards()) {
StoreStats store = shardStats.getStats().getStore();
assertThat(shardStats.getShardRouting().toString(), store.reservedSizeInBytes(), equalTo(0L));
assertThat(shardStats.getShardRouting().toString(), store.sizeInBytes(), equalTo(0L));
}
if (indicesStatsResponse.getShards().length > 0) {
assertThat(indicesStatsResponse.getTotal().getStore().reservedSizeInBytes(), equalTo(0L));
assertThat(indicesStatsResponse.getTotal().getStore().sizeInBytes(), equalTo(0L));
}
}
}, "test-stats-watcher");
statsWatcher.start();
final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
TEST_REQUEST_TIMEOUT,
restoredIndexName,
fsRepoName,
snapshotInfo.snapshotId().getName(),
indexName,
indexSettingsBuilder.build(),
Strings.EMPTY_ARRAY,
true,
MountSearchableSnapshotRequest.Storage.SHARED_CACHE
);
final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get();
assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
final Map<Integer, SnapshotIndexShardStatus> snapshotShards = clusterAdmin().prepareSnapshotStatus(TEST_REQUEST_TIMEOUT, fsRepoName)
.setSnapshots(snapshotInfo.snapshotId().getName())
.get()
.getSnapshots()
.get(0)
.getIndices()
.get(indexName)
.getShards();
ensureGreen(restoredIndexName);
final IndicesStatsResponse indicesStatsResponse = indicesAdmin().prepareStats(restoredIndexName).clear().setStore(true).get();
assertThat(indicesStatsResponse.getShards().length, greaterThan(0));
long totalExpectedSize = 0;
for (ShardStats shardStats : indicesStatsResponse.getShards()) {
StoreStats store = shardStats.getStats().getStore();
final ShardRouting shardRouting = shardStats.getShardRouting();
assertThat(shardRouting.toString(), store.reservedSizeInBytes(), equalTo(0L));
assertThat(shardRouting.toString(), store.sizeInBytes(), equalTo(0L));
// the original shard size from the snapshot
final long originalSize = snapshotShards.get(shardRouting.getId()).getStats().getTotalSize();
assertThat(shardRouting.toString(), store.totalDataSetSizeInBytes(), equalTo(originalSize));
totalExpectedSize += originalSize;
}
final StoreStats store = indicesStatsResponse.getTotal().getStore();
assertThat(store.totalDataSetSizeInBytes(), equalTo(totalExpectedSize));
statsWatcherRunning.set(false);
statsWatcher.join();
final Settings settings = indicesAdmin().prepareGetSettings(TEST_REQUEST_TIMEOUT, restoredIndexName)
.get()
.getIndexToSettings()
.get(restoredIndexName);
assertThat(SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING.get(settings), equalTo(snapshotName));
assertThat(IndexModule.INDEX_STORE_TYPE_SETTING.get(settings), equalTo(SEARCHABLE_SNAPSHOT_STORE_TYPE));
assertThat(IndexModule.INDEX_RECOVERY_TYPE_SETTING.get(settings), equalTo(SNAPSHOT_RECOVERY_STATE_FACTORY_KEY));
assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(settings));
assertTrue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.exists(settings));
assertTrue(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.exists(settings));
assertThat(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(settings).toString(), equalTo("false"));
assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings), equalTo(expectedReplicas));
assertThat(DataTier.TIER_PREFERENCE_SETTING.get(settings), equalTo(expectedDataTiersPreference));
assertTrue(SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING.get(settings));
assertTrue(DiskThresholdDecider.SETTING_IGNORE_DISK_WATERMARKS.get(settings));
assertThat(IndexSettings.INDEX_CHECK_ON_STARTUP.get(settings), equalTo(indexCheckOnStartup));
checkSoftDeletesNotEagerlyLoaded(restoredIndexName);
assertTotalHits(restoredIndexName, originalAllHits, originalBarHits);
assertRecoveryStats(restoredIndexName, false);
// TODO: fix
// assertSearchableSnapshotStats(restoredIndexName, true, nonCachedExtensions);
ensureGreen(restoredIndexName);
assertBusy(() -> assertShardFolders(restoredIndexName, true), 30, TimeUnit.SECONDS);
assertThat(
clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT)
.clear()
.setMetadata(true)
.setIndices(restoredIndexName)
.get()
.getState()
.metadata()
.getProject()
.index(restoredIndexName)
.getTimestampRange(),
sameInstance(IndexLongFieldRange.UNKNOWN)
);
if (deletedBeforeMount) {
assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(0));
assertAcked(indicesAdmin().prepareAliases(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).addAlias(restoredIndexName, aliasName));
} else if (indexName.equals(restoredIndexName) == false) {
assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(1));
assertAcked(
indicesAdmin().prepareAliases(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(indexName).alias(aliasName).mustExist(true))
.addAlias(restoredIndexName, aliasName)
);
}
assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(1));
assertTotalHits(aliasName, originalAllHits, originalBarHits);
final var request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT).setIndex(restoredIndexName)
.setShard(0)
.setPrimary(true);
request.includeYesDecisions(true);
final var diskDeciderDecision = safeGet(client().execute(TransportClusterAllocationExplainAction.TYPE, request)).getExplanation()
.getShardAllocationDecision()
.getMoveDecision()
.getCanRemainDecision()
.getDecisions()
.stream()
.filter(d -> d.label().equals(DiskThresholdDecider.NAME))
.findFirst()
.orElseThrow();
assertThat(diskDeciderDecision.type(), equalTo(Decision.Type.YES));
assertThat(
diskDeciderDecision.getExplanation(),
oneOf("disk watermarks are ignored on this index", "there is only a single data node present")
);
internalCluster().fullRestart();
assertTotalHits(restoredIndexName, originalAllHits, originalBarHits);
assertRecoveryStats(restoredIndexName, false);
assertTotalHits(aliasName, originalAllHits, originalBarHits);
// TODO: fix
// assertSearchableSnapshotStats(restoredIndexName, false, nonCachedExtensions);
internalCluster().ensureAtLeastNumDataNodes(2);
final DiscoveryNode dataNode = randomFrom(
clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getDataNodes().values()
);
updateIndexSettings(
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(
IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(),
dataNode.getName()
),
restoredIndexName
);
assertFalse(
clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, restoredIndexName)
.setWaitForNoRelocatingShards(true)
.setWaitForEvents(Priority.LANGUID)
.get()
.isTimedOut()
);
assertTotalHits(restoredIndexName, originalAllHits, originalBarHits);
assertRecoveryStats(restoredIndexName, false);
// TODO: fix
// assertSearchableSnapshotStats(restoredIndexName, false, nonCachedExtensions);
updateIndexSettings(
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
.putNull(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey()),
restoredIndexName
);
assertTotalHits(restoredIndexName, originalAllHits, originalBarHits);
assertRecoveryStats(restoredIndexName, false);
final String clonedIndexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
assertAcked(
indicesAdmin().prepareResizeIndex(restoredIndexName, clonedIndexName)
.setResizeType(ResizeType.CLONE)
.setSettings(
Settings.builder()
.putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey())
.putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey())
.put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT)
.build()
)
);
ensureGreen(clonedIndexName);
assertTotalHits(clonedIndexName, originalAllHits, originalBarHits);
final Settings clonedIndexSettings = indicesAdmin().prepareGetSettings(TEST_REQUEST_TIMEOUT, clonedIndexName)
.get()
.getIndexToSettings()
.get(clonedIndexName);
assertFalse(clonedIndexSettings.hasValue(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()));
assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING.getKey()));
assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.getKey()));
assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.getKey()));
assertFalse(clonedIndexSettings.hasValue(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()));
assertAcked(indicesAdmin().prepareDelete(restoredIndexName));
assertThat(indicesAdmin().prepareGetAliases(TEST_REQUEST_TIMEOUT, aliasName).get().getAliases().size(), equalTo(0));
assertAcked(indicesAdmin().prepareAliases(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).addAlias(clonedIndexName, aliasName));
assertTotalHits(aliasName, originalAllHits, originalBarHits);
}