in fdbbackup/backup.actor.cpp [3147:4374]
int main(int argc, char* argv[]) {
platformInit();
int status = FDB_EXIT_SUCCESS;
std::string commandLine;
for (int a = 0; a < argc; a++) {
if (a)
commandLine += ' ';
commandLine += argv[a];
}
try {
#ifdef ALLOC_INSTRUMENTATION
g_extra_memory = new uint8_t[1000000];
#endif
registerCrashHandler();
// Set default of line buffering standard out and error
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
ProgramExe programExe = getProgramType(argv[0]);
BackupType backupType = BackupType::UNDEFINED;
RestoreType restoreType = RestoreType::UNKNOWN;
DBType dbType = DBType::UNDEFINED;
std::unique_ptr<CSimpleOpt> args;
switch (programExe) {
case ProgramExe::AGENT:
args = std::make_unique<CSimpleOpt>(argc, argv, g_rgAgentOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case ProgramExe::DR_AGENT:
args = std::make_unique<CSimpleOpt>(argc, argv, g_rgDBAgentOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case ProgramExe::BACKUP:
// Display backup help, if no arguments
if (argc < 2) {
printBackupUsage(false);
return FDB_EXIT_ERROR;
} else {
// Get the backup type
backupType = getBackupType(argv[1]);
// Create the appropriate simple opt
switch (backupType) {
case BackupType::START:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupStartOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::STATUS:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupStatusOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::ABORT:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupAbortOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::CLEANUP:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupCleanupOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::WAIT:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupWaitOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::DISCONTINUE:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupDiscontinueOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::PAUSE:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupPauseOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::RESUME:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupPauseOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::EXPIRE:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupExpireOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::DELETE_BACKUP:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupDeleteOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::DESCRIBE:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupDescribeOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::DUMP:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupDumpOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::LIST:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupListOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::QUERY:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupQueryOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::MODIFY:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupModifyOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::TAGS:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgBackupTagsOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case BackupType::UNDEFINED:
default:
args =
std::make_unique<CSimpleOpt>(argc, argv, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
}
}
break;
case ProgramExe::DB_BACKUP:
// Display backup help, if no arguments
if (argc < 2) {
printDBBackupUsage(false);
return FDB_EXIT_ERROR;
} else {
// Get the backup type
dbType = getDBType(argv[1]);
// Create the appropriate simple opt
switch (dbType) {
case DBType::START:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgDBStartOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case DBType::STATUS:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgDBStatusOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case DBType::SWITCH:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgDBSwitchOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case DBType::ABORT:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgDBAbortOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case DBType::PAUSE:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgDBPauseOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case DBType::RESUME:
args = std::make_unique<CSimpleOpt>(
argc - 1, &argv[1], g_rgDBPauseOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
case DBType::UNDEFINED:
default:
args =
std::make_unique<CSimpleOpt>(argc, argv, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
break;
}
}
break;
case ProgramExe::RESTORE:
if (argc < 2) {
printRestoreUsage(false);
return FDB_EXIT_ERROR;
}
// Get the restore operation type
restoreType = getRestoreType(argv[1]);
if (restoreType == RestoreType::UNKNOWN) {
args = std::make_unique<CSimpleOpt>(argc, argv, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
} else {
args = std::make_unique<CSimpleOpt>(
argc - 1, argv + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
}
break;
case ProgramExe::FASTRESTORE_TOOL:
if (argc < 2) {
printFastRestoreUsage(false);
return FDB_EXIT_ERROR;
}
// Get the restore operation type
restoreType = getRestoreType(argv[1]);
if (restoreType == RestoreType::UNKNOWN) {
args = std::make_unique<CSimpleOpt>(argc, argv, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
} else {
args = std::make_unique<CSimpleOpt>(
argc - 1, argv + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
}
break;
case ProgramExe::UNDEFINED:
default:
fprintf(stderr, "FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
fprintf(stderr, "ERROR: Unable to determine program type based on executable `%s'\n", argv[0]);
return FDB_EXIT_ERROR;
break;
}
std::string destinationContainer;
bool describeDeep = false;
bool describeTimestamps = false;
int initialSnapshotIntervalSeconds =
0; // The initial snapshot has a desired duration of 0, meaning go as fast as possible.
int snapshotIntervalSeconds = CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC;
std::string clusterFile;
std::string sourceClusterFile;
std::string baseUrl;
std::string expireDatetime;
Version expireVersion = invalidVersion;
std::string expireRestorableAfterDatetime;
Version expireRestorableAfterVersion = std::numeric_limits<Version>::max();
std::vector<std::pair<std::string, std::string>> knobs;
std::string tagName = BackupAgentBase::getDefaultTag().toString();
bool tagProvided = false;
std::string restoreContainer;
std::string addPrefix;
std::string removePrefix;
Standalone<VectorRef<KeyRangeRef>> backupKeys;
Standalone<VectorRef<KeyRangeRef>> backupKeysFilter;
int maxErrors = 20;
Version beginVersion = invalidVersion;
Version restoreVersion = invalidVersion;
std::string restoreTimestamp;
WaitForComplete waitForDone{ false };
StopWhenDone stopWhenDone{ true };
UsePartitionedLog usePartitionedLog{ false }; // Set to true to use new backup system
IncrementalBackupOnly incrementalBackupOnly{ false };
OnlyApplyMutationLogs onlyApplyMutationLogs{ false };
InconsistentSnapshotOnly inconsistentSnapshotOnly{ false };
ForceAction forceAction{ false };
bool trace = false;
bool quietDisplay = false;
bool dryRun = false;
std::string traceDir = "";
std::string traceFormat = "";
std::string traceLogGroup;
uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE;
uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE;
ESOError lastError;
PartialBackup partial{ true };
DstOnly dstOnly{ false };
LocalityData localities;
uint64_t memLimit = 8LL << 30;
Optional<uint64_t> ti;
BackupTLSConfig tlsConfig;
Version dumpBegin = 0;
Version dumpEnd = std::numeric_limits<Version>::max();
std::string restoreClusterFileDest;
std::string restoreClusterFileOrig;
bool jsonOutput = false;
DeleteData deleteData{ false };
Optional<std::string> encryptionKeyFile;
BackupModifyOptions modifyOptions;
if (argc == 1) {
printUsage(programExe, false);
return FDB_EXIT_ERROR;
}
#ifdef _WIN32
// Windows needs a gentle nudge to format floats correctly
//_set_output_format(_TWO_DIGIT_EXPONENT);
#endif
while (args->Next()) {
lastError = args->LastError();
switch (lastError) {
case SO_SUCCESS:
break;
case SO_ARG_INVALID_DATA:
fprintf(stderr, "ERROR: invalid argument to option `%s'\n", args->OptionText());
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
case SO_ARG_INVALID:
fprintf(stderr, "ERROR: argument given for option `%s'\n", args->OptionText());
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
case SO_ARG_MISSING:
fprintf(stderr, "ERROR: missing argument for option `%s'\n", args->OptionText());
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
case SO_OPT_INVALID:
fprintf(stderr, "ERROR: unknown option `%s'\n", args->OptionText());
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
default:
fprintf(stderr, "ERROR: argument given for option `%s'\n", args->OptionText());
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
}
int optId = args->OptionId();
switch (optId) {
case OPT_HELP:
printUsage(programExe, false);
return FDB_EXIT_SUCCESS;
break;
case OPT_DEVHELP:
printUsage(programExe, true);
return FDB_EXIT_SUCCESS;
break;
case OPT_VERSION:
printVersion();
return FDB_EXIT_SUCCESS;
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
return FDB_EXIT_SUCCESS;
break;
case OPT_NOBUFSTDOUT:
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
break;
case OPT_BUFSTDOUTERR:
setvbuf(stdout, NULL, _IOFBF, BUFSIZ);
setvbuf(stderr, NULL, _IOFBF, BUFSIZ);
break;
case OPT_QUIET:
quietDisplay = true;
break;
case OPT_DRYRUN:
dryRun = true;
break;
case OPT_DELETE_DATA:
deleteData.set(true);
break;
case OPT_MIN_CLEANUP_SECONDS:
knobs.emplace_back("min_cleanup_seconds", args->OptionArg());
break;
case OPT_FORCE:
forceAction.set(true);
break;
case OPT_TRACE:
trace = true;
break;
case OPT_TRACE_DIR:
trace = true;
traceDir = args->OptionArg();
break;
case OPT_TRACE_FORMAT:
if (!validateTraceFormat(args->OptionArg())) {
fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args->OptionArg());
}
traceFormat = args->OptionArg();
break;
case OPT_TRACE_LOG_GROUP:
traceLogGroup = args->OptionArg();
break;
case OPT_LOCALITY: {
Optional<std::string> localityKey = extractPrefixedArgument("--locality", args->OptionSyntax());
if (!localityKey.present()) {
fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", args->OptionSyntax());
return FDB_EXIT_ERROR;
}
Standalone<StringRef> key = StringRef(localityKey.get());
std::transform(key.begin(), key.end(), mutateString(key), ::tolower);
localities.set(key, Standalone<StringRef>(std::string(args->OptionArg())));
break;
}
case OPT_EXPIRE_BEFORE_DATETIME:
expireDatetime = args->OptionArg();
break;
case OPT_EXPIRE_RESTORABLE_AFTER_DATETIME:
expireRestorableAfterDatetime = args->OptionArg();
break;
case OPT_EXPIRE_BEFORE_VERSION:
case OPT_EXPIRE_RESTORABLE_AFTER_VERSION:
case OPT_EXPIRE_MIN_RESTORABLE_DAYS:
case OPT_EXPIRE_DELETE_BEFORE_DAYS: {
const char* a = args->OptionArg();
long long ver = 0;
if (!sscanf(a, "%lld", &ver)) {
fprintf(stderr, "ERROR: Could not parse expiration version `%s'\n", a);
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
// Interpret the value as days worth of versions relative to now (negative)
if (optId == OPT_EXPIRE_MIN_RESTORABLE_DAYS || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) {
ver = -ver * 24 * 60 * 60 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
}
if (optId == OPT_EXPIRE_BEFORE_VERSION || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS)
expireVersion = ver;
else
expireRestorableAfterVersion = ver;
break;
}
case OPT_RESTORE_TIMESTAMP:
restoreTimestamp = args->OptionArg();
break;
case OPT_BASEURL:
baseUrl = args->OptionArg();
break;
case OPT_RESTORE_CLUSTERFILE_DEST:
restoreClusterFileDest = args->OptionArg();
break;
case OPT_RESTORE_CLUSTERFILE_ORIG:
restoreClusterFileOrig = args->OptionArg();
break;
case OPT_CLUSTERFILE:
clusterFile = args->OptionArg();
break;
case OPT_DEST_CLUSTER:
clusterFile = args->OptionArg();
break;
case OPT_SOURCE_CLUSTER:
sourceClusterFile = args->OptionArg();
break;
case OPT_CLEANUP:
partial.set(false);
break;
case OPT_DSTONLY:
dstOnly.set(true);
break;
case OPT_KNOB: {
Optional<std::string> knobName = extractPrefixedArgument("--knob", args->OptionSyntax());
if (!knobName.present()) {
fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", args->OptionSyntax());
return FDB_EXIT_ERROR;
}
knobs.emplace_back(knobName.get(), args->OptionArg());
break;
}
case OPT_BACKUPKEYS:
try {
addKeyRange(args->OptionArg(), backupKeys);
} catch (Error&) {
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
break;
case OPT_BACKUPKEYS_FILTER:
try {
addKeyRange(args->OptionArg(), backupKeysFilter);
} catch (Error&) {
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
break;
case OPT_DESTCONTAINER:
destinationContainer = args->OptionArg();
// If the url starts with '/' then prepend "file://" for backwards compatibility
if (StringRef(destinationContainer).startsWith(LiteralStringRef("/")))
destinationContainer = std::string("file://") + destinationContainer;
modifyOptions.destURL = destinationContainer;
break;
case OPT_SNAPSHOTINTERVAL:
case OPT_INITIAL_SNAPSHOT_INTERVAL:
case OPT_MOD_ACTIVE_INTERVAL: {
const char* a = args->OptionArg();
int seconds;
if (!sscanf(a, "%d", &seconds)) {
fprintf(stderr, "ERROR: Could not parse snapshot interval `%s'\n", a);
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
if (optId == OPT_SNAPSHOTINTERVAL) {
snapshotIntervalSeconds = seconds;
modifyOptions.snapshotIntervalSeconds = seconds;
} else if (optId == OPT_INITIAL_SNAPSHOT_INTERVAL) {
initialSnapshotIntervalSeconds = seconds;
} else if (optId == OPT_MOD_ACTIVE_INTERVAL) {
modifyOptions.activeSnapshotIntervalSeconds = seconds;
}
break;
}
case OPT_MOD_VERIFY_UID:
modifyOptions.verifyUID = args->OptionArg();
break;
case OPT_WAITFORDONE:
waitForDone.set(true);
break;
case OPT_NOSTOPWHENDONE:
stopWhenDone.set(false);
break;
case OPT_USE_PARTITIONED_LOG:
usePartitionedLog.set(true);
break;
case OPT_INCREMENTALONLY:
incrementalBackupOnly.set(true);
onlyApplyMutationLogs.set(true);
break;
case OPT_ENCRYPTION_KEY_FILE:
encryptionKeyFile = args->OptionArg();
break;
case OPT_RESTORECONTAINER:
restoreContainer = args->OptionArg();
// If the url starts with '/' then prepend "file://" for backwards compatibility
if (StringRef(restoreContainer).startsWith(LiteralStringRef("/")))
restoreContainer = std::string("file://") + restoreContainer;
break;
case OPT_DESCRIBE_DEEP:
describeDeep = true;
break;
case OPT_DESCRIBE_TIMESTAMPS:
describeTimestamps = true;
break;
case OPT_PREFIX_ADD:
addPrefix = args->OptionArg();
break;
case OPT_PREFIX_REMOVE:
removePrefix = args->OptionArg();
break;
case OPT_ERRORLIMIT: {
const char* a = args->OptionArg();
if (!sscanf(a, "%d", &maxErrors)) {
fprintf(stderr, "ERROR: Could not parse max number of errors `%s'\n", a);
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
break;
}
case OPT_RESTORE_BEGIN_VERSION: {
const char* a = args->OptionArg();
long long ver = 0;
if (!sscanf(a, "%lld", &ver)) {
fprintf(stderr, "ERROR: Could not parse database beginVersion `%s'\n", a);
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
beginVersion = ver;
break;
}
case OPT_RESTORE_VERSION: {
const char* a = args->OptionArg();
long long ver = 0;
if (!sscanf(a, "%lld", &ver)) {
fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a);
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
restoreVersion = ver;
break;
}
case OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY: {
inconsistentSnapshotOnly.set(true);
break;
}
#ifdef _WIN32
case OPT_PARENTPID: {
auto pid_str = args->OptionArg();
int parent_pid = atoi(pid_str);
auto pHandle = OpenProcess(SYNCHRONIZE, FALSE, parent_pid);
if (!pHandle) {
TraceEvent("ParentProcessOpenError").GetLastError();
fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError());
throw platform_error();
}
startThread(&parentWatcher, pHandle);
break;
}
#endif
case OPT_TAGNAME:
tagName = args->OptionArg();
tagProvided = true;
break;
case OPT_CRASHONERROR:
g_crashOnError = true;
break;
case OPT_MEMLIMIT:
ti = parse_with_suffix(args->OptionArg(), "MiB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
memLimit = ti.get();
break;
case OPT_BLOB_CREDENTIALS:
tlsConfig.blobCredentials.push_back(args->OptionArg());
break;
#ifndef TLS_DISABLED
case TLSConfig::OPT_TLS_PLUGIN:
args->OptionArg();
break;
case TLSConfig::OPT_TLS_CERTIFICATES:
tlsConfig.tlsCertPath = args->OptionArg();
break;
case TLSConfig::OPT_TLS_PASSWORD:
tlsConfig.tlsPassword = args->OptionArg();
break;
case TLSConfig::OPT_TLS_CA_FILE:
tlsConfig.tlsCAPath = args->OptionArg();
break;
case TLSConfig::OPT_TLS_KEY:
tlsConfig.tlsKeyPath = args->OptionArg();
break;
case TLSConfig::OPT_TLS_VERIFY_PEERS:
tlsConfig.tlsVerifyPeers = args->OptionArg();
break;
#endif
case OPT_DUMP_BEGIN:
dumpBegin = parseVersion(args->OptionArg());
break;
case OPT_DUMP_END:
dumpEnd = parseVersion(args->OptionArg());
break;
case OPT_JSON:
jsonOutput = true;
break;
}
}
// Process the extra arguments
for (int argLoop = 0; argLoop < args->FileCount(); argLoop++) {
switch (programExe) {
case ProgramExe::AGENT:
fprintf(stderr, "ERROR: Backup Agent does not support argument value `%s'\n", args->File(argLoop));
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
// Add the backup key range
case ProgramExe::BACKUP:
// Error, if the keys option was not specified
if (backupKeys.size() == 0) {
fprintf(stderr, "ERROR: Unknown backup option value `%s'\n", args->File(argLoop));
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
// Otherwise, assume the item is a key range
else {
try {
addKeyRange(args->File(argLoop), backupKeys);
} catch (Error&) {
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
}
break;
case ProgramExe::RESTORE:
fprintf(stderr, "ERROR: FDB Restore does not support argument value `%s'\n", args->File(argLoop));
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
case ProgramExe::FASTRESTORE_TOOL:
fprintf(
stderr, "ERROR: FDB Fast Restore Tool does not support argument value `%s'\n", args->File(argLoop));
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
case ProgramExe::DR_AGENT:
fprintf(stderr, "ERROR: DR Agent does not support argument value `%s'\n", args->File(argLoop));
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
case ProgramExe::DB_BACKUP:
// Error, if the keys option was not specified
if (backupKeys.size() == 0) {
fprintf(stderr, "ERROR: Unknown DR option value `%s'\n", args->File(argLoop));
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
// Otherwise, assume the item is a key range
else {
try {
addKeyRange(args->File(argLoop), backupKeys);
} catch (Error&) {
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
}
}
break;
case ProgramExe::UNDEFINED:
default:
return FDB_EXIT_ERROR;
}
}
if (trace) {
if (!traceLogGroup.empty())
setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(traceLogGroup));
if (traceDir.empty())
setNetworkOption(FDBNetworkOptions::TRACE_ENABLE);
else
setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(traceDir));
if (!traceFormat.empty()) {
setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(traceFormat));
}
setNetworkOption(FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING);
}
setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING);
// deferred TLS options
if (!tlsConfig.setupTLS()) {
return 1;
}
Error::init();
std::set_new_handler(&platform::outOfMemory);
setMemoryQuota(memLimit);
Database db;
Database sourceDb;
FileBackupAgent ba;
Key tag;
Future<Optional<Void>> f;
Future<Optional<int>> fstatus;
Reference<IBackupContainer> c;
try {
setupNetwork(0, UseMetrics::True);
} catch (Error& e) {
fprintf(stderr, "ERROR: %s\n", e.what());
return FDB_EXIT_ERROR;
}
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
for (const auto& [knobName, knobValueString] : knobs) {
try {
auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
g_knobs.setKnob(knobName, knobValue);
} catch (Error& e) {
if (e.code() == error_code_invalid_option_value) {
fprintf(stderr,
"WARNING: Invalid value '%s' for knob option '%s'\n",
knobValueString.c_str(),
knobName.c_str());
TraceEvent(SevWarnAlways, "InvalidKnobValue")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
} else {
fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
TraceEvent(SevError, "FailedToSetKnob")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString))
.error(e);
throw;
}
}
}
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
g_knobs.initialize(Randomize::False, IsSimulated::False);
TraceEvent("ProgramStart")
.setMaxEventLength(12000)
.detail("SourceVersion", getSourceVersion())
.detail("Version", FDB_VT_VERSION)
.detail("PackageName", FDB_VT_PACKAGE_NAME)
.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL))
.setMaxFieldLength(10000)
.detail("CommandLine", commandLine)
.setMaxFieldLength(0)
.detail("MemoryLimit", memLimit)
.trackLatest("ProgramStart");
// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents
// are logged. This thread will eventually run the network, so call it now.
TraceEvent::setNetworkThread();
// Sets up blob credentials, including one from the environment FDB_BLOB_CREDENTIALS.
tlsConfig.setupBlobCredentials();
// Opens a trace file if trace is set (and if a trace file isn't already open)
// For most modes, initCluster() will open a trace file, but some fdbbackup operations do not require
// a cluster so they should use this instead.
auto initTraceFile = [&]() {
if (trace)
openTraceFile(NetworkAddress(), traceRollSize, traceMaxLogsSize, traceDir, "trace", traceLogGroup);
};
auto initCluster = [&](bool quiet = false) {
Optional<Database> result = connectToCluster(clusterFile, localities, quiet);
if (result.present()) {
db = result.get();
}
return result.present();
};
auto initSourceCluster = [&](bool required, bool quiet = false) {
if (!sourceClusterFile.size() && required) {
if (!quiet) {
fprintf(stderr, "ERROR: source cluster file is required\n");
}
return false;
}
Optional<Database> result = connectToCluster(sourceClusterFile, localities, quiet);
if (result.present()) {
sourceDb = result.get();
}
return result.present();
};
switch (programExe) {
case ProgramExe::AGENT:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(runAgent(db));
break;
case ProgramExe::BACKUP:
switch (backupType) {
case BackupType::START: {
if (!initCluster())
return FDB_EXIT_ERROR;
// Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable.
openBackupContainer(argv[0], destinationContainer, encryptionKeyFile);
f = stopAfter(submitBackup(db,
destinationContainer,
initialSnapshotIntervalSeconds,
snapshotIntervalSeconds,
backupKeys,
tagName,
dryRun,
waitForDone,
stopWhenDone,
usePartitionedLog,
incrementalBackupOnly));
break;
}
case BackupType::MODIFY: {
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(modifyBackup(db, tagName, modifyOptions));
break;
}
case BackupType::STATUS:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(statusBackup(db, tagName, ShowErrors::True, jsonOutput));
break;
case BackupType::ABORT:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(abortBackup(db, tagName));
break;
case BackupType::CLEANUP:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(cleanupMutations(db, deleteData));
break;
case BackupType::WAIT:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(waitBackup(db, tagName, stopWhenDone));
break;
case BackupType::DISCONTINUE:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(discontinueBackup(db, tagName, waitForDone));
break;
case BackupType::PAUSE:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(changeBackupResumed(db, true));
break;
case BackupType::RESUME:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(changeBackupResumed(db, false));
break;
case BackupType::EXPIRE:
initTraceFile();
// Must have a usable cluster if either expire DateTime options were used
if (!expireDatetime.empty() || !expireRestorableAfterDatetime.empty()) {
if (!initCluster())
return FDB_EXIT_ERROR;
}
f = stopAfter(expireBackupData(argv[0],
destinationContainer,
expireVersion,
expireDatetime,
db,
forceAction,
expireRestorableAfterVersion,
expireRestorableAfterDatetime,
encryptionKeyFile));
break;
case BackupType::DELETE_BACKUP:
initTraceFile();
f = stopAfter(deleteBackupContainer(argv[0], destinationContainer));
break;
case BackupType::DESCRIBE:
initTraceFile();
// If timestamp lookups are desired, require a cluster file
if (describeTimestamps && !initCluster())
return FDB_EXIT_ERROR;
// Only pass database optionDatabase Describe will lookup version timestamps if a cluster file was
// given, but quietly skip them if not.
f = stopAfter(describeBackup(argv[0],
destinationContainer,
describeDeep,
describeTimestamps ? Optional<Database>(db) : Optional<Database>(),
jsonOutput,
encryptionKeyFile));
break;
case BackupType::LIST:
initTraceFile();
f = stopAfter(listBackup(baseUrl));
break;
case BackupType::TAGS:
if (!initCluster())
return FDB_EXIT_ERROR;
f = stopAfter(listBackupTags(db));
break;
case BackupType::QUERY:
initTraceFile();
f = stopAfter(queryBackup(argv[0],
destinationContainer,
backupKeysFilter,
restoreVersion,
restoreClusterFileOrig,
restoreTimestamp,
Verbose{ !quietDisplay }));
break;
case BackupType::DUMP:
initTraceFile();
f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd));
break;
case BackupType::UNDEFINED:
default:
fprintf(stderr, "ERROR: Unsupported backup action %s\n", argv[1]);
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
}
break;
case ProgramExe::RESTORE:
if (dryRun) {
if (restoreType != RestoreType::START) {
fprintf(stderr, "Restore dry run only works for 'start' command\n");
return FDB_EXIT_ERROR;
}
// Must explicitly call trace file options handling if not calling Database::createDatabase()
initTraceFile();
} else {
if (restoreClusterFileDest.empty()) {
fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n");
return FDB_EXIT_ERROR;
}
if (!fileExists(restoreClusterFileDest)) {
fprintf(stderr,
"Restore destination cluster file '%s' does not exist.\n",
restoreClusterFileDest.c_str());
return FDB_EXIT_ERROR;
}
try {
db = Database::createDatabase(restoreClusterFileDest, Database::API_VERSION_LATEST);
} catch (Error& e) {
fprintf(stderr,
"Restore destination cluster file '%s' invalid: %s\n",
restoreClusterFileDest.c_str(),
e.what());
return FDB_EXIT_ERROR;
}
}
switch (restoreType) {
case RestoreType::START:
f = stopAfter(runRestore(db,
restoreClusterFileOrig,
tagName,
restoreContainer,
backupKeys,
beginVersion,
restoreVersion,
restoreTimestamp,
!dryRun,
Verbose{ !quietDisplay },
waitForDone,
addPrefix,
removePrefix,
onlyApplyMutationLogs,
inconsistentSnapshotOnly,
encryptionKeyFile));
break;
case RestoreType::WAIT:
f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), Verbose::True)));
break;
case RestoreType::ABORT:
f = stopAfter(
map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void {
printf("RESTORE_ABORT Tag: %s State: %s\n",
tagName.c_str(),
FileBackupAgent::restoreStateText(s).toString().c_str());
return Void();
}));
break;
case RestoreType::STATUS:
// If no tag is specifically provided then print all tag status, don't just use "default"
if (tagProvided)
tag = tagName;
f = stopAfter(map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void {
printf("%s\n", s.c_str());
return Void();
}));
break;
default:
throw restore_error();
}
break;
case ProgramExe::FASTRESTORE_TOOL:
// Support --dest-cluster-file option as fdbrestore does
if (dryRun) {
if (restoreType != RestoreType::START) {
fprintf(stderr, "Restore dry run only works for 'start' command\n");
return FDB_EXIT_ERROR;
}
// Must explicitly call trace file options handling if not calling Database::createDatabase()
initTraceFile();
} else {
if (restoreClusterFileDest.empty()) {
fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n");
return FDB_EXIT_ERROR;
}
if (!fileExists(restoreClusterFileDest)) {
fprintf(stderr,
"Restore destination cluster file '%s' does not exist.\n",
restoreClusterFileDest.c_str());
return FDB_EXIT_ERROR;
}
try {
db = Database::createDatabase(restoreClusterFileDest, Database::API_VERSION_LATEST);
} catch (Error& e) {
fprintf(stderr,
"Restore destination cluster file '%s' invalid: %s\n",
restoreClusterFileDest.c_str(),
e.what());
return FDB_EXIT_ERROR;
}
}
// TODO: We have not implemented the code commented out in this case
switch (restoreType) {
case RestoreType::START:
f = stopAfter(runFastRestoreTool(db,
tagName,
restoreContainer,
backupKeys,
restoreVersion,
!dryRun,
Verbose{ !quietDisplay },
waitForDone));
break;
case RestoreType::WAIT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n");
throw restore_error();
// f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) );
break;
case RestoreType::ABORT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_ABORT yet!\n");
throw restore_error();
// f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)),
//[tagName](FileBackupAgent::ERestoreState s) -> Void { printf("Tag: %s State:
//%s\n", tagName.c_str(),
// FileBackupAgent::restoreStateText(s).toString().c_str()); return Void();
// }) );
break;
case RestoreType::STATUS:
printf("[TODO][ERROR] FastRestore does not support RESTORE_STATUS yet!\n");
throw restore_error();
// If no tag is specifically provided then print all tag status, don't just use "default"
if (tagProvided)
tag = tagName;
// f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void {
// printf("%s\n", s.c_str());
// return Void();
// }) );
break;
default:
throw restore_error();
}
break;
case ProgramExe::DR_AGENT:
if (!initCluster() || !initSourceCluster(true)) {
return FDB_EXIT_ERROR;
}
f = stopAfter(runDBAgent(sourceDb, db));
break;
case ProgramExe::DB_BACKUP:
if (!initCluster() || !initSourceCluster(dbType != DBType::ABORT || !dstOnly)) {
return FDB_EXIT_ERROR;
}
switch (dbType) {
case DBType::START:
f = stopAfter(submitDBBackup(sourceDb, db, backupKeys, tagName));
break;
case DBType::STATUS:
f = stopAfter(statusDBBackup(sourceDb, db, tagName, maxErrors));
break;
case DBType::SWITCH:
f = stopAfter(switchDBBackup(sourceDb, db, backupKeys, tagName, forceAction));
break;
case DBType::ABORT:
f = stopAfter(abortDBBackup(sourceDb, db, tagName, partial, dstOnly));
break;
case DBType::PAUSE:
f = stopAfter(changeDBBackupResumed(sourceDb, db, true));
break;
case DBType::RESUME:
f = stopAfter(changeDBBackupResumed(sourceDb, db, false));
break;
case DBType::UNDEFINED:
default:
fprintf(stderr, "ERROR: Unsupported DR action %s\n", argv[1]);
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
break;
}
break;
case ProgramExe::UNDEFINED:
default:
return FDB_EXIT_ERROR;
}
runNetwork();
if (f.isValid() && f.isReady() && !f.isError() && !f.get().present()) {
status = FDB_EXIT_ERROR;
}
if (fstatus.isValid() && fstatus.isReady() && !fstatus.isError() && fstatus.get().present()) {
status = fstatus.get().get();
}
#ifdef ALLOC_INSTRUMENTATION
{
std::cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " "
<< FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " "
<< FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " "
<< FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " "
<< FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " "
<< FastAllocator<16384>::pageCount << std::endl;
std::vector<std::pair<std::string, const char*>> typeNames;
for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) {
std::string s;
#ifdef __linux__
char* demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL);
if (demangled) {
s = demangled;
if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
free(demangled);
} else
s = i->first;
#else
s = i->first;
if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
else if (StringRef(s).startsWith(LiteralStringRef("class ")))
s = s.substr(LiteralStringRef("class ").size());
else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
s = s.substr(LiteralStringRef("struct ").size());
#endif
typeNames.emplace_back(s, i->first);
}
std::sort(typeNames.begin(), typeNames.end());
for (int i = 0; i < typeNames.size(); i++) {
const char* n = typeNames[i].second;
auto& f = allocInstr[n];
printf("%+d\t%+d\t%d\t%d\t%s\n",
f.allocCount,
-f.deallocCount,
f.allocCount - f.deallocCount,
f.maxAllocated,
typeNames[i].first.c_str());
}
// We're about to exit and clean up data structures, this will wreak havoc on allocation recording
memSample_entered = true;
}
#endif
} catch (Error& e) {
TraceEvent(SevError, "MainError").error(e);
status = FDB_EXIT_MAIN_ERROR;
} catch (boost::system::system_error& e) {
if (g_network) {
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
} else {
fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code().value());
}
status = FDB_EXIT_MAIN_EXCEPTION;
} catch (std::exception& e) {
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
status = FDB_EXIT_MAIN_EXCEPTION;
}
flushAndExit(status);
}