in eden/mononoke/cmds/backfill_derived_data/main.rs [145:455]
fn main(fb: FacebookInit) -> Result<()> {
let app = args::MononokeAppBuilder::new("Utility to work with bonsai derived data")
.with_advanced_args_hidden()
.with_fb303_args()
.with_repo_required(RepoRequirement::AtLeastOne)
.with_scuba_logging_args()
.build()
.about("Utility to work with bonsai derived data")
.subcommand(
SubCommand::with_name(SUBCOMMAND_BACKFILL)
.about("backfill derived data for public commits")
.arg(
Arg::with_name(ARG_DERIVED_DATA_TYPE)
.required(true)
.index(1)
.possible_values(POSSIBLE_DERIVED_TYPES)
.help("derived data type for which backfill will be run"),
)
.arg(
Arg::with_name(ARG_SKIP)
.long(ARG_SKIP)
.takes_value(true)
.help("skip this number of changesets"),
)
.arg(
Arg::with_name(ARG_LIMIT)
.long(ARG_LIMIT)
.takes_value(true)
.help("backfill at most this number of changesets"),
)
.arg(
Arg::with_name(ARG_REGENERATE)
.long(ARG_REGENERATE)
.help("regenerate derivations even if mapping contains changeset"),
)
.arg(
Arg::with_name(ARG_PREFETCHED_COMMITS_PATH)
.long(ARG_PREFETCHED_COMMITS_PATH)
.takes_value(true)
.required(false)
.help("a file with a list of bonsai changesets to backfill"),
)
.arg(
Arg::with_name(ARG_BATCH_SIZE)
.long(ARG_BATCH_SIZE)
.default_value(DEFAULT_BATCH_SIZE_STR)
.help("number of changesets in each derivation batch"),
)
.arg(
Arg::with_name(ARG_PARALLEL)
.long(ARG_PARALLEL)
.help("derive commits within a batch in parallel"),
)
.arg(
Arg::with_name(ARG_GAP_SIZE)
.long(ARG_GAP_SIZE)
.takes_value(true)
.help("size of gap to leave in derived data types that support gaps"),
)
.arg(
Arg::with_name(ARG_BACKFILL_CONFIG_NAME)
.long(ARG_BACKFILL_CONFIG_NAME)
.help("sets the name for backfilling derived data types config")
.takes_value(true),
),
)
.subcommand(
SubCommand::with_name(SUBCOMMAND_TAIL)
.about("tail public commits and fill derived data")
.arg(
Arg::with_name(ARG_DERIVED_DATA_TYPE)
.required(false)
.multiple(true)
.index(1)
.possible_values(POSSIBLE_DERIVED_TYPES)
// TODO(stash): T66492899 remove unused value
.help("Unused, will be deleted soon"),
)
.arg(
Arg::with_name(ARG_USE_SHARED_LEASES)
.long(ARG_USE_SHARED_LEASES)
.takes_value(false)
.required(false)
.help(concat!(
"By default the derived data tailer doesn't compete with ",
"other mononoke services for a derived data lease, so ",
"it will derive the data even if another mononoke service ",
"(e.g. mononoke_server, scs_server, ...) are already ",
"deriving it.\n\n",
"This flag disables this behaviour, meaning this command ",
"will compete for the derived data lease with other ",
"mononoke services and start deriving only if the lease ",
"is obtained.",
)),
)
.arg(
Arg::with_name(ARG_STOP_ON_IDLE)
.long(ARG_STOP_ON_IDLE)
.help("Stop tailing or backfilling when there is nothing left"),
)
.arg(
Arg::with_name(ARG_BATCHED)
.long(ARG_BATCHED)
.takes_value(false)
.required(false)
.help("Use batched deriver instead of calling `::derive` periodically"),
)
.arg(
Arg::with_name(ARG_BATCH_SIZE)
.long(ARG_BATCH_SIZE)
.default_value(DEFAULT_BATCH_SIZE_STR)
.help("number of changesets in each derivation batch"),
)
.arg(
Arg::with_name(ARG_PARALLEL)
.long(ARG_PARALLEL)
.help("derive commits within a batch in parallel"),
)
.arg(
Arg::with_name(ARG_BACKFILL)
.long(ARG_BACKFILL)
.help("also backfill derived data types configured for backfilling"),
)
.arg(
Arg::with_name(ARG_SLICED)
.long(ARG_SLICED)
.help("pre-slice repository using the skiplist index when backfilling"),
)
.arg(
Arg::with_name(ARG_SLICE_SIZE)
.long(ARG_SLICE_SIZE)
.default_value(DEFAULT_SLICE_SIZE_STR)
.help("number of generations to include in each generation slice"),
)
.arg(
Arg::with_name(ARG_GAP_SIZE)
.long(ARG_GAP_SIZE)
.takes_value(true)
.help("size of gap to leave in derived data types that support gaps"),
)
.arg(
Arg::with_name(ARG_BACKFILL_CONFIG_NAME)
.long(ARG_BACKFILL_CONFIG_NAME)
.help("sets the name for backfilling derived data types config")
.takes_value(true),
),
)
.subcommand(
SubCommand::with_name(SUBCOMMAND_SINGLE)
.about("backfill single changeset (mainly for performance testing purposes)")
.arg(
Arg::with_name(ARG_ALL_TYPES)
.long(ARG_ALL_TYPES)
.required(false)
.takes_value(false)
.help("derive all derived data types enabled for this repo"),
)
.arg(
Arg::with_name(ARG_CHANGESET)
.required(true)
.index(1)
.help("changeset by {hg|bonsai} hash or bookmark"),
)
.arg(
Arg::with_name(ARG_DERIVED_DATA_TYPE)
.required(false)
.index(2)
.multiple(true)
.conflicts_with(ARG_ALL_TYPES)
.possible_values(POSSIBLE_DERIVED_TYPES)
.help("derived data type for which backfill will be run"),
),
)
.subcommand(
SubCommand::with_name(SUBCOMMAND_BACKFILL_ALL)
.about("backfill all/many derived data types at once")
.arg(
Arg::with_name(ARG_DERIVED_DATA_TYPE)
.conflicts_with(ARG_ALL_TYPES)
.possible_values(POSSIBLE_DERIVED_TYPES)
.required(false)
.takes_value(true)
.multiple(true)
.help(concat!(
"derived data type for which backfill will be run, ",
"all enabled and backfilling types if not specified",
)),
)
.arg(
Arg::with_name(ARG_ALL_TYPES)
.long(ARG_ALL_TYPES)
.required(false)
.takes_value(false)
.help("derive all derived data types enabled for this repo"),
)
.arg(
Arg::with_name(ARG_BATCH_SIZE)
.long(ARG_BATCH_SIZE)
.default_value(DEFAULT_BATCH_SIZE_STR)
.help("number of changesets in each derivation batch"),
)
.arg(
Arg::with_name(ARG_PARALLEL)
.long(ARG_PARALLEL)
.help("derive commits within a batch in parallel"),
)
.arg(
Arg::with_name(ARG_SLICED).long(ARG_SLICED).help(
"pre-slice repository into generation slices using the skiplist index",
),
)
.arg(
Arg::with_name(ARG_SLICE_SIZE)
.long(ARG_SLICE_SIZE)
.default_value(DEFAULT_SLICE_SIZE_STR)
.help("number of generations to include in each generation slice"),
)
.arg(
Arg::with_name(ARG_GAP_SIZE)
.long(ARG_GAP_SIZE)
.takes_value(true)
.help("size of gap to leave in derived data types that support gaps"),
)
.arg(
Arg::with_name(ARG_BACKFILL_CONFIG_NAME)
.long(ARG_BACKFILL_CONFIG_NAME)
.help("sets the name for backfilling derived data types config")
.takes_value(true),
),
)
.subcommand(
regenerate::DeriveOptions::add_opts(
commit_discovery::CommitDiscoveryOptions::add_opts(
SubCommand::with_name(SUBCOMMAND_BENCHMARK)
.about("benchmark derivation of a list of commits")
.long_about(
"note that this command WILL DERIVE data and save it to storage",
),
),
)
.arg(
Arg::with_name(ARG_DERIVED_DATA_TYPE)
.required(true)
.index(1)
.multiple(true)
.conflicts_with(ARG_ALL_TYPES)
.possible_values(POSSIBLE_DERIVED_TYPES)
.help("derived data type for which backfill will be run"),
)
.arg(
Arg::with_name(ARG_ALL_TYPES)
.long(ARG_ALL_TYPES)
.required(false)
.takes_value(false)
.help("derive all derived data types enabled for this repo"),
)
.arg(
Arg::with_name(ARG_JSON)
.long(ARG_JSON)
.required(false)
.takes_value(false)
.help("Print result in json format"),
),
)
.subcommand(
regenerate::DeriveOptions::add_opts(
commit_discovery::CommitDiscoveryOptions::add_opts(
SubCommand::with_name(SUBCOMMAND_VALIDATE)
.about("rederive the commits and make sure they are saved to the storage")
.long_about("this command won't write anything new to the storage"),
),
)
.arg(
Arg::with_name(ARG_DERIVED_DATA_TYPE)
.required(true)
.index(1)
.multiple(true)
.conflicts_with(ARG_ALL_TYPES)
.possible_values(POSSIBLE_DERIVED_TYPES)
.help("derived data type for which backfill will be run"),
)
.arg(
Arg::with_name(ARG_VALIDATE_CHUNK_SIZE)
.long(ARG_VALIDATE_CHUNK_SIZE)
.default_value(DEFAULT_VALIDATE_CHUNK_SIZE)
.help("how many commits to validate at once."),
)
.arg(
Arg::with_name(ARG_JSON)
.long(ARG_JSON)
.required(false)
.takes_value(false)
.help("Print result in json format"),
),
);
let matches = app.get_matches(fb)?;
let logger = matches.logger();
let reponame = args::get_repo_name(matches.config_store(), &matches)?;
let mut scuba_sample_builder = matches.scuba_sample_builder();
scuba_sample_builder.add("reponame", reponame);
let ctx = create_ctx(fb, logger, scuba_sample_builder, &matches);
helpers::block_execute(
run_subcmd(fb, ctx, &logger, &matches),
fb,
&std::env::var("TW_JOB_NAME").unwrap_or("backfill_derived_data".to_string()),
&logger,
&matches,
cmdlib::monitoring::AliveService,
)
}