in rust-create-cascade/src/main.rs [713:1000]
fn main() {
let mut err = false;
let args = Cli::parse();
stderrlog::new()
.module(module_path!())
.verbosity(args.verbose)
.init()
.unwrap();
let known_dir = &args.known;
let revoked_dir = &args.revoked;
let prev_revset_file = &args.prev_revset;
let reason_set = args.reason_set;
let delta_reason_set = args.delta_reason_set;
let filter_type = args.filter_type;
let ct_logs_json = &args.ct_logs_json;
let out_dir = &args.outdir;
let filter_file = &out_dir.join("filter");
let stash_file = &out_dir.join("filter.stash");
let revset_file = &out_dir.join("revset.bin");
let delta_dir = &out_dir.join("delta");
let delta_filter_file = &out_dir.join("filter.delta");
let reason_codes_csv_file = &out_dir.join("reason_codes.csv");
if !known_dir.exists() {
error!("{} not found", known_dir.display());
err = true;
}
if !revoked_dir.exists() {
error!("{} not found", revoked_dir.display());
err = true;
}
if filter_type == FilterType::Clubcard && !ct_logs_json.exists() {
error!("{} not found", ct_logs_json.display());
err = true;
}
if args.clobber {
if out_dir.exists() && std::fs::remove_dir_all(out_dir).is_err() {
error!("could not clobber output directory");
err = true;
}
} else {
for f in [
filter_file,
stash_file,
revset_file,
delta_dir,
delta_filter_file,
] {
if f.exists() {
error!(
"{} exists! Will not overwrite without --clobber.",
f.display()
);
err = true;
}
}
}
if !out_dir.exists() && std::fs::create_dir_all(out_dir).is_err() {
error!("Could not create out directory: {}", out_dir.display());
err = true;
}
if !delta_dir.exists() && std::fs::create_dir_all(delta_dir).is_err() {
error!("Could not create delta directory: {}", delta_dir.display());
err = true;
}
if err {
return;
}
let hash_alg = match args.murmurhash3 {
true => HashAlgorithm::MurmurHash3,
false => HashAlgorithm::Sha256,
};
let statsd_prefix = match (filter_type, reason_set) {
(FilterType::Cascade, ReasonSet::All) => "crlite.generate",
(FilterType::Cascade, ReasonSet::Specified) => "crlite.generate.specified_reasons",
(FilterType::Cascade, ReasonSet::Priority) => "crlite.generate.priority_reasons",
(FilterType::Clubcard, ReasonSet::All) => "crlite.clubcard.generate",
(FilterType::Clubcard, ReasonSet::Specified) => {
"crlite.clubcard.generate.specified_reasons"
}
(FilterType::Clubcard, ReasonSet::Priority) => "crlite.clubcard.generate.priority_reasons",
};
let statsd_client = match args.statsd_host {
Some(ref statsd_host) if statsd_host.contains(':') => {
// host specified with port
statsd::Client::new(statsd_host, statsd_prefix).ok()
}
Some(ref statsd_host) => {
// use default port
statsd::Client::new(format!("{}:{}", statsd_host, 8125), statsd_prefix).ok()
}
None => None,
};
if args.statsd_host.is_some() && statsd_client.is_none() {
info!("Could not connect to statsd {}", args.statsd_host.unwrap());
}
info!("Counting serials");
let filter_stats = count_all(
revoked_dir,
known_dir,
reason_set,
Some(reason_codes_csv_file),
);
info!(
"Lower bound when splitting by issuer is {:.0} bytes",
filter_stats.split_by_issuer_lower_bound
);
info!(
"Lower bound when splitting by issuer and expiry is {:.0} bytes",
filter_stats.split_by_issuer_and_expiry_lower_bound
);
info!(
"Found {} 'revoked' and {} 'not revoked' serial numbers",
filter_stats.exact_revoked_count, filter_stats.approx_ok_count
);
info!("Revocation reason codes: {:#?}", filter_stats.reasons);
let timer_start = Instant::now();
let filter_bytes = match filter_type {
FilterType::Clubcard => {
info!("Generating clubcard");
create_clubcard(
filter_file,
revoked_dir,
known_dir,
ct_logs_json,
reason_set,
)
}
FilterType::Cascade => {
info!("Generating cascade");
create_cascade(
filter_file,
filter_stats.exact_revoked_count,
filter_stats.approx_ok_count,
revoked_dir,
known_dir,
hash_alg,
reason_set,
)
}
};
let timer_finish = Instant::now() - timer_start;
info!("Finished in {} seconds", timer_finish.as_secs());
info!("Generating stash file");
let timer_start = Instant::now();
write_revset_and_delta(
delta_dir,
revset_file,
prev_revset_file,
revoked_dir,
known_dir,
delta_reason_set,
statsd_client.as_ref(),
);
write_stash(
stash_file,
delta_dir,
delta_reason_set,
statsd_client.as_ref(),
);
let timer_finish = Instant::now() - timer_start;
info!("Finished in {} seconds", timer_finish.as_secs());
info!("Counting delta serials");
let delta_stats = count_all(delta_dir, known_dir, delta_reason_set, None);
info!(
"Lower bound when splitting by issuer is {:.0} bytes",
delta_stats.split_by_issuer_lower_bound
);
info!(
"Lower bound when splitting by issuer and expiry is {:.0} bytes",
delta_stats.split_by_issuer_and_expiry_lower_bound
);
info!(
"Found {} 'revoked' serial numbers in delta",
delta_stats.exact_revoked_count
);
info!("Revocation reason codes: {:#?}", delta_stats.reasons);
info!("Generating delta filter");
let timer_start = Instant::now();
let delta_filter_bytes = match filter_type {
FilterType::Clubcard => {
info!("Generating clubcard");
create_clubcard(
delta_filter_file,
delta_dir,
known_dir,
ct_logs_json,
delta_reason_set,
)
}
FilterType::Cascade => {
info!("Generating cascade");
create_cascade(
delta_filter_file,
delta_stats.exact_revoked_count,
delta_stats.approx_ok_count,
delta_dir,
known_dir,
hash_alg,
delta_reason_set,
)
}
};
let timer_finish = Instant::now() - timer_start;
info!("Finished in {} seconds", timer_finish.as_secs());
if let Some(client) = statsd_client {
client.gauge("time", timer_finish.as_secs() as f64);
client.gauge("filter_size", filter_bytes.len() as f64);
client.gauge(
"filter_by_issuer_lower_bound",
filter_stats.split_by_issuer_lower_bound,
);
client.gauge(
"filter_by_issuer_and_expiry_lower_bound",
filter_stats.split_by_issuer_and_expiry_lower_bound,
);
client.gauge("not_revoked", filter_stats.approx_ok_count as f64);
client.gauge("revoked", filter_stats.exact_revoked_count as f64);
client.gauge("delta_filter_size", delta_filter_bytes.len() as f64);
client.gauge(
"delta_by_issuer_lower_bound",
delta_stats.split_by_issuer_lower_bound,
);
client.gauge(
"delta_by_issuer_and_expiry_lower_bound",
delta_stats.split_by_issuer_and_expiry_lower_bound,
);
client.gauge("delta_not_revoked", delta_stats.approx_ok_count as f64);
client.gauge("delta_revoked", delta_stats.exact_revoked_count as f64);
client.gauge(
"revoked.unspecified",
filter_stats.reasons.unspecified as f64,
);
client.gauge(
"revoked.key_compromise",
filter_stats.reasons.key_compromise as f64,
);
client.gauge(
"revoked.ca_compromise",
filter_stats.reasons.ca_compromise as f64,
);
client.gauge(
"revoked.affiliation_changed",
filter_stats.reasons.affiliation_changed as f64,
);
client.gauge("revoked.superseded", filter_stats.reasons.superseded as f64);
client.gauge(
"revoked.cessation_of_operation",
filter_stats.reasons.cessation_of_operation as f64,
);
client.gauge(
"revoked.certificate_hold",
filter_stats.reasons.certificate_hold as f64,
);
client.gauge(
"revoked.remove_from_crl",
filter_stats.reasons.remove_from_crl as f64,
);
client.gauge(
"revoked.privilege_withdrawn",
filter_stats.reasons.privilege_withdrawn as f64,
);
client.gauge(
"revoked.aa_compromise",
filter_stats.reasons.aa_compromise as f64,
);
}
info!("Done");
}