in otava/main.py [0:0]
def script_main(conf: Config, args: List[str] = None):
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
parser = argparse.ArgumentParser(description="Hunts performance regressions in Fallout results")
subparsers = parser.add_subparsers(dest="command")
list_tests_parser = subparsers.add_parser("list-tests", help="list available tests")
list_tests_parser.add_argument("group", help="name of the group of the tests", nargs="*")
list_metrics_parser = subparsers.add_parser(
"list-metrics", help="list available metrics for a test"
)
list_metrics_parser.add_argument("test", help="name of the test")
subparsers.add_parser("list-groups", help="list available groups of tests")
analyze_parser = subparsers.add_parser(
"analyze",
help="analyze performance test results",
formatter_class=argparse.RawTextHelpFormatter,
)
analyze_parser.add_argument("tests", help="name of the test or group of the tests", nargs="+")
analyze_parser.add_argument(
"--update-grafana",
help="Update Grafana dashboards with appropriate annotations of change points",
action="store_true",
)
analyze_parser.add_argument(
"--update-postgres",
help="Update PostgreSQL database results with change points",
action="store_true",
)
analyze_parser.add_argument(
"--update-bigquery",
help="Update BigQuery database results with change points",
action="store_true",
)
analyze_parser.add_argument(
"--notify-slack",
help="Send notification containing a summary of change points to given Slack channels",
nargs="+",
)
analyze_parser.add_argument(
"--cph-report-since",
help="Sets a limit on the date range of the Change Point History reported to Slack. Same syntax as --since.",
metavar="DATE",
dest="cph_report_since",
)
analyze_parser.add_argument(
"--output",
help="Output format for the generated report.",
choices=list(ReportType),
dest="report_type",
default=ReportType.LOG,
type=ReportType,
)
setup_data_selector_parser(analyze_parser)
setup_analysis_options_parser(analyze_parser)
regressions_parser = subparsers.add_parser("regressions", help="find performance regressions")
regressions_parser.add_argument(
"tests", help="name of the test or group of the tests", nargs="+"
)
setup_data_selector_parser(regressions_parser)
setup_analysis_options_parser(regressions_parser)
remove_annotations_parser = subparsers.add_parser("remove-annotations")
remove_annotations_parser.add_argument(
"tests", help="name of the test or test group", nargs="*"
)
remove_annotations_parser.add_argument(
"--force", help="don't ask questions, just do it", dest="force", action="store_true"
)
subparsers.add_parser(
"validate", help="validates the tests and metrics defined in the configuration"
)
try:
args = parser.parse_args(args=args)
otava = Otava(conf)
if args.command == "list-groups":
otava.list_test_groups()
if args.command == "list-tests":
group_names = args.group if args.group else None
otava.list_tests(group_names)
if args.command == "list-metrics":
test = otava.get_test(args.test)
otava.list_metrics(test)
if args.command == "analyze":
update_grafana_flag = args.update_grafana
update_postgres_flag = args.update_postgres
update_bigquery_flag = args.update_bigquery
slack_notification_channels = args.notify_slack
slack_cph_since = parse_datetime(args.cph_report_since)
data_selector = data_selector_from_args(args)
options = analysis_options_from_args(args)
report_type = args.report_type
tests = otava.get_tests(*args.tests)
tests_analyzed_series = {test.name: None for test in tests}
for test in tests:
try:
analyzed_series = otava.analyze(
test, selector=data_selector, options=options, report_type=report_type
)
if update_grafana_flag:
if not isinstance(test, GraphiteTestConfig):
raise GrafanaError("Not a Graphite test")
otava.update_grafana_annotations(test, analyzed_series)
if update_postgres_flag:
if not isinstance(test, PostgresTestConfig):
raise PostgresError("Not a Postgres test")
otava.update_postgres(test, analyzed_series)
if update_bigquery_flag:
if not isinstance(test, BigQueryTestConfig):
raise BigQueryError("Not a BigQuery test")
otava.update_bigquery(test, analyzed_series)
if slack_notification_channels:
tests_analyzed_series[test.name] = analyzed_series
except DataImportError as err:
logging.error(err.message)
except GrafanaError as err:
logging.error(
f"Failed to update grafana dashboards for {test.name}: {err.message}"
)
except PostgresError as err:
logging.error(
f"Failed to update postgres database for {test.name}: {err.message}"
)
if slack_notification_channels:
otava.notify_slack(
tests_analyzed_series,
selector=data_selector,
channels=slack_notification_channels,
since=slack_cph_since,
)
if args.command == "regressions":
data_selector = data_selector_from_args(args)
options = analysis_options_from_args(args)
tests = otava.get_tests(*args.tests)
regressing_test_count = 0
errors = 0
for test in tests:
try:
regressions = otava.regressions(test, selector=data_selector, options=options)
if regressions:
regressing_test_count += 1
except OtavaError as err:
logging.error(err.message)
errors += 1
except DataImportError as err:
logging.error(err.message)
errors += 1
if regressing_test_count == 0:
print("No regressions found!")
elif regressing_test_count == 1:
print("Regressions in 1 test found")
else:
print(f"Regressions in {regressing_test_count} tests found")
if errors > 0:
print("Some tests were skipped due to import / analyze errors. Consult error log.")
if args.command == "remove-annotations":
if args.tests:
tests = otava.get_tests(*args.tests)
for test in tests:
otava.remove_grafana_annotations(test, args.force)
else:
otava.remove_grafana_annotations(None, args.force)
if args.command == "validate":
otava.validate()
if args.command is None:
parser.print_usage()
except TestConfigError as err:
logging.error(err.message)
exit(1)
except GraphiteError as err:
logging.error(err.message)
exit(1)
except GrafanaError as err:
logging.error(err.message)
exit(1)
except DataImportError as err:
logging.error(err.message)
exit(1)
except OtavaError as err:
logging.error(err.message)
exit(1)
except DateFormatError as err:
logging.error(err.message)
exit(1)
except NotificationError as err:
logging.error(err.message)
exit(1)