in utils/consolidated_test_report.py [0:0]
def generate_report(consolidated_data):
"""Generate a comprehensive markdown report from consolidated data."""
report = []
# Add report header
report.append("# Diffusers Nightly Test Report")
report.append(f"Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
# Removed comparison section
# Add summary section
total = consolidated_data["total_stats"]
report.append("## Summary")
# Get duration stats if available
duration_stats = consolidated_data.get("duration_stats", {})
total_duration = duration_stats.get("total_duration", 0)
summary_table = [
["Total Tests", total["tests"]],
["Passed", total["passed"]],
["Failed", total["failed"]],
["Skipped", total["skipped"]],
["Success Rate", f"{(total['passed'] / total['tests'] * 100):.2f}%" if total["tests"] > 0 else "N/A"],
["Total Duration", f"{total_duration:.2f}s" if total_duration else "N/A"],
]
report.append(tabulate(summary_table, tablefmt="pipe"))
report.append("")
# Removed duration distribution section
# Add test suites summary
report.append("## Test Suites")
# Include duration in test suites table if available
suite_durations = consolidated_data.get("duration_stats", {}).get("suite_durations", {})
if suite_durations:
suites_table = [["Test Suite", "Tests", "Passed", "Failed", "Skipped", "Success Rate", "Duration (s)"]]
else:
suites_table = [["Test Suite", "Tests", "Passed", "Failed", "Skipped", "Success Rate"]]
# Sort test suites by success rate (ascending - least successful first)
sorted_suites = sorted(
consolidated_data["test_suites"].items(),
key=lambda x: (x[1]["stats"]["passed"] / x[1]["stats"]["tests"] * 100) if x[1]["stats"]["tests"] > 0 else 0,
reverse=False,
)
for suite_name, suite_data in sorted_suites:
stats = suite_data["stats"]
success_rate = f"{(stats['passed'] / stats['tests'] * 100):.2f}%" if stats["tests"] > 0 else "N/A"
if suite_durations:
duration = suite_durations.get(suite_name, 0)
suites_table.append(
[
suite_name,
stats["tests"],
stats["passed"],
stats["failed"],
stats["skipped"],
success_rate,
f"{duration:.2f}",
]
)
else:
suites_table.append(
[suite_name, stats["tests"], stats["passed"], stats["failed"], stats["skipped"], success_rate]
)
report.append(tabulate(suites_table, headers="firstrow", tablefmt="pipe"))
report.append("")
# Add slowest tests section
slowest_tests = consolidated_data.get("slowest_tests", [])
if slowest_tests:
report.append("## Slowest Tests")
slowest_table = [["Rank", "Test", "Duration (s)", "Test Suite"]]
for i, test in enumerate(slowest_tests, 1):
# Skip entries that don't contain actual test names
if "< 0.05 secs were omitted" in test["test"]:
continue
slowest_table.append([i, test["test"], f"{test['duration']:.2f}", test["suite"]])
report.append(tabulate(slowest_table, headers="firstrow", tablefmt="pipe"))
report.append("")
# Add failures section if there are any
failed_suites = [s for s in sorted_suites if s[1]["stats"]["failed"] > 0]
if failed_suites:
report.append("## Failures")
# Group failures by module for cleaner organization
failures_by_module = {}
for suite_name, suite_data in failed_suites:
# Extract failures data for this suite
for failure in suite_data.get("failures", []):
test_name = failure["test"]
# If test name doesn't look like a full path, try to reconstruct it
if not ("/" in test_name or "::" in test_name) and "." in test_name:
# For simple 'TestClass.test_method' format, try to get full path from suite name
# Form: tests_<suite>_cuda -> tests/<suite>/test_<suite>.py::TestClass::test_method
if suite_name.startswith("tests_") and "_cuda" in suite_name:
# Extract component name from suite
component = suite_name.replace("tests_", "").replace("_cuda", "")
if "." in test_name:
class_name, method_name = test_name.split(".", 1)
possible_path = f"tests/{component}/test_{component}.py::{class_name}::{method_name}"
# Use this constructed path if it seems reasonable
if "test_" in method_name:
test_name = possible_path
# Extract module name from test name
if "::" in test_name:
# For path/file.py::TestClass::test_method format
parts = test_name.split("::")
module_name = parts[-2] if len(parts) >= 2 else "Other" # TestClass
elif "." in test_name:
# For TestClass.test_method format
parts = test_name.split(".")
module_name = parts[0] # TestClass
else:
module_name = "Other"
# Skip module names that don't look like class/module names
if (
module_name.startswith(("e.g", "i.e", "etc"))
or module_name.replace(".", "").isdigit()
or len(module_name) < 3
):
module_name = "Other"
# Add to the module group
if module_name not in failures_by_module:
failures_by_module[module_name] = []
# Prepend the suite name if the test name doesn't already have a full path
if "/" not in test_name and suite_name not in test_name:
full_test_name = f"{suite_name}::{test_name}"
else:
full_test_name = test_name
# Add this failure to the module group
failures_by_module[module_name].append(
{"test": full_test_name, "original_test": test_name, "error": failure["error"]}
)
# Create a list of failing tests for each module
if failures_by_module:
for module_name, failures in sorted(failures_by_module.items()):
report.append(f"### {module_name}")
# Put all failed tests in a single code block
report.append("```")
for failure in failures:
# Show test path and error message if available
if failure.get("error") and failure["error"] != "No error message":
report.append(f"{failure['test']} - {failure['error']}")
else:
report.append(failure["test"])
report.append("```")
report.append("") # Add space between modules
else:
report.append("*No detailed failure information available*")
report.append("")
return "\n".join(report)