# ============================================================================= # URL Checker for Markdown Files # ============================================================================= # This script scans all Markdown files in a repository for URLs and checks # whether they are valid. It handles both absolute URLs (http/https) and # relative file paths, providing a detailed report of broken links. # ============================================================================= import os import re import requests import subprocess from urllib.parse import urljoin, urlparse from datetime import datetime import ipaddress from colorama import init import sys import argparse # Initialize colorama for Windows compatibility and force color output in GitHub Actions init(strip=False, convert=False) # ============================================================================= # CONFIGURATION # ============================================================================= # ANSI color codes for terminal output class Colors: OKGREEN = '\033[92m' # Green for success FAIL = '\033[91m' # Red for errors INFO = '\033[96m' # Cyan for neutral/informational NEUTRAL = '\033[93m' # Yellow for "no links found" category SPECIAL = '\033[95m' # Magenta for "categories with no broken links" ENDC = '\033[0m' def get_repo_root(): """Find the root directory of the Git repository.""" try: return subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], text=True).strip() except subprocess.CalledProcessError: return '.' # Default to current directory if not in a Git repo # Script configuration settings REPO_PATH = get_repo_root() SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) LOG_DIR = os.path.join(SCRIPT_DIR, 'logs') # Create logs directory and handle any errors gracefully try: os.makedirs(LOG_DIR, exist_ok=True) print(f"Logs will be saved to: {LOG_DIR}") except Exception as e: print(f"Warning: Could not create logs directory: {e}") LOG_DIR = SCRIPT_DIR # Fallback to script directory print(f"Using fallback log directory: {LOG_DIR}") TIMEOUT = 15 # Request timeout in seconds - increase this if you get many timeout errors HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; LinkChecker/1.0)"} # Browser-like user agent # File types to check - maps extensions to descriptive names SUPPORTED_FILE_TYPES = { # Existing file types '.md': 'Markdown', '.html': 'HTML', '.htm': 'HTML', '.js': 'JavaScript', '.jsx': 'React', '.ts': 'TypeScript', '.tsx': 'React TypeScript', '.py': 'Python', '.css': 'CSS', '.scss': 'SCSS', '.json': 'JSON', '.yaml': 'YAML', '.yml': 'YAML', '.xml': 'XML', '.cs': 'C#', '.java': 'Java', '.txt': 'Text', '.rst': 'reStructuredText', # Shell/Bash scripting '.sh': 'Shell', '.bash': 'Bash', '.zsh': 'ZShell', '.ksh': 'KornShell', # PowerShell '.ps1': 'PowerShell', '.psm1': 'PowerShell Module', '.psd1': 'PowerShell Data', # Windows batch/command '.bat': 'Batch', '.cmd': 'Command', # Other scripting languages '.pl': 'Perl', '.pm': 'Perl Module', '.rb': 'Ruby', '.php': 'PHP', '.r': 'R', '.R': 'R', '.lua': 'Lua', '.tcl': 'Tcl', '.groovy': 'Groovy', '.awk': 'Awk', '.sed': 'Sed', # Configuration files '.ini': 'INI', '.conf': 'Config', '.cfg': 'Config', '.toml': 'TOML', '.env': 'Environment', } # Regular expressions for URL detection and processing # Markdown link pattern MD_URL_REGEX = re.compile(r'\[.*?\]\((.*?)\)') # Finds markdown links: [text](url) # HTML link patterns HTML_HREF_REGEX = re.compile(r']+href=["\'](.*?)["\']', re.IGNORECASE) HTML_SRC_REGEX = re.compile(r'<(?:img|script|iframe)[^>]+src=["\'](.*?)["\']', re.IGNORECASE) HTML_LINK_HREF_REGEX = re.compile(r']+href=["\'](.*?)["\']', re.IGNORECASE) HTML_META_CONTENT_REGEX = re.compile(r']+content=["\'](.*?)["\']', re.IGNORECASE) # CSS url() pattern CSS_URL_REGEX = re.compile(r'url\(["\']?(.*?)["\']?\)', re.IGNORECASE) # JavaScript/TypeScript URL patterns JS_URL_REGEX = re.compile(r'(?:(?:\'|")(?:https?://|/|\.\.?/)[^\s\'">]+(?:\'|")|import\s+(?:.+from\s+)?[\'"]([^\'"]+)[\'"])') # Python URL patterns PY_URL_REGEX = re.compile(r'(?:\'|")((?:https?://|/|\.\.?/)[^\s\'"]+)(?:\'|")') PY_IMPORT_REGEX = re.compile(r'(?:from|import)\s+([a-zA-Z0-9_.]+)') # JSON/YAML URL patterns JSON_URL_REGEX = re.compile(r'(?:\'|")((?:https?://|/|\.\.?/)[^\s\'"]+)(?:\'|")') # XML URL patterns XML_URL_REGEX = re.compile(r'(?:href|src|url)=["\'](.*?)["\']', re.IGNORECASE) # Shell/Bash URL patterns - matches URLs in quotes, wget/curl commands, etc. SHELL_URL_REGEX = re.compile(r'(?:(?:\'|")((?:https?://|/|\.\.?/)[^\s\'"]+)(?:\'|")|(?:wget|curl)\s+(?:-[a-zA-Z]+\s+)*(?:\'|")?([^\s\'"]+)(?:\'|")?)') # PowerShell URL patterns - matches URLs in quotes, as parameters, etc. PS_URL_REGEX = re.compile(r'(?:(?:\'|")((?:https?://|/|\.\.?/)[^\s\'"]+)(?:\'|")|(?:Invoke-WebRequest|Invoke-RestMethod)\s+(?:-[a-zA-Z]+\s+)*(?:\'|")?([^\s\'"]+)(?:\'|")?)') # Batch/CMD URL patterns BATCH_URL_REGEX = re.compile(r'(?:(?:https?://|/|\.\.?/)[^\s\'"]+)') # Perl/Ruby patterns (similar to Python) SCRIPT_URL_REGEX = re.compile(r'(?:\'|")((?:https?://|/|\.\.?/)[^\s\'"]+)(?:\'|")') # Config file patterns (ini, env, etc.) - looks for URLs after = or : characters CONFIG_URL_REGEX = re.compile(r'(?:=|:)\s*[\'"]?((?:https?://|/|\.\.?/)[^\s\'"]+)[\'"]?') EMAIL_REGEX = re.compile(r'^mailto:') # Detects email links ANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') # For stripping color codes HEADER_LINK_REGEX = re.compile(r'#[-\w]+$') # Matches markdown header links like #header-name # URLs to skip checking - add frequently timing out domains here KNOWN_VALID_DOMAINS = [ "learn.microsoft.com", "whatismyip.com", "www.linkedin.com", "linkedin.com", "icanhazip.com", "shell.azure.com", "aka.ms", # Microsoft's URL shortener "go.microsoft.com", # Another Microsoft URL shortener "api.fabric.microsoft.com", # Microsoft Fabric API "dashboards.kusto.windows.net", # Microsoft Kusto dashboards "api.powerbi.com", # PowerBI API domain "analysis.windows.net", # Power BI analysis domain "api.kusto.windows.net", # Kusto API # Add more domains to skip here as needed ] # Add a list of domains that may have certificate issues but should be considered valid TRUSTED_DOMAINS_WITH_CERT_ISSUES = [ "jumpstartcdn-", # Azure Front Door CDN domains used by aka.ms redirects "azurefd.net", # Azure Front Door domain ] # Function to detect false positive URLs that should be skipped def is_false_positive(url): """Check if a URL is a known false positive pattern that should be skipped.""" # Skip direct domain matches first (most efficient check) try: parsed_url = urlparse(url) if parsed_url.netloc in KNOWN_VALID_DOMAINS: print(f"Skipping trusted domain URL: {url}") return True except: pass # Continue with other checks if parsing fails # Simple string patterns that should be skipped simple_skip_patterns = [ "http://\\", "http:\\", "http://\\\\", "http:\\\\\\", "https://\\", "https://\\\\" ] for pattern in simple_skip_patterns: if pattern in url: print(f"Skipping URL with backslashes: {url}") return True # Template Base URL patterns template_patterns = [ r'\(\$templateBaseUrl', r'\(\$env:templateBaseUrl', r'\(\$Using:templateBaseUrl', r'\$Env:templateBaseUrl', # PowerShell environment variable syntax ] # Storage account and GitHub patterns placeholder_patterns = [ r'https://\{STORAGEACCOUNT\}\.blob\.core\.windows\.net/', r'https://\$githubPat@github\.com/\$githubUser/\$appsRepo\.git', r'http://\$URL:\$PORT', r'https://\$\(\$HCIBoxConfig\.WACVMName\)\.', r'https://\$stagingStorageAccountName\.blob\.core\.windows\.net/\$containerName/config', ] # PowerShell variable names that look like URLs or paths but aren't actual URLs powershell_variable_patterns = [ r'\$websiteUrls', # Variable holding website URLs r'\$websiteUrls\[', # With array indexing r'\$websiteUrls\.', # With property/method access r'\$mqttExplorerReleasesUrl', # MQTT Explorer releases URL variable r'\$mqttExplorerReleaseDownloadUrl', # MQTT Explorer download URL variable r'\$terminalDownloadUri', # Terminal download URI variable r'\$uri', # Generic URI variable r'\$url', # Generic URL variable r'\$downloadUrl', # Download URL variable r'\$aksEEReleasesUrl', # AKS EE releases URL variable r'\$AKSEEReleaseDownloadUrl', # AKS EE download URL variable r'\$localPathStorageUrl', # Local path storage URL variable r'\$acsadeployYamlUrl', # ACSA deploy YAML URL variable r'\$aksEEk3sUrl', # AKS EE K3s URL variable r'\$githubApiUrl', # GitHub API URL variable r'\$fabricHeaders', # Fabric API headers variable r'\$_', # PowerShell automatic variable for current pipeline object ] # Script files and commands that appear to be URLs but aren't script_file_patterns = [ r'get_helm\.sh', # Helm installation script r'http://\\', # Escaped backslash in URL (not a real URL) r'http://\\\\', # Multiple escaped backslashes in URL r'http://\\\\\S*', # Multiple escaped backslashes with any additional characters r'https://\\\\\S*', # Multiple escaped backslashes with HTTPS ] # Escaped backslashes in URLs or JSON path patterns - expanded patterns escaped_backslash_patterns = [ r'http://\\+', # One or more backslashes after http:// r'https://\\+', # One or more backslashes after https:// r'http:\\+', # Backslashes without forward slashes r'https:\\+', # Backslashes without forward slashes r'http://\\\\\S*', # Multiple escaped backslashes with any additional characters r'http://\\', # Single backslash r'http:/\\', # Malformed backslash r'https://\\', # HTTPS with backslash ] # Template variable patterns (JavaScript-style ${var} and shell-style $var) template_variable_patterns = [ r'http://\${[^}]+}', # ${variable} format r'https://\${[^}]+}', r'http://\${[^}]+}:[0-9]+', # With port r'https://\${[^}]+}:[0-9]+', r'http://\${[^}]+}:[0-9]+/\w+', # With path after port r'https://\${[^}]+}:[0-9]+/\w+', r'http://\$[a-zA-Z0-9_]+', # $variable format (without braces) r'https://\$[a-zA-Z0-9_]+', r'http://\$[a-zA-Z0-9_]+:[0-9]+', # With port r'https://\$[a-zA-Z0-9_]+:[0-9]+', r'https://\$[a-zA-Z0-9_]+/\w+', # With path (no port) r'https://\${[^}]+}/\w+', # With path (no port) for braced variables r'https://[^/]+/\$[a-zA-Z0-9_]+', # Variable in path r'https://[^/]+/\${[^}]+}', # Braced variable in path r'https://\$Env:[a-zA-Z0-9_]+', # PowerShell Env variables in URLs r'https://\$env:[a-zA-Z0-9_]+', # PowerShell env variables in URLs (lowercase) ] # Query string variable patterns query_variable_patterns = [ r'https://[^?]+\?[^=]+=\$[a-zA-Z0-9_]+', # https://example.com?param=$variable r'https://[^?]+\?[^=]+=\${[^}]+}', # https://example.com?param=${variable} ] # XML namespace URLs that aren't meant to be accessed directly xml_namespace_urls = [ 'http://www.w3.org/2000/svg', 'http://www.w3.org/1999/xlink', ] # Special placeholder hostnames (typically used in configs/templates) placeholder_hostnames = [ r'influxPlaceholder', ] # Patterns for specific GitHub raw URLs that are placeholders github_raw_urls = [ r'https://raw\.githubusercontent\.com/microsoft/azure_arc/main/azure_jumpstart_ag/', r'https://raw\.githubusercontent\.com/microsoft/azure_arc/main/.+/' ] # Local script file patterns that shouldn't be checked as URLs local_script_patterns = [ r'^\.\/[a-zA-Z0-9_-]+\.sh$', # ./script.sh r'^\.\/[a-zA-Z0-9_-]+\.ps1$', # ./script.ps1 r'^\.\/[a-zA-Z0-9_-]+\.bat$', # ./script.bat r'^\.\/[a-zA-Z0-9_-]+\.cmd$', # ./script.cmd r'\.\/akri\.sh$', # ./akri.sh specifically ] # GitHub API URL patterns with variables github_api_variable_patterns = [ r'\$gitHubAPIBaseUri\/repos\/\$githubUser\/\$appsRepo', r'\$gitHubAPIBaseUri\/repos\/[^\/]+\/[^\/]+', r'\$githubApiUrl', r'api\.github\.com\/repos\/\$[a-zA-Z0-9_]+\/', ] # Additional Management API domains that are valid but often give auth errors management_api_domains = [ r'management\.core\.windows\.net', ] # HTTP verbs that are commonly used in PowerShell scripts and not actual URLs http_verbs = [ r'^Get$', r'^POST$', r'^GET$', r'^PUT$', r'^PATCH$', r'^DELETE$', r'^OPTIONS$', r'^HEAD$', r'^CONNECT$', r'^TRACE$', r'^Post$' ] # Check for standalone HTTP verbs for verb in http_verbs: if re.match(verb, url): print(f"Skipping HTTP verb: {url}") return True # Check for placeholder hostnames for hostname in placeholder_hostnames: if re.search(rf'https?://{hostname}(?::[0-9]+)?/?', url, re.IGNORECASE): print(f"Skipping placeholder hostname URL: {url}") return True # Check for specific GitHub raw URLs that are placeholders for pattern in github_raw_urls: if re.search(pattern, url): print(f"Skipping GitHub raw placeholder URL: {url}") return True # Check if URL matches any of the template patterns for pattern in template_patterns: if re.search(pattern, url): print(f"Skipping false positive template URL: {url}") return True # Check if URL matches any of the placeholder patterns for pattern in placeholder_patterns: if re.search(pattern, url): print(f"Skipping false positive placeholder URL: {url}") return True # Check if URL matches any of the PowerShell variable patterns for pattern in powershell_variable_patterns: if re.search(pattern, url): print(f"Skipping PowerShell variable URL: {url}") return True # Check if URL matches any of the script file patterns for pattern in script_file_patterns: if re.search(pattern, url): print(f"Skipping script file or command URL: {url}") return True # Check if URL matches escaped backslash patterns for pattern in escaped_backslash_patterns: if re.search(pattern, url): print(f"Skipping escaped backslash URL pattern: {url}") return True # Check if URL matches any of the template variable patterns for pattern in template_variable_patterns: if re.search(pattern, url): print(f"Skipping template variable URL: {url}") return True # Check if URL matches any of the query variable patterns for pattern in query_variable_patterns: if re.search(pattern, url): print(f"Skipping query variable URL: {url}") return True # Check for local script file patterns for pattern in local_script_patterns: if re.search(pattern, url): print(f"Skipping local script file: {url}") return True # Check for GitHub API URL variable patterns for pattern in github_api_variable_patterns: if re.search(pattern, url): print(f"Skipping GitHub API URL variable: {url}") return True # Check if URL contains management API domains for domain in management_api_domains: if re.search(domain, url): print(f"Skipping management API domain: {url}") return True # Check if URL is an XML namespace if url in xml_namespace_urls or url.startswith('http://www.w3.org/2000/svg') or url.startswith('http://www.w3.org/1999/xlink'): print(f"Skipping XML namespace URL: {url}") return True # Additional check for specific URLs that we know are problematic hardcoded_urls_to_skip = [ "https://api.fabric.microsoft.com", "https://api.powerbi.com", "https://dashboards.kusto.windows.net", "https://api.kusto.windows.net", "https://analysis.windows.net", "https://wabi-us-central-b-primary-redirect.analysis.windows.net", "https://raw.githubusercontent.com/microsoft/azure_arc/main/azure_jumpstart_ag/", "http://influxPlaceholder:8086", "https://management.core.windows.net/", # Azure Management API ] if url in hardcoded_urls_to_skip: print(f"Skipping hardcoded URL: {url}") return True return False # Image file extensions to identify image links IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.ico'] # SVG files get special treatment SVG_EXTENSIONS = ['.svg'] # Parse command line arguments def parse_arguments(): """Parse command-line arguments.""" parser = argparse.ArgumentParser( description="Check URLs in files for validity." ) parser.add_argument( "--dir", help="Only check files in this directory (relative to script location)" ) parser.add_argument( "--timeout", type=int, default=TIMEOUT, help=f"Timeout in seconds for HTTP requests (default: {TIMEOUT})" ) parser.add_argument( "--exclude", nargs="*", default=[], help="Folders to exclude from checking (can specify multiple paths)" ) return parser.parse_args() # ============================================================================= # FILE & URL PROCESSING FUNCTIONS # ============================================================================= def find_files_to_check(exclude_folders=None): """ Find all supported files in the repository, skipping 'archive' folders and any user-specified excluded folders. Args: exclude_folders: List of folder paths to exclude Returns: List of file paths to check """ if exclude_folders is None: exclude_folders = [] # Convert exclude_folders to absolute paths for easier comparison abs_exclude_folders = [] for folder in exclude_folders: if os.path.isabs(folder): abs_exclude_folders.append(os.path.normpath(folder)) else: abs_exclude_folders.append(os.path.normpath(os.path.join(REPO_PATH, folder))) if exclude_folders: print(f"Excluding folders: {', '.join(exclude_folders)}") files_to_check = [] for root, dirs, files in os.walk(REPO_PATH): # Skip 'archive' folders, hidden directories, and excluded folders dirs[:] = [d for d in dirs if d.lower() != 'archive' and not d.startswith('.')] # Check if the current directory should be excluded if any(os.path.abspath(root).startswith(excluded) for excluded in abs_exclude_folders): print(f"Skipping excluded directory: {root}") dirs[:] = [] # Skip all subdirectories continue for file in files: file_ext = os.path.splitext(file)[1].lower() # Check if this is a supported file type if file_ext in SUPPORTED_FILE_TYPES: files_to_check.append(os.path.join(root, file)) return files_to_check def find_files_in_directory(directory, exclude_folders=None): """ Find all supported files in the given directory, excluding specified folders. Args: directory: Directory to search in exclude_folders: List of folder paths to exclude Returns: List of file paths to check """ if exclude_folders is None: exclude_folders = [] # Convert exclude_folders to absolute paths for easier comparison abs_exclude_folders = [] for folder in exclude_folders: if os.path.isabs(folder): abs_exclude_folders.append(os.path.normpath(folder)) else: abs_exclude_folders.append(os.path.normpath(os.path.join(directory, folder))) files_to_check = [] for root, dirs, files in os.walk(directory): # Check if the current directory should be excluded if any(os.path.abspath(root).startswith(excluded) for excluded in abs_exclude_folders): print(f"Skipping excluded directory: {root}") dirs[:] = [] # Skip all subdirectories continue for file in files: file_ext = os.path.splitext(file)[1].lower() # Check if this is a supported file type if file_ext in SUPPORTED_FILE_TYPES: files_to_check.append(os.path.join(root, file)) return files_to_check def extract_urls_by_file_type(file_path): """Extract URLs from a file based on its extension.""" file_ext = os.path.splitext(file_path)[1].lower() urls = [] file_type = SUPPORTED_FILE_TYPES.get(file_ext, 'Unknown') try: with open(file_path, 'r', encoding='utf-8') as f: content = f.read() # Markdown files if file_ext == '.md': # Use the existing markdown link extraction with open(file_path, 'r', encoding='utf-8') as md_file: for line in md_file: matches = MD_URL_REGEX.findall(line) # Strip quotes from URLs cleaned_matches = [match.strip('"\'') for match in matches] urls.extend(cleaned_matches) # HTML files elif file_ext in ['.html', '.htm']: urls.extend([url for url in HTML_HREF_REGEX.findall(content) if url]) urls.extend([url for url in HTML_SRC_REGEX.findall(content) if url]) urls.extend([url for url in HTML_LINK_HREF_REGEX.findall(content) if url]) urls.extend([url for url in HTML_META_CONTENT_REGEX.findall(content) if url and (url.startswith('http') or url.startswith('/') or url.startswith('.'))]) # CSS files elif file_ext in ['.css', '.scss']: urls.extend([url for url in CSS_URL_REGEX.findall(content) if url]) # JavaScript/TypeScript files elif file_ext in ['.js', '.jsx', '.ts', '.tsx']: js_matches = JS_URL_REGEX.findall(content) # Handle the case where we have groups in the regex for match in js_matches: if isinstance(match, tuple): urls.extend([m for m in match if m]) else: urls.append(match) # Python files elif file_ext == '.py': urls.extend([url for url in PY_URL_REGEX.findall(content) if url]) # Python imports are special - we don't check these as URLs but could in the future # JSON/YAML files elif file_ext in ['.json', '.yaml', '.yml']: urls.extend([url for url in JSON_URL_REGEX.findall(content) if url]) # XML files elif file_ext == '.xml': urls.extend([url for url in XML_URL_REGEX.findall(content) if url]) # Shell/Bash scripts elif file_ext in ['.sh', '.bash', '.zsh', '.ksh']: shell_matches = SHELL_URL_REGEX.findall(content) for match in shell_matches: if isinstance(match, tuple): urls.extend([m for m in match if m]) else: urls.append(match) # PowerShell scripts elif file_ext in ['.ps1', '.psm1', '.psd1']: ps_matches = PS_URL_REGEX.findall(content) for match in ps_matches: if isinstance(match, tuple): urls.extend([m for m in match if m]) else: urls.append(match) # Batch/CMD scripts elif file_ext in ['.bat', '.cmd']: urls.extend([url for url in BATCH_URL_REGEX.findall(content) if url]) # Perl/Ruby/Other scripting languages elif file_ext in ['.pl', '.pm', '.rb', '.php', '.lua', '.tcl', '.groovy', '.awk', '.r', '.R']: urls.extend([url for url in SCRIPT_URL_REGEX.findall(content) if url]) # Configuration files elif file_ext in ['.ini', '.conf', '.cfg', '.toml', '.env']: urls.extend([url for url in CONFIG_URL_REGEX.findall(content) if url]) # For other file types, use a generic approach to find http(s) URLs else: generic_url_regex = re.compile(r'(?:https?://[^\s\'">]+)') urls.extend([url for url in generic_url_regex.findall(content) if url]) print(f"Found {len(urls)} URLs in {file_type} file: {file_path}") except Exception as e: print(f"Error processing file {file_path}: {str(e)}") return urls def extract_urls(file_path): """Extract all URLs from a file using the appropriate method based on file type.""" return extract_urls_by_file_type(file_path) def extract_headers(md_file): """Extract all headers from a markdown file and convert to slug format for link validation.""" headers = [] # Only attempt to extract headers from markdown files if not md_file.lower().endswith('.md'): print(f"Warning: Attempted to extract headers from non-markdown file: {md_file}") return headers try: with open(md_file, 'r', encoding='utf-8') as f: for line in f: if line.strip().startswith('#'): # Extract the header text (remove the # and any leading/trailing whitespace) header_text = line.lstrip('#').strip() # Convert to lowercase header_text_lower = header_text.lower() # Remove markdown formatting (bold, italic, code) header_text_clean = re.sub(r'[*_`]', '', header_text_lower) # Create slug: keep only alphanumeric chars and hyphens, replace spaces with hyphens header_slug = re.sub(r'[^\w\- ]', '', header_text_clean) header_slug = re.sub(r'\s+', '-', header_slug) # Add to the list of headers headers.append(header_slug) print(f"Found header: '{header_text}' -> slug: '{header_slug}'") except Exception as e: print(f"Warning: Could not extract headers from {md_file}: {str(e)}") return headers def is_ip_based_url(url): """Check if a URL uses an IP address instead of a domain name.""" try: host = urlparse(url).hostname ipaddress.ip_address(host) return True except ValueError: return False # Define a list of temporary error status codes TEMPORARY_ERROR_CODES = [502, 503, 504, 429] # Added 429 (Too Many Requests) def check_absolute_url(url, md_file=None, retries=3): """ Check if an absolute URL (http/https) is reachable. Args: url: The URL to check md_file: Source markdown file containing this URL retries: Number of attempts before giving up Returns: Log entry string with result """ # Extract domain from URL for domain-based verification parsed_url = urlparse(url) domain = parsed_url.netloc is_trusted_domain = domain in KNOWN_VALID_DOMAINS print(f"Checking absolute URL: {url}") print(f"Domain: {domain}, Trusted: {is_trusted_domain}") attempt = 0 while attempt < retries: try: # Make the request with configured timeout response = requests.get(url, headers=HEADERS, allow_redirects=True, timeout=TIMEOUT, stream=True) if response.status_code < 400: log_entry = f"{Colors.OKGREEN}[OK ABSOLUTE] {url}{Colors.ENDC}" print(log_entry) return log_entry elif response.status_code in TEMPORARY_ERROR_CODES: # For temporary errors, handle differently based on trusted status print(f"Status Code {response.status_code} for {url}. Retrying... ({attempt + 1}/{retries})") attempt += 1 if attempt >= retries: file_info = f" (in file: {md_file})" if md_file else "" if is_trusted_domain: # For trusted domains, mark as OK even with temporary errors log_entry = f"{Colors.OKGREEN}[OK ABSOLUTE] {url} (trusted domain with temporary status code: {response.status_code}){file_info}{Colors.ENDC}" print(log_entry) return log_entry else: # For non-trusted domains, still mark as broken but note it might be temporary log_entry = f"{Colors.FAIL}[BROKEN ABSOLUTE] {url} - Temporary error: {response.status_code}{file_info}{Colors.ENDC}" print(log_entry) return log_entry else: file_info = f" (in file: {md_file})" if md_file else "" # For non-temporary errors, mark as broken even for trusted domains log_entry = f"{Colors.FAIL}[BROKEN ABSOLUTE] {url} - Status Code: {response.status_code}{file_info}{Colors.ENDC}" print(log_entry) return log_entry except requests.RequestException as e: file_info = f" (in file: {md_file})" if md_file else "" # For connection errors on trusted domains, consider as temporarily unavailable if is_trusted_domain and isinstance(e, ( requests.Timeout, requests.ConnectionError, requests.TooManyRedirects )): # Last retry and it's a trusted domain with connection issues if attempt >= retries - 1: log_entry = f"{Colors.OKGREEN}[OK ABSOLUTE] {url} (trusted domain, connection issue: {type(e).__name__}){file_info}{Colors.ENDC}" print(log_entry) return log_entry # Special handling for certificate errors on trusted domains if isinstance(e, requests.exceptions.SSLError): if any(trusted_domain in domain for trusted_domain in TRUSTED_DOMAINS_WITH_CERT_ISSUES) or any(trusted_domain in url for trusted_domain in TRUSTED_DOMAINS_WITH_CERT_ISSUES): log_entry = f"{Colors.OKGREEN}[OK ABSOLUTE] {url} (trusted domain with certificate issue){file_info}{Colors.ENDC}" print(log_entry) return log_entry log_entry = f"{Colors.FAIL}[BROKEN ABSOLUTE] {url} - Error: {e}{file_info}{Colors.ENDC}" print(log_entry) attempt += 1 if attempt < retries: print(f"Retrying... ({attempt}/{retries})") else: return log_entry def find_case_insensitive_path(path): """ Tries to find an existing path with case-insensitive matching. Useful for systems where the filesystem is case-sensitive but the URLs might not match case. Args: path: The path to check Returns: The correct path if found with different case, None otherwise """ # If the path exists exactly as provided, no need to search if os.path.exists(path): return path # Not found, try to match case-insensitively dirname, basename = os.path.split(path) # If the directory doesn't exist, we can't check its contents if not os.path.isdir(dirname): return None try: # Check if a case-insensitive match exists in the parent directory for entry in os.listdir(dirname): if entry.lower() == basename.lower(): return os.path.join(dirname, entry) except (PermissionError, FileNotFoundError): pass return None def find_path_case_insensitive(base_path, rel_path): """ Find a path with case-insensitive matching, handling multi-level paths. Args: base_path: Starting directory for the search rel_path: Relative path to find (can include multiple directories) Returns: Full corrected path if found, None otherwise """ # Handle empty path if not rel_path: return base_path # Split the path into components, handling both forward and back slashes path_parts = re.split(r'[/\\]', rel_path) path_parts = [part for part in path_parts if part] # Remove empty parts current_path = base_path print(f"Starting case-insensitive path search from: {current_path}") print(f"Looking for path components: {path_parts}") # Process each path component for i, part in enumerate(path_parts): # Skip if the component is '.' (current directory) if part == '.': continue # Handle '..' (parent directory) - just use it directly as it doesn't need case correction if part == '..': current_path = os.path.dirname(current_path) print(f"Going up to parent directory: {current_path}") continue # Try to find a case-insensitive match for this component found = False try: if os.path.exists(os.path.join(current_path, part)): # Exact match exists, use it directly current_path = os.path.join(current_path, part) found = True print(f"Exact match found for '{part}': {current_path}") else: # Try case-insensitive match for entry in os.listdir(current_path): if entry.lower() == part.lower(): current_path = os.path.join(current_path, entry) found = True print(f"Case-insensitive match found for '{part}': {entry} at {current_path}") break except (PermissionError, FileNotFoundError, NotADirectoryError) as e: print(f"Error accessing {current_path}: {str(e)}") return None if not found: print(f"No match found for component '{part}' in {current_path}") return None # Add trailing slash if the original path had one if rel_path.endswith('/') and not current_path.endswith(os.sep): current_path += os.sep print(f"Final resolved path: {current_path}") return current_path def check_relative_url(url, md_file): """ Check if a relative file path exists in the filesystem. Args: url: Relative path to check md_file: Source markdown file containing this path Returns: Tuple containing: (log_entry, is_image, is_svg, is_root_relative, has_anchor) """ # Flag to track if URL has an anchor has_anchor = '#' in url anchor_text = None # Handle header links (e.g., #section-name or file.md#section-name) if has_anchor and md_file.lower().endswith('.md'): base_url, anchor = url.split('#', 1) anchor_text = anchor # If it's a same-page link (just #header) if not base_url: headers = extract_headers(md_file) if anchor in headers: log_entry = f"{Colors.OKGREEN}[OK HEADER] #{anchor} (header in {md_file}){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor else: log_entry = f"{Colors.FAIL}[BROKEN HEADER] #{anchor} (header not found in {md_file}){Colors.ENDC}" print(f"Available headers in {md_file}: {', '.join(headers)}") print(log_entry) return log_entry, False, False, False, has_anchor else: # Construct the target path based on the base_url target_file = os.path.join(os.path.dirname(md_file), base_url) # Handle the case where the base_url points to a directory if os.path.isdir(target_file): print(f"Base URL {base_url} points to a directory: {target_file}") # Check if an _index.md file exists in the directory index_file = os.path.join(target_file, "_index.md") if os.path.exists(index_file): log_entry = f"{Colors.OKGREEN}[OK RELATIVE] {index_file}#{anchor} (directory with _index.md, anchor not validated){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor # Also check for other common index files for index_name in ["index.md", "README.md"]: index_file = os.path.join(target_file, index_name) if os.path.exists(index_file): log_entry = f"{Colors.OKGREEN}[OK RELATIVE] {index_file}#{anchor} (directory with {index_name}, anchor not validated){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor # Check if file exists without case sensitivity case_insensitive_path = find_path_case_insensitive(os.path.dirname(md_file), base_url) if case_insensitive_path and os.path.exists(case_insensitive_path): # Found with case-insensitive match if os.path.isdir(case_insensitive_path): # It's a directory, check for index files for index_name in ["_index.md", "index.md", "README.md"]: index_file = os.path.join(case_insensitive_path, index_name) if os.path.exists(index_file): log_entry = f"{Colors.OKGREEN}[OK RELATIVE] {index_file}#{anchor} (directory with {index_name}, case-insensitive match, anchor not validated){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor else: # It's a file log_entry = f"{Colors.OKGREEN}[OK RELATIVE] {case_insensitive_path}#{anchor} (file exists, case-insensitive match, anchor not validated){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor # Original check if file exists (case sensitive) if os.path.exists(target_file): log_entry = f"{Colors.OKGREEN}[OK RELATIVE] {target_file}#{anchor} (file exists, anchor not validated){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor else: log_entry = f"{Colors.FAIL}[BROKEN RELATIVE WITH ANCHOR] {target_file}#{anchor} (file not found){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor # Handle hash in URL for non-markdown source files elif has_anchor: base_url, anchor = url.split('#', 1) anchor_text = anchor # For non-markdown file links with anchors, we just check if the file exists if not base_url: # Same-file anchor in non-markdown file, we can't validate this log_entry = f"{Colors.OKGREEN}[OK HEADER] #{anchor} (in non-markdown file {md_file}){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor else: target_file = os.path.join(os.path.dirname(md_file), base_url) if os.path.exists(target_file): log_entry = f"{Colors.OKGREEN}[OK RELATIVE] {target_file}#{anchor} (file exists, anchor not validated){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor else: log_entry = f"{Colors.FAIL}[BROKEN RELATIVE WITH ANCHOR] {target_file}#{anchor} (file not found){Colors.ENDC}" print(log_entry) return log_entry, False, False, False, has_anchor # Check if it's an SVG file is_svg = any(url.lower().endswith(ext) for ext in SVG_EXTENSIONS) # Check if it's an image file is_image = not is_svg and any(url.lower().endswith(ext) for ext in IMAGE_EXTENSIONS) # Handle root-relative URLs (starting with /) is_root_relative = url.startswith('/') if is_root_relative: # URLs starting with / are relative to repo root, not the current file file_path = os.path.join(REPO_PATH, url[1:]) # Remove leading / and join with repo root print(f"Root-relative path detected. Checking against repo root: {file_path}") else: # Regular document-relative URL file_path = os.path.join(os.path.dirname(md_file), url) file_type = "SVG" if is_svg else "image" if is_image else "root-relative" if is_root_relative else "relative" print(f"Checking {file_type} URL: {file_path}") # -- New Approach: Handle case sensitivity more robustly -- # Check if path exists directly path_exists = os.path.exists(file_path) # If path doesn't exist, try case-insensitive matching if not path_exists: print(f"Path not found: {file_path}") print(f"Trying case-insensitive path resolution...") # For directory URLs (ending with /) if url.endswith('/'): # Split the file_path into components path_parts = os.path.normpath(file_path).split(os.sep) # Start from an existing directory current = os.path.dirname(md_file) if not is_root_relative else REPO_PATH built_path = current # Process each segment of the relative path rel_segments = url.rstrip('/').split('/') print(f"Processing relative segments: {rel_segments}") for segment in rel_segments: if segment == '..': # Go up one directory current = os.path.dirname(current) built_path = current print(f"Going up to parent: {current}") elif segment == '.': # Stay in current directory continue else: # Try to find a case-insensitive match for this segment if os.path.exists(os.path.join(current, segment)): # Exact case match current = os.path.join(current, segment) built_path = current print(f"Exact match found: {segment}") else: found = False try: for item in os.listdir(current): if item.lower() == segment.lower(): current = os.path.join(current, item) built_path = current print(f"Case-insensitive match found: {segment} -> {item}") found = True break except (PermissionError, FileNotFoundError, NotADirectoryError) as e: print(f"Error accessing {current}: {str(e)}") if not found: print(f"No match found for segment: {segment} in {current}") break if os.path.exists(built_path): file_path = built_path path_exists = True print(f"Successfully resolved case-insensitive path: {built_path}") # Check for default files in the directory if os.path.isdir(built_path): for default_file in ['_index.md', 'index.md', 'README.md']: default_path = os.path.join(built_path, default_file) if os.path.exists(default_path): file_path = default_path print(f"Found default file: {default_path}") break # If path still doesn't exist and it's a directory URL, try to check for markdown files if not path_exists and url.endswith('/') and os.path.isdir(os.path.dirname(file_path)): try: md_files = [f for f in os.listdir(file_path) if f.endswith('.md')] if md_files: path_exists = True file_path = os.path.join(file_path, md_files[0]) # Use the first markdown file found print(f"Directory contains markdown files: {', '.join(md_files)}") else: print(f"Directory exists but contains no markdown files") except PermissionError: print(f"Permission error accessing directory: {file_path}") except FileNotFoundError: print(f"Directory doesn't exist: {file_path}") if path_exists: if is_svg: log_entry = f"{Colors.OKGREEN}[OK SVG] {file_path}{Colors.ENDC}" elif is_image: log_entry = f"{Colors.OKGREEN}[OK IMAGE] {file_path}{Colors.ENDC}" elif is_root_relative: log_entry = f"{Colors.OKGREEN}[OK ROOT-RELATIVE] {file_path} (root-relative path: {url}){Colors.ENDC}" else: log_entry = f"{Colors.OKGREEN}[OK RELATIVE] {file_path}{Colors.ENDC}" print(log_entry) return log_entry, is_image, is_svg, is_root_relative, has_anchor else: if is_svg: log_entry = f"{Colors.FAIL}[BROKEN SVG] {file_path} (SVG in {md_file}){Colors.ENDC}" elif is_image: log_entry = f"{Colors.FAIL}[BROKEN IMAGE] {file_path} (image in {md_file}){Colors.ENDC}" elif is_root_relative: log_entry = f"{Colors.FAIL}[BROKEN ROOT-RELATIVE] {file_path} (root-relative path: {url} in {md_file}){Colors.ENDC}" else: # Update the log message to indicate whether the URL has an anchor or not if has_anchor: log_entry = f"{Colors.FAIL}[BROKEN RELATIVE WITH ANCHOR] {url} (relative path in {md_file}){Colors.ENDC}" else: log_entry = f"{Colors.FAIL}[BROKEN RELATIVE WITHOUT ANCHOR] {url} (relative path in {md_file}){Colors.ENDC}" print(log_entry) return log_entry, is_image, is_svg, is_root_relative, has_anchor def strip_ansi_escape_codes(text): """Remove ANSI color codes from text (for clean log files).""" return ANSI_ESCAPE_REGEX.sub('', text) # ============================================================================= # MAIN EXECUTION # ============================================================================= def main(): # Parse arguments args = parse_arguments() # Override timeout if provided global TIMEOUT if args.timeout: TIMEOUT = args.timeout print(f"Using custom timeout: {TIMEOUT} seconds") # Lists to track results broken_absolute_urls = [] ok_absolute_urls = [] broken_relative_urls_with_anchor = [] broken_relative_urls_without_anchor = [] ok_relative_urls = [] broken_image_urls = [] ok_image_urls = [] broken_svg_urls = [] ok_svg_urls = [] broken_header_urls = [] ok_header_urls = [] broken_root_relative_urls = [] ok_root_relative_urls = [] no_links_types = [] # If a specific directory is provided, only check files there if args.dir: script_dir = os.path.dirname(os.path.abspath(__file__)) test_dir = os.path.join(script_dir, args.dir) print(f"Only checking files in test directory: {test_dir}") files_to_check = find_files_in_directory(test_dir, args.exclude) else: files_to_check = find_files_to_check(args.exclude) # Create log file with timestamp timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') log_file_with_timestamp = os.path.join(LOG_DIR, f'broken_urls_{timestamp}.log') print(f"Starting URL check on {len(files_to_check)} files...") start_time = datetime.now() # Process all files and URLs - write to log in real-time for monitoring with open(log_file_with_timestamp, 'w', encoding='utf-8') as log: log.write(f"URL Checker Results\n\n") log.write(f"Log generated on: {timestamp}\n") log.write("Processing URLs in real-time...\n\n") log.flush() for file_path in files_to_check: file_ext = os.path.splitext(file_path)[1].lower() file_type = SUPPORTED_FILE_TYPES.get(file_ext, 'Unknown') print(f"Processing {file_type} file: {file_path}") urls = extract_urls(file_path) for url in urls: # Skip email links if EMAIL_REGEX.match(url): print(f"Skipping email URL: {url}") continue # Skip localhost and IP-based URLs if url.startswith("http://localhost") or is_ip_based_url(url): print(f"Skipping localhost or IP-based URL: {url}") continue # Skip false positive URLs if is_false_positive(url): continue # Add error handling for URL parsing try: # Check URL based on whether it's absolute or relative parsed_url = urlparse(url) if parsed_url.scheme in ('http', 'https'): # It's an absolute URL - pass the file path to track source log_entry = check_absolute_url(url, file_path) if "[OK ABSOLUTE]" in log_entry: ok_absolute_urls.append(log_entry) else: broken_absolute_urls.append(log_entry) else: # Strip quotes before further processing to avoid false positives url_clean = url.strip('"\'') try: parsed_clean = urlparse(url_clean) # Check again if it's actually an absolute URL after stripping quotes if parsed_clean.scheme in ('http', 'https'): # Skip false positive URLs after cleaning if is_false_positive(url_clean): continue log_entry = check_absolute_url(url_clean, file_path) if "[OK ABSOLUTE]" in log_entry: ok_absolute_urls.append(log_entry) else: broken_absolute_urls.append(log_entry) else: # It's a relative URL, image, SVG, root-relative, or header link log_entry, is_image, is_svg, is_root_relative, has_anchor = check_relative_url(url, file_path) # ...existing categorization code... if "[BROKEN HEADER]" in log_entry: broken_header_urls.append(log_entry) elif "[OK HEADER]" in log_entry: ok_header_urls.append(log_entry) elif is_svg: if "[OK SVG]" in log_entry: ok_svg_urls.append(log_entry) else: broken_svg_urls.append(log_entry) elif is_image: if "[OK IMAGE]" in log_entry: ok_image_urls.append(log_entry) else: broken_image_urls.append(log_entry) elif is_root_relative: if "[OK ROOT-RELATIVE]" in log_entry: ok_root_relative_urls.append(log_entry) else: broken_root_relative_urls.append(log_entry) else: if "[OK RELATIVE]" in log_entry: ok_relative_urls.append(log_entry) else: # Use the new log message format for categorization if "[BROKEN RELATIVE WITH ANCHOR]" in log_entry: broken_relative_urls_with_anchor.append(log_entry) elif "[BROKEN RELATIVE WITHOUT ANCHOR]" in log_entry: broken_relative_urls_without_anchor.append(log_entry) except ValueError as e: # Handle URL parsing errors for the cleaned URL error_message = str(e) log_entry = f"{Colors.FAIL}[MALFORMED URL] {url_clean} - Error: {error_message} (in file: {file_path}){Colors.ENDC}" print(log_entry) broken_absolute_urls.append(log_entry) except ValueError as e: # Handle URL parsing errors error_message = str(e) if "Invalid IPv6 URL" in error_message: log_entry = f"{Colors.FAIL}[MALFORMED URL] {url} - Invalid IPv6 URL format (in file: {file_path}){Colors.ENDC}" else: log_entry = f"{Colors.FAIL}[MALFORMED URL] {url} - Error: {error_message} (in file: {file_path}){Colors.ENDC}" print(log_entry) broken_absolute_urls.append(log_entry) # Write to log file (real-time monitoring) log.write(strip_ansi_escape_codes(log_entry) + "\n") log.flush() # Calculate runtime end_time = datetime.now() runtime_duration = end_time - start_time runtime_seconds = runtime_duration.total_seconds() # Create a human-readable runtime string if runtime_seconds < 60: runtime_str = f"{runtime_seconds:.2f} seconds" elif runtime_seconds < 3600: runtime_str = f"{runtime_seconds/60:.2f} minutes ({runtime_duration})" else: runtime_str = f"{runtime_seconds/3600:.2f} hours ({runtime_duration})" # Write the log file with organized results with open(log_file_with_timestamp, 'w', encoding='utf-8') as log: log.write(f"URL Checker Results\n\n") log.write(f"Log generated on: {timestamp}\n") log.write(f"Runtime: {runtime_str}\n") log.write(f"Runtime duration: {runtime_duration}\n\n") # Write broken sections first (most important) log.write(f"=== Broken Absolute URLs ({len(broken_absolute_urls)} links found) ===\n\n") if broken_absolute_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in broken_absolute_urls) + "\n\n") else: log.write("No broken absolute URLs found.\n\n") log.write(f"=== Broken Relative URLs Without Anchors ({len(broken_relative_urls_without_anchor)} links found) ===\n\n") if broken_relative_urls_without_anchor: log.write("\n".join(strip_ansi_escape_codes(url) for url in broken_relative_urls_without_anchor) + "\n\n") else: log.write("No broken relative URLs without anchors found.\n\n") log.write(f"=== Broken Relative URLs With Anchors ({len(broken_relative_urls_with_anchor)} links found) ===\n\n") if broken_relative_urls_with_anchor: log.write("\n".join(strip_ansi_escape_codes(url) for url in broken_relative_urls_with_anchor) + "\n\n") else: log.write("No broken relative URLs with anchors found.\n\n") log.write(f"=== Broken Root-Relative URLs ({len(broken_root_relative_urls)} links found) ===\n\n") if broken_root_relative_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in broken_root_relative_urls) + "\n\n") else: log.write("No broken root-relative URLs found.\n\n") log.write(f"=== Broken Image URLs ({len(broken_image_urls)} links found) ===\n\n") if broken_image_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in broken_image_urls) + "\n\n") else: log.write("No broken image URLs found.\n\n") log.write(f"=== Broken SVG URLs ({len(broken_svg_urls)} links found) ===\n\n") if broken_svg_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in broken_svg_urls) + "\n\n") else: log.write("No broken SVG URLs found.\n\n") log.write(f"=== Broken Header Links ({len(broken_header_urls)} links found) ===\n\n") if broken_header_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in broken_header_urls) + "\n\n") else: log.write("No broken header links found.\n\n") log.write(f"=== OK Absolute URLs ({len(ok_absolute_urls)} links found) ===\n\n") if ok_absolute_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in ok_absolute_urls) + "\n\n") else: log.write("No absolute URLs found.\n\n") log.write(f"=== OK Relative URLs ({len(ok_relative_urls)} links found) ===\n\n") if ok_relative_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in ok_relative_urls) + "\n\n") else: log.write("No relative URLs found.\n\n") log.write(f"=== OK Root-Relative URLs ({len(ok_root_relative_urls)} links found) ===\n\n") if ok_root_relative_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in ok_root_relative_urls) + "\n\n") else: log.write("No root-relative URLs found.\n\n") log.write(f"=== OK Image URLs ({len(ok_image_urls)} links found) ===\n\n") if ok_image_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in ok_image_urls) + "\n\n") else: log.write("No image URLs found.\n\n") log.write(f"=== OK SVG URLs ({len(ok_svg_urls)} links found) ===\n\n") if ok_svg_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in ok_svg_urls) + "\n\n") else: log.write("No SVG URLs found.\n\n") log.write(f"=== OK Header Links ({len(ok_header_urls)} links found) ===\n\n") if ok_header_urls: log.write("\n".join(strip_ansi_escape_codes(url) for url in ok_header_urls) + "\n\n") else: log.write("No header links found.\n\n") # Add summary with improved informative title and hierarchical format total_broken = (len(broken_absolute_urls) + len(broken_relative_urls_with_anchor) + len(broken_relative_urls_without_anchor) + len(broken_root_relative_urls) + len(broken_image_urls) + len(broken_svg_urls) + len(broken_header_urls) ) total_ok = len(ok_absolute_urls) + len(ok_relative_urls) + len(ok_root_relative_urls) + len(ok_image_urls) + len(ok_svg_urls) + len(ok_header_urls) total_links = total_broken + total_ok # Updated categorization logic no_links_types = [] # Categories with no links at all (neither broken nor OK) zero_broken_types = [] # Categories with OK links but no broken links broken_types = [] # Categories with broken links # Absolute URLs if len(broken_absolute_urls) == 0 and len(ok_absolute_urls) == 0: no_links_types.append(("Absolute URLs", 0)) elif len(broken_absolute_urls) == 0: zero_broken_types.append(("Absolute URLs", len(ok_absolute_urls))) else: broken_types.append(("Absolute URLs", len(broken_absolute_urls))) # Relative URLs without anchors and with anchors combined if len(broken_relative_urls_without_anchor) == 0 and len(broken_relative_urls_with_anchor) == 0 and len(ok_relative_urls) == 0: no_links_types.append(("Relative URLs", 0)) elif len(broken_relative_urls_without_anchor) == 0 and len(broken_relative_urls_with_anchor) == 0: zero_broken_types.append(("Relative URLs", len(ok_relative_urls))) else: # Count broken relative URLs with and without anchors separately if len(broken_relative_urls_without_anchor) > 0: broken_types.append(("Relative URLs without anchors", len(broken_relative_urls_without_anchor))) if len(broken_relative_urls_with_anchor) > 0: broken_types.append(("Relative URLs with anchors", len(broken_relative_urls_with_anchor))) # Root-relative URLs if len(broken_root_relative_urls) == 0 and len(ok_root_relative_urls) == 0: no_links_types.append(("Root-relative URLs", 0)) elif len(broken_root_relative_urls) == 0: zero_broken_types.append(("Root-relative URLs", len(ok_root_relative_urls))) else: broken_types.append(("Root-relative URLs", len(broken_root_relative_urls))) # Image URLs if len(broken_image_urls) == 0 and len(ok_image_urls) == 0: no_links_types.append(("Image URLs", 0)) elif len(broken_image_urls) == 0: zero_broken_types.append(("Image URLs", len(ok_image_urls))) else: broken_types.append(("Image URLs", len(broken_image_urls))) # SVG URLs if len(broken_svg_urls) == 0 and len(ok_svg_urls) == 0: no_links_types.append(("SVG URLs", 0)) elif len(broken_svg_urls) == 0: zero_broken_types.append(("SVG URLs", len(ok_svg_urls))) else: broken_types.append(("SVG URLs", len(broken_svg_urls))) # Header links if len(broken_header_urls) == 0 and len(ok_header_urls) == 0: no_links_types.append(("Header links", 0)) elif len(broken_header_urls) == 0: zero_broken_types.append(("Header links", len(ok_header_urls))) else: broken_types.append(("Header links", len(broken_header_urls))) # Write modernized summary to log file log.write("\n" + "═" * 80 + "\n") log.write(f"📊 LINK VALIDATION SUMMARY ({total_links} links checked)\n") log.write("═" * 80 + "\n\n") # Always show broken links section if there are any broken links if total_broken > 0: log.write(f"❌ BROKEN LINKS: {total_broken}\n") # Only show categories that actually have broken links for category, count in broken_types: log.write(f" • {category}: {count}\n") log.write("\n") else: log.write(f"✅ BROKEN LINKS: 0 (All links are valid!)\n\n") # Show categories with no links found if no_links_types: log.write(f"📭 NO LINKS FOUND: {len(no_links_types)}\n") for category, _ in no_links_types: log.write(f" • {category}\n") log.write("\n") # Show categories with no broken links (but have OK links) if zero_broken_types: log.write(f"🔍 CATEGORIES WITH NO BROKEN LINKS: {len(zero_broken_types)}\n") for category, count in zero_broken_types: log.write(f" • {category}: {count} OK links\n") log.write("\n") log.write(f"✅ OK LINKS: {total_ok}\n\n") # Add runtime to log summary log.write(f"⏱️ RUNTIME: {runtime_str}\n\n") # Add final conclusion with emoji broken_links_found = bool(broken_absolute_urls or broken_relative_urls_with_anchor or broken_relative_urls_without_anchor or broken_root_relative_urls or broken_image_urls or broken_svg_urls or broken_header_urls) if broken_links_found: log.write(f"❌ Broken links were found. Check the logs for details.\n") else: log.write(f"✅ All links are valid!\n") # Print results to console print(f"Check complete. See {log_file_with_timestamp} for details.") print(f"\nLog generated on: {timestamp}") print(f"{Colors.INFO}Runtime: {runtime_str}{Colors.ENDC}") print(f"Runtime duration: {runtime_duration}") print(f"Total broken absolute URLs: {len(broken_absolute_urls)}") print(f"Total broken relative URLs (without anchors): {len(broken_relative_urls_without_anchor)}") print(f"Total broken relative URLs (with anchors): {len(broken_relative_urls_with_anchor)}") print(f"Total OK absolute URLs: {len(ok_absolute_urls)}") print(f"Total OK relative URLs: {len(ok_relative_urls)}") print(f"Total broken root-relative URLs: {len(broken_root_relative_urls)}") print(f"Total OK root-relative URLs: {len(ok_root_relative_urls)}") print(f"Total broken image URLs: {len(broken_image_urls)}") print(f"Total OK image URLs: {len(ok_image_urls)}") print(f"Total broken SVG URLs: {len(broken_svg_urls)}") print(f"Total OK SVG URLs: {len(ok_svg_urls)}") print(f"Total broken header links: {len(broken_header_urls)}") print(f"Total OK header links: {len(ok_header_urls)}") # Update these sections to match log file format print(f"\n=== Broken Absolute URLs ({len(broken_absolute_urls)} links found) ===") if broken_absolute_urls: for url in broken_absolute_urls: print(f"{Colors.FAIL}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No broken absolute URLs found.") print(f"\n=== Broken Relative URLs Without Anchors ({len(broken_relative_urls_without_anchor)} links found) ===") if broken_relative_urls_without_anchor: for url in broken_relative_urls_without_anchor: print(f"{Colors.FAIL}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No broken relative URLs without anchors found.") print(f"\n=== Broken Relative URLs With Anchors ({len(broken_relative_urls_with_anchor)} links found) ===") if broken_relative_urls_with_anchor: for url in broken_relative_urls_with_anchor: print(f"{Colors.FAIL}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No broken relative URLs with anchors found.") print(f"\n=== Broken Root-Relative URLs ({len(broken_root_relative_urls)} links found) ===") if broken_root_relative_urls: for url in broken_root_relative_urls: print(f"{Colors.FAIL}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No broken root-relative URLs found.") print(f"\n=== Broken Image URLs ({len(broken_image_urls)} links found) ===") if broken_image_urls: for url in broken_image_urls: print(f"{Colors.FAIL}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No broken image URLs found.") print(f"\n=== Broken SVG URLs ({len(broken_svg_urls)} links found) ===") if broken_svg_urls: for url in broken_svg_urls: print(f"{Colors.FAIL}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No broken SVG URLs found.") print(f"\n=== Broken Header Links ({len(broken_header_urls)} links found) ===") if broken_header_urls: for url in broken_header_urls: print(f"{Colors.FAIL}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No broken header links found.") print(f"\n=== OK Absolute URLs ({len(ok_absolute_urls)} links found) ===") if ok_absolute_urls: for url in ok_absolute_urls: print(f"{Colors.OKGREEN}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No absolute URLs found.") print(f"\n=== OK Relative URLs ({len(ok_relative_urls)} links found) ===") if ok_relative_urls: for url in ok_relative_urls: print(f"{Colors.OKGREEN}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No relative URLs found.") print(f"\n=== OK Root-Relative URLs ({len(ok_root_relative_urls)} links found) ===") if ok_root_relative_urls: for url in ok_root_relative_urls: print(f"{Colors.OKGREEN}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No root-relative URLs found.") print(f"\n=== OK Image URLs ({len(ok_image_urls)} links found) ===") if ok_image_urls: for url in ok_image_urls: print(f"{Colors.OKGREEN}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No image URLs found.") print(f"\n=== OK SVG URLs ({len(ok_svg_urls)} links found) ===") if ok_svg_urls: for url in ok_svg_urls: print(f"{Colors.OKGREEN}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No SVG URLs found.") print(f"\n=== OK Header Links ({len(ok_header_urls)} links found) ===") if ok_header_urls: for url in ok_header_urls: print(f"{Colors.OKGREEN}{strip_ansi_escape_codes(url)}{Colors.ENDC}") else: print("No header links found.") # Print modernized summary table with improved title and color coding total_broken = (len(broken_absolute_urls) + len(broken_relative_urls_with_anchor) + len(broken_relative_urls_without_anchor) + len(broken_root_relative_urls) + len(broken_image_urls) + len(broken_svg_urls) + len(broken_header_urls)) total_ok = len(ok_absolute_urls) + len(ok_relative_urls) + len(ok_root_relative_urls) + len(ok_image_urls) + len(ok_svg_urls) + len(ok_header_urls) total_links = total_broken + total_ok # Enhanced title with borders - keep this one cyan print(f"\n{Colors.INFO}═════════════════════════════════════════════════════════{Colors.ENDC}") print(f"{Colors.INFO}📊 LINK VALIDATION SUMMARY ({total_links} links checked){Colors.ENDC}") print(f"{Colors.INFO}═════════════════════════════════════════════════════════{Colors.ENDC}") print() # Always show broken links section if there are any broken links if total_broken > 0: print(f"{Colors.FAIL}❌ BROKEN LINKS: {total_broken}{Colors.ENDC}") # Only show categories that actually have broken links for category, count in broken_types: print(f"{Colors.FAIL} • {category}: {count}{Colors.ENDC}") print() else: print(f"{Colors.OKGREEN}✅ BROKEN LINKS: 0 (All links are valid!){Colors.ENDC}") print() # Show categories with no links found if no_links_types: print(f"{Colors.NEUTRAL}📭 NO LINKS FOUND: {len(no_links_types)}{Colors.ENDC}") for category, _ in no_links_types: print(f"{Colors.NEUTRAL} • {category}{Colors.ENDC}") print() # Show categories with no broken links but with OK links - use SPECIAL color (magenta) if zero_broken_types: print(f"{Colors.SPECIAL}🔍 CATEGORIES WITH NO BROKEN LINKS: {len(zero_broken_types)}{Colors.ENDC}") for category, count in zero_broken_types: print(f"{Colors.SPECIAL} • {category}: {count} OK links{Colors.ENDC}") print() # Keep this green for consistency with checkmarks print(f"{Colors.OKGREEN}✅ OK LINKS: {total_ok}{Colors.ENDC}") print() # Add runtime to console summary with emoji - use the same color as the section headers print(f"{Colors.INFO}⏱️ RUNTIME: {runtime_str}{Colors.ENDC}") print() # Determine if any broken links were found broken_links_found = bool(broken_absolute_urls or broken_relative_urls_with_anchor or broken_relative_urls_without_anchor or broken_root_relative_urls or broken_image_urls or broken_svg_urls or broken_header_urls) # Add a message about where the log file is saved - use the same color as the section headers print(f"{Colors.INFO}📄 FULL LOGS: {log_file_with_timestamp}{Colors.ENDC}") print() # Exit with appropriate code and final conclusion if broken_links_found: print(f"{Colors.FAIL}❌ Broken links were found. Check the logs for details.{Colors.ENDC}") sys.exit(1) # Exit code 1 signals that broken links were found else: print(f"{Colors.OKGREEN}✅ All links are valid!{Colors.ENDC}") sys.exit(0) # Exit code 0 signals that all links are valid if __name__ == "__main__": main()