#!/usr/bin/env python3 import errtee # pylint: disable=unused-import from collections import defaultdict import gzip import json from os.path import dirname, join, realpath from urlutils import UrlCache """ Reads the list of files in https://downloads.apache.org/zzz/find-ls.gz Creates: ../../site/json/foundation/releases.json Format: { top-level dir: { release-id: date}, ... } The release id is derived from the filename by removing common suffixes etc, see cleanFilename() The date comes from the first entry ../../site/json/foundation/releases-files.json Format: { top-level dir: { release-id: [list of files for that release-id]}, ... } """ # Listing generated by find(1) -ls FIND_LS = 'https://downloads.apache.org/zzz/find-ls2.txt.gz' # key: committee-id, value: dict(key: release version, value: date) releases = defaultdict(dict) # key: committee-id, value: dict(key: release version, value: list of file names for the release) files = defaultdict(lambda: defaultdict(list)) def cleanFilename(filename): """ Attempts to determine the release id to which a file belongs Strips extensions such as .tgz etc, then suffixes such as -sources Replaces qualifiers such as -assembly-, -parent- by '-' Returns the simplified filename . N.B. the ordering is significant, as the list is only scanned once """ for suffix in ['-all', '-src', '_src', '.src', '-sources', '_sources', '-source', '-bin', '-dist', '-source-release', '-source-relase', '-tests', '-test', '-debug', '-uber', '-macosx', '-distribution', '-example', '-native', '-win', '-win32', '-linux', '-pack', '-packaged', '-current', '-embedded', '-py', '-py2', '-py2.6', '-py2.7', '-no', 'unix-distro', 'windows-distro', 'with', '-dep', '-standalone', '-webapp', '-dom', '-om', '-32bit', '-i386', '_i386', '.i386', '-minimal', '-jettyconfig', '-py2.py3-none-any', 'newkey', 'oldkey', 'jars', '-jre13', '-hadoop1', '-hadoop2', '-project', '-with-dependencies', '-client', '-server', 'server-webapps', '-full', '-all', '-standard', '-for-javaee', '-for-tomcat', 'hadoop1-scala2', '-deployer', '-fulldocs', '-embed', '-apps', '-app', '-ref', '-installer', '-bundle', '-java']: # The above list could be simplified further if filename.endswith(suffix): filename = filename[0:len(filename)-len(suffix)] for repl in ['-assembly-', '-minimal-', '-doc-', '-src-', '-webapp-', '-standalone-', '-parent-', '-project-', '-win32-']: filename = filename.replace(repl, '-') return filename def cleanReleases(committeeId): if len(releases[committeeId]) == 0: del releases[committeeId] del files[committeeId] # all source releases must be one of these VALID_TYPES = ['tgz', 'gz', 'zip', 'xz', 'bz2'] # for gz, xz and bz2, the next extension must be tar TAR_TYPES = ['gz', 'xz', 'bz2'] # file name stems that finish with these strings are not source archives: NON_SOURCE_ENDS = ['-amd64', '-aarch64', '-arm64', '.bin', '-bin', '-binary', '-deps', '-docs', '-javadoc', '-doc', '-lib', '-lib-debug', '-manual', '-site', '-x64', '-x86', 'x86_64', '-ia32', '-i64', '-war', '-64bit', '-arm64bit', '-doc', '-apidocs', '-bundle'] # stems that match these strings are not source archives: NON_SOURCE_MATCH = ['-bin-', '-binary-', '-docs-', 'x86-windows', 'x64-windows'] # Warning: beware of accidentally matching Maven plugins! # filters for dirs, matches and ends that may only apply to certain PMCs CTTEE_FILTERS = { "solr": { "ENDS": ['-slim'], "MATCH": [], "DIRS": ['helm-charts'] } } # Don't visit these directories SKIP_DIRS = ['META', 'aarch64current', 'bin', 'binaries', 'binary', 'changes', 'cpp', 'css', 'doc', 'docs', 'eclipse', 'features', 'hidden', 'images', 'issuesfixed', 'notes', 'patches', 'php', 'py', 'py3', 'repos', 'ruby', 'stable', 'stable1', 'stable2', 'styles', 'tmp', 'updatesite', 'website', 'wikipages'] def parseFile(committeeId, file, date, path): parts = file.split('.') ext = parts.pop() # final extension if not ext in VALID_TYPES or (ext in TAR_TYPES and parts.pop() != 'tar'): return stem = ".".join(parts) # the filename stem without the archive suffice(s) if (any(stem.endswith(end) for end in NON_SOURCE_ENDS + CTTEE_FILTERS.get(committeeId,{}).get('ENDS',[])) or any(mat in stem for mat in NON_SOURCE_MATCH + CTTEE_FILTERS.get(committeeId,{}).get('MATCH',[]))): return filename = cleanFilename(stem) if len(filename) > 1: if filename not in releases[committeeId]: releases[committeeId][filename] = date files[committeeId][filename] = [] print(f" - {filename}\t\t\t{file}") files[committeeId][filename].append(path) def main(): uc = UrlCache(silent=True) find_ls = uc.get(FIND_LS, name='find-ls2.txt.gz') # -rw-rw-r-- 1 svnwc svnwc 479 2022-06-17 12:55 UTC ./.htaccess # 0 1 2 3 4 5 6 7 8 with gzip.open(find_ls, mode='rt') as r: for l in r: fields = l.split() # split the find line (the split drops the final LF) if not fields[0].startswith('-'): # only want plain files continue path = fields[8][2:] # last entry on line is the path; also drop the ./ prefix segs = path.split('/') if len(segs) == 1: # ignore top level files continue file = segs.pop() # basename # Ignore invisible files if file.startswith('.') or file in ['favicon.ico', 'META']: continue committeeId = segs[0] if any( seg in SKIP_DIRS + CTTEE_FILTERS.get(committeeId,{}).get('DIRS',[]) for seg in segs): # print('SKIP', segs) continue if committeeId in ['zzz']: continue if committeeId == 'incubator': podling = segs[1] committeeId = f'incubator-{podling}' # Now store the info stamp = fields[5] parseFile(committeeId, file, stamp, path) if __name__ == '__main__': mypath = realpath(__file__) assert '/scripts/cronjobs/' in mypath, "Expected this source file to be under scripts/cronjobs !" myhome = dirname(dirname(dirname(mypath))) # home dir is ../.. jsondir = join(myhome, 'site', 'json', 'foundation') # where the JSON files go main() print("Writing releases.json") with open(join(jsondir, "releases.json"), "w", encoding='utf-8') as f: json.dump(releases, f, sort_keys=True, indent=0) print("Writing releases-files.json") with open(join(jsondir, "releases-files.json"), "w", encoding='utf-8') as f: json.dump(files, f, sort_keys=True, indent=0) print("All done!")