def process_distributions()

in plugins/asfdata.py [0:0]

• 86 lines of code
• 23 McCabe index (conditional complexity)

def process_distributions(project, src, sort_revision, debug):
    if debug:
        print(f'releases: {project}')

    # current date information will help process svn ls results
    gatherDate = datetime.datetime.utcnow()
    gatherYear = gatherDate.year

    # information to accumulate
    signatures = {}
    checksums = {}
    fsizes = {}
    dtms = {}
    versions = {}
    revisions = {}
    keys = None # ensure defined before use

    # read the output from svn ls -Rv
    url = f'https://dist.apache.org/repos/dist/release/{project}'
    if debug:
        print(f'releases: {url}')
    with os_popen(['svn', 'ls', '-Rv', url]) as s:
        for line in s.stdout:
            line = line.strip()
            listing = line.split(' ')
            if line[-1:] == '/':
                # skip directories
                continue
            if sort_revision:
                revision = int(listing[0])
            else:
                revision = 0
            # user = listing[1]
            if listing[-6] == '':
                # dtm in the past year
                dtm1 = datetime.datetime.strptime(" ".join(listing[-4:-2]) + " " + str(gatherYear), SVN_DATE_FORMAT)
                if dtm1 > gatherDate:
                    dtm1 = datetime.datetime.strptime(" ".join(listing[-4:-2]) + " " + str(gatherYear - 1), SVN_DATE_FORMAT)
                fsize = listing[-5]
            else:
                # dtm older than one year
                dtm1 = datetime.datetime.strptime(" ".join(listing[-5:-1]), SVN_DATE_FORMAT)
                fsize = listing[-6]
            # date is close enough
            dtm = dtm1.strftime("%m/%d/%Y")
            # covert to number of MB
            if float(fsize) > 524288:
                fsize = ('%.2f' % bytesto(fsize, 'm')) + ' MB'
            else:
                fsize = ('%.2f' % bytesto(fsize, 'k')) + ' KB'
            # line is path
            line = listing[-1]
            # fields are parts of the path
            fields = line.split('/')
            # filename os the final part
            filename = fields[-1]
            # parts includes the whole path
            parts = line.split('.')
            # use the path as a key for each release
            release = line
            if filename:
                if re.search('KEYS(\\.txt)?$', filename):
                    # save the KEYS file url
                    keys = f'https://downloads.apache.org/{project}/{line}'
                elif re.search('\\.(asc|sig)$', filename, flags=re.IGNORECASE):
                    # we key a release off of a signature. remove the extension
                    release = '.'.join(parts[:-1])
                    signatures[release] = filename
                    # the path to the signature is used as the version
                    versions[release] = '/'.join(fields[:-1])
                    # we use the revision for sorting
                    revisions[release] = revision
                    if re.search(src, filename):
                        # put source distributions in the front (it is a reverse sort)
                        revisions[release] = revision + 100000
                elif re.search('\\.(sha512|sha1|sha256|sha|md5|mds)$', filename, flags=re.IGNORECASE):
                    # some projects checksum their signatures
                    part0 = ".".join(line.split('.')[-2:-1])
                    if part0 == "asc":
                        # skip files that are hashes of signatures
                        continue
                    # strip the extension to get the release name
                    release = '.'.join(parts[:-1])
                    checksums[release] = filename
                else:
                    # for the released file save the size and dtm
                    fsizes[release] = fsize
                    dtms[release] = dtm

    # separate versions.
    each_version = {}
    for rel in signatures:
        version = versions[rel]
        if version not in each_version:
            each_version[version] = []
        release = rel[len(version) + 1:]
        try:
            each_version[version].append( Distribution(release=release,
                                                       revision=revisions[rel],
                                                       signature=signatures[rel],
                                                       checksum=checksums[rel],
                                                       dtm=dtms[rel],
                                                       fsize=fsizes[rel]))
        except Exception:
            traceback.print_exc()

    distributions = []
    for version in each_version:
        each_version[version].sort(key=lambda x: (-x.revision, x.release))
        distributions.append( Version(version=version,
                                      name=' '.join(version.split('/')),
                                      revision=each_version[version][0].revision,
                                      release=each_version[version]))
    distributions.sort(key=lambda x: (-x.revision, x.version))
    return keys, distributions