in tools/build/build.py [0:0]
def pull_source(dst_dir: tempfile.TemporaryDirectory, build: dict):
ASF = 'ASF (dist.apache.org)'
GHB = 'GitHub'
LFS = 'Local Filesystem'
source_location = choice_prompt(
'Where is the source you\'re building from stored?',
[ASF, GHB, LFS],
ASF
)
if source_location == ASF:
DEV = 'Dev area (release candidates)'
REL = 'Most recent release area'
ARC = 'Archive (full release history)'
url_map = {
DEV: 'https://dist.apache.org/repos/dist/dev/',
REL: 'https://dist.apache.org/repos/dist/release/',
ARC: 'https://archive.apache.org/dist/'
}
release_area = choice_prompt(
'Where is the release you\'re looking for?',
[DEV, REL, ARC]
)
url = url_map[release_area] + 'sdap/'
response = requests.get(url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
versions = [
node.text.rstrip('/') for node in soup.find_all('a') if node.get('href').rstrip('/') not in ['KEYS', '..']
]
# Extra filtering to remove some special values in archive HTML page
versions = [
v for v in versions if v not in ['Parent Directory', 'Name', 'Last modified', 'Size', 'Description']
]
if len(versions) == 0:
print('There is nothing in this area to build...')
exit(0)
version = choice_prompt(
'Choose a release/release candidate to build',
versions,
)
url = url + version + '/'
response = requests.get(url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
def remove_suffixes(s: str, suffixes):
for suffix in suffixes:
s = s.removesuffix(suffix)
return s
build_artifacts = list(set([
remove_suffixes(node.text, ['.sha512', '.asc'])
for node in soup.find_all('a') if node.get('href').rstrip('/') not in ['KEYS', '..']
]))
build_artifacts = [
a for a in build_artifacts if a not in ['Parent Directory', 'Name', 'Last modified', 'Size', 'Description']
]
nexus_tarball = None
ingester_tarball = None
for artifact in build_artifacts:
if '-nexus-' in artifact:
if any([build['webapp'], build['solr'], build['solr-init']]):
nexus_tarball = os.path.join(dst_dir.name, artifact)
else:
continue
elif '-ingester-' in artifact:
if any([build['cm'], build['gi']]):
ingester_tarball = os.path.join(dst_dir.name, artifact)
else:
continue
for ext in ['', '.sha512', '.asc']:
filename = artifact + ext
dst = os.path.join(dst_dir.name, filename)
print(f'Downloading {url + artifact + ext}')
response = requests.get(url + artifact + ext)
response.raise_for_status()
with open(dst, 'wb') as fp:
fp.write(response.content)
fp.flush()
print(f'Verifying checksum for {artifact}')
m = hashlib.sha512()
m.update(open(os.path.join(dst_dir.name, artifact), 'rb').read())
if m.hexdigest() != open(os.path.join(dst_dir.name, artifact) + '.sha512', 'r').read().split(' ')[0]:
raise ValueError('Bad checksum!')
print(f'Verifying signature for {artifact}')
try:
run_subprocess(
[GPG, '--verify', os.path.join(dst_dir.name, artifact) + '.asc', os.path.join(dst_dir.name, artifact)],
True
)
except:
raise ValueError('Bad signature!')
print('Extracting release source files...')
if any([build['webapp'], build['solr'], build['solr-init']]):
run_subprocess(
[TAR, 'xvf', nexus_tarball, '-C', dst_dir.name],
suppress_output=True
)
shutil.move(
os.path.join(dst_dir.name, 'Apache-SDAP', nexus_tarball.split('/')[-1].removesuffix('.tar.gz')),
os.path.join(dst_dir.name, 'nexus')
)
if any([build['cm'], build['gi']]):
run_subprocess(
[TAR, 'xvf', ingester_tarball, '-C', dst_dir.name],
suppress_output=True
)
shutil.move(
os.path.join(dst_dir.name, 'Apache-SDAP', ingester_tarball.split('/')[-1].removesuffix('.tar.gz')),
os.path.join(dst_dir.name, 'ingester')
)
elif source_location == GHB:
if any([build['webapp'], build['solr'], build['solr-init']]):
if not yes_no_prompt('Will you be using a fork for the Nexus repository? Y/[N]: ', False):
nexus_repo = ASF_NEXUS_REPO
else:
nexus_repo = basic_prompt('Enter Nexus fork URL')
# TODO Maybe fetch list of branches?
nexus_branch = basic_prompt('Enter Nexus branch to build')
print(f'Cloning Nexus repo {nexus_repo} at {nexus_branch}')
run_subprocess(
[GIT, 'clone', '--branch', nexus_branch, nexus_repo],
suppress_output=True,
cwd=dst_dir.name
)
shutil.move(
os.path.join(dst_dir.name, 'sdap-nexus'),
os.path.join(dst_dir.name, 'nexus')
)
if any([build['cm'], build['gi']]):
if not yes_no_prompt('Will you be using a fork for the Ingester repository? Y/[N]: ', False):
ingester_repo = ASF_INGESTER_REPO
else:
ingester_repo = basic_prompt('Enter Ingester fork URL')
# TODO Maybe fetch list of branches?
ingester_branch = basic_prompt('Enter Ingester branch to build')
print(f'Cloning Nexus repo {ingester_repo} at {ingester_branch}')
run_subprocess(
[GIT, 'clone', '--branch', ingester_branch, ingester_repo],
suppress_output=True,
cwd=dst_dir.name
)
shutil.move(
os.path.join(dst_dir.name, 'sdap-ingester'),
os.path.join(dst_dir.name, 'ingester')
)
else:
print('NOTE: Building from local FS should only be done for testing purposes. Please use other sources for '
'official release images (ASF) or anything pushed publicly for production or distribution outside of an '
'official release (GitHub).')
if any([build['webapp'], build['solr'], build['solr-init']]):
path = basic_prompt('Enter path to Nexus repository')
if not os.path.isdir(path):
print(f'{path} either does not exist or is not a directory')
exit(1)
print(f'Copying Nexus {os.path.abspath(path)} -> {os.path.join(dst_dir.name, "nexus")}')
shutil.copytree(
path,
os.path.join(dst_dir.name, 'nexus')
)
if any([build['cm'], build['gi']]):
path = basic_prompt('Enter path to Ingester repository')
if not os.path.isdir(path):
print(f'{path} either does not exist or is not a directory')
exit(1)
print(f'Copying Ingester {os.path.abspath(path)} -> {os.path.join(dst_dir.name, "nexus")}')
shutil.copytree(
path,
os.path.join(dst_dir.name, 'ingester')
)