in lib/ramble/spack/ci.py [0:0]
def reproduce_ci_job(url, work_dir):
""" Given a url to gitlab artifacts.zip from a failed 'spack ci rebuild' job,
attempt to setup an environment in which the failure can be reproduced
locally. This entails the following:
First download and extract artifacts. Then look through those artifacts
to glean some information needed for the reproduer (e.g. one of the
artifacts contains information about the version of spack tested by
gitlab, another is the generated pipeline yaml containing details
of the job like the docker image used to run it). The output of this
function is a set of printed instructions for running docker and then
commands to run to reproduce the build once inside the container.
"""
download_and_extract_artifacts(url, work_dir)
lock_file = fs.find(work_dir, 'spack.lock')[0]
concrete_env_dir = os.path.dirname(lock_file)
tty.debug('Concrete environment directory: {0}'.format(
concrete_env_dir))
yaml_files = fs.find(work_dir, ['*.yaml', '*.yml'])
tty.debug('yaml files:')
for yaml_file in yaml_files:
tty.debug(' {0}'.format(yaml_file))
pipeline_yaml = None
# Try to find the dynamically generated pipeline yaml file in the
# reproducer. If the user did not put it in the artifacts root,
# but rather somewhere else and exported it as an artifact from
# that location, we won't be able to find it.
for yf in yaml_files:
with open(yf) as y_fd:
yaml_obj = syaml.load(y_fd)
if 'variables' in yaml_obj and 'stages' in yaml_obj:
pipeline_yaml = yaml_obj
if pipeline_yaml:
tty.debug('\n{0} is likely your pipeline file'.format(yf))
# Find the install script in the unzipped artifacts and make it executable
install_script = fs.find(work_dir, 'install.sh')[0]
st = os.stat(install_script)
os.chmod(install_script, st.st_mode | stat.S_IEXEC)
# Find the repro details file. This just includes some values we wrote
# during `spack ci rebuild` to make reproduction easier. E.g. the job
# name is written here so we can easily find the configuration of the
# job from the generated pipeline file.
repro_file = fs.find(work_dir, 'repro.json')[0]
repro_details = None
with open(repro_file) as fd:
repro_details = json.load(fd)
repro_dir = os.path.dirname(repro_file)
rel_repro_dir = repro_dir.replace(work_dir, '').lstrip(os.path.sep)
# Find the spack info text file that should contain the git log
# of the HEAD commit used during the CI build
spack_info_file = fs.find(work_dir, 'spack_info.txt')[0]
with open(spack_info_file) as fd:
spack_info = fd.read()
# Access the specific job configuration
job_name = repro_details['job_name']
job_yaml = None
if job_name in pipeline_yaml:
job_yaml = pipeline_yaml[job_name]
if job_yaml:
tty.debug('Found job:')
tty.debug(job_yaml)
job_image = None
setup_result = False
if 'image' in job_yaml:
job_image_elt = job_yaml['image']
if 'name' in job_image_elt:
job_image = job_image_elt['name']
else:
job_image = job_image_elt
tty.msg('Job ran with the following image: {0}'.format(job_image))
# Because we found this job was run with a docker image, so we will try
# to print a "docker run" command that bind-mounts the directory where
# we extracted the artifacts.
# Destination of bind-mounted reproduction directory. It makes for a
# more faithful reproducer if everything appears to run in the same
# absolute path used during the CI build.
mount_as_dir = '/work'
if repro_details:
mount_as_dir = repro_details['ci_project_dir']
mounted_repro_dir = os.path.join(mount_as_dir, rel_repro_dir)
# We will also try to clone spack from your local checkout and
# reproduce the state present during the CI build, and put that into
# the bind-mounted reproducer directory.
# Regular expressions for parsing that HEAD commit. If the pipeline
# was on the gitlab spack mirror, it will have been a merge commit made by
# gitub and pushed by the sync script. If the pipeline was run on some
# environment repo, then the tested spack commit will likely have been
# a regular commit.
commit_1 = None
commit_2 = None
commit_regex = re.compile(r"commit\s+([^\s]+)")
merge_commit_regex = re.compile(r"Merge\s+([^\s]+)\s+into\s+([^\s]+)")
# Try the more specific merge commit regex first
m = merge_commit_regex.search(spack_info)
if m:
# This was a merge commit and we captured the parents
commit_1 = m.group(1)
commit_2 = m.group(2)
else:
# Not a merge commit, just get the commit sha
m = commit_regex.search(spack_info)
if m:
commit_1 = m.group(1)
setup_result = False
if commit_1:
if commit_2:
setup_result = setup_spack_repro_version(
work_dir, commit_2, merge_commit=commit_1)
else:
setup_result = setup_spack_repro_version(work_dir, commit_1)
if not setup_result:
setup_msg = """
This can happen if the spack you are using to run this command is not a git
repo, or if it is a git repo, but it does not have the commits needed to
recreate the tested merge commit. If you are trying to reproduce a spack
PR pipeline job failure, try fetching the latest develop commits from
mainline spack and make sure you have the most recent commit of the PR
branch in your local spack repo. Then run this command again.
Alternatively, you can also manually clone spack if you know the version
you want to test.
"""
tty.error('Failed to automatically setup the tested version of spack '
'in your local reproduction directory.')
print(setup_msg)
# In cases where CI build was run on a shell runner, it might be useful
# to see what tags were applied to the job so the user knows what shell
# runner was used. But in that case in general, we cannot do nearly as
# much to set up the reproducer.
job_tags = None
if 'tags' in job_yaml:
job_tags = job_yaml['tags']
tty.msg('Job ran with the following tags: {0}'.format(job_tags))
inst_list = []
# Finally, print out some instructions to reproduce the build
if job_image:
inst_list.append('\nRun the following command:\n\n')
inst_list.append(' $ docker run --rm -v {0}:{1} -ti {2}\n'.format(
work_dir, mount_as_dir, job_image))
inst_list.append('\nOnce inside the container:\n\n')
else:
inst_list.append('\nOnce on the tagged runner:\n\n')
if not setup_result:
inst_list.append(' - Clone spack and acquire tested commit\n')
inst_list.append('{0}'.format(spack_info))
spack_root = '<spack-clone-path>'
else:
spack_root = '{0}/spack'.format(mount_as_dir)
inst_list.append(' - Activate the environment\n\n')
inst_list.append(' $ source {0}/share/spack/setup-env.sh\n'.format(
spack_root))
inst_list.append(
' $ spack env activate --without-view {0}\n\n'.format(
mounted_repro_dir if job_image else repro_dir))
inst_list.append(' - Run the install script\n\n')
inst_list.append(' $ {0}\n'.format(
os.path.join(mounted_repro_dir, 'install.sh')
if job_image else install_script))
print(''.join(inst_list))