def stage_files()

in ark-demo/pipelines/nfhl/nfhl_gcs_stage.py [0:0]

• 30 lines of code
• 10 McCabe index (conditional complexity)

def stage_files(state_fips, gcs_bucket_name, gcs_path):
    for state, fips in state_fips.items():
        start_date = datetime.now()
        end_date = start_date - timedelta(days=365)
        cur_date = start_date

        # search for most recent NFHL backwards from current date
        while cur_date > end_date:
            nfhl_file = file_template.format(fips, cur_date.strftime('%Y%m%d'))
            nfhl_url = url_template + nfhl_file

            cur_date -= timedelta(days=1)

            logging.info('Attempting {}...'.format(nfhl_url))
            filepath, dl_status = download(nfhl_file, nfhl_url, gcs_bucket_name, gcs_path)

            if dl_status == 404:
                continue
            elif dl_status == 'exists':
                if filepath is None:
                    logging.info('GCS path exists, next state...')
                    break
                else:
                    logging.info('File exists on disk, skipping to upload...')
            elif filepath is None:
                logging.info(dl_status)
                continue
            else:
                logging.info('Download complete {}'.format(nfhl_file))

            gcs_path, ul_status = upload(filepath, gcs_bucket_name, gcs_path)

            if ul_status != 'success':
                logging.warning('Upload skipped: {} {}'.format(ul_status, gcs_path))
            if gcs_path is None:
                logging.error(ul_status)

            break