def write_failure_lines()

in treeherder/log_parser/failureline.py [0:0]

• 33 lines of code
• 11 McCabe index (conditional complexity)

def write_failure_lines(job_log, log_iter):
    failure_lines = []
    failure_lines_cutoff = settings.FAILURE_LINES_CUTOFF
    log_list = list(islice(log_iter, failure_lines_cutoff + 1))

    if len(log_list) > failure_lines_cutoff:
        # Alter the N+1th log line to indicate the list was truncated.
        log_list[-1].update(action="truncated")

    transformer = None
    with transaction.atomic():
        try:
            failure_lines = create(job_log, log_list)
        except DataError as e:
            logger.warning(f"Got DataError inserting failure_line: {e.args}")
        except OperationalError as e:
            logger.warning("Got OperationalError inserting failure_line")
            # Retry iff this error is the "incorrect String Value" error
            if e.args[0] == 1366:
                # Sometimes get an error if we can't save a string as MySQL pseudo-UTF8
                transformer = replace_astral
            else:
                raise
        except IntegrityError:
            logger.warning("Got IntegrityError inserting failure_line")

            def exclude_lines(log_list):
                exclude = set(
                    FailureLine.objects.filter(
                        job_log=job_log, line__in=[item["line"] for item in log_list]
                    ).values_list("line", flat=True)
                )
                return (item for item in log_list if item["line"] not in exclude)

            transformer = exclude_lines

    # If we hit an error that might be solved by transofrming the data then retry
    if transformer is not None:
        with transaction.atomic():
            log_list = list(transformer(log_list))
            failure_lines = create(job_log, log_list)

    return failure_lines