gcpdiag/lint/dataflow/err_2023_009_splunk_err_invalid_cert.py [33:86]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
]
LOG_FILTER = [
    'severity=ERROR',
    'jsonPayload.message: ("{}")'.format('" AND "'.join(MATCH_STRINGS)),
]

project_logs = {}
MAX_JOBS_TO_DISPLAY = 10


def prepare_rule(context: models.Context):
  project_id = context.project_id
  log_name = 'log_id("dataflow.googleapis.com/worker")'
  # f'projects/{project_id}/logs/dataflow.googleapis.com%2Fworker'
  project_logs[project_id] = logs.query(
      project_id=project_id,
      resource_type='dataflow_step',
      log_name=log_name,
      filter_str=' AND '.join(LOG_FILTER))  # "returns LogsQuery object"


def run_rule(context: models.Context, report: lint.LintReportRuleInterface):
  """Check that no (Splunk) job has invalid certification path error."""
  project = crm.get_project(context.project_id)

  # skip entire rule if logging is disabled
  if not apis.is_enabled(context.project_id, 'logging'):
    report.add_skipped(project, 'logging api is disabled')
    return

  # skip entire rule if dataflow API is disabled
  if not apis.is_enabled(context.project_id, 'dataflow'):
    report.add_skipped(project, 'dataflow api is disabled')
    return

  if (context.project_id in project_logs and
      project_logs[context.project_id].entries):
    failed_jobs = set()
    for log_entry in project_logs[context.project_id].entries:
      current_entry = get_path(log_entry, ('jsonPayload', 'message'), '')

      if log_entry['severity'] != 'ERROR' or all(
          m not in current_entry for m in MATCH_STRINGS):
        continue

      job_id = get_path(log_entry, ('resource', 'labels', 'job_id'))
      failed_jobs.add(job_id)

    if failed_jobs:
      extra_jobs = (f', and {len(failed_jobs) - MAX_JOBS_TO_DISPLAY} more jobs'
                    if len(failed_jobs) > MAX_JOBS_TO_DISPLAY else '')

      report.add_failed(
          project,
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



gcpdiag/lint/dataflow/err_2023_012_dataflow_spanner_oom.py [35:88]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
]
LOG_FILTER = [
    'severity=ERROR',
    'jsonPayload.message: ("{}")'.format('" AND "'.join(MATCH_STRINGS)),
]

project_logs = {}
MAX_JOBS_TO_DISPLAY = 10


def prepare_rule(context: models.Context):
  project_id = context.project_id
  log_name = 'log_id("dataflow.googleapis.com/worker")'
  # f'projects/{project_id}/logs/dataflow.googleapis.com%2Fworker'
  project_logs[project_id] = logs.query(
      project_id=project_id,
      resource_type='dataflow_step',
      log_name=log_name,
      filter_str=' AND '.join(LOG_FILTER))  # "returns LogsQuery object"


def run_rule(context: models.Context, report: lint.LintReportRuleInterface):
  """Check that no (dataflow) job writing to spanner failed due to OOM."""
  project = crm.get_project(context.project_id)

  # skip entire rule if logging is disabled
  if not apis.is_enabled(context.project_id, 'logging'):
    report.add_skipped(project, 'logging api is disabled')
    return

  # skip entire rule if dataflow API is disabled
  if not apis.is_enabled(context.project_id, 'dataflow'):
    report.add_skipped(project, 'dataflow api is disabled')
    return

  if (context.project_id in project_logs and
      project_logs[context.project_id].entries):
    failed_jobs = set()
    for log_entry in project_logs[context.project_id].entries:
      current_entry = get_path(log_entry, ('jsonPayload', 'message'), '')

      if log_entry['severity'] != 'ERROR' or all(
          m not in current_entry for m in MATCH_STRINGS):
        continue

      job_id = get_path(log_entry, ('resource', 'labels', 'job_id'))
      failed_jobs.add(job_id)

    if failed_jobs:
      extra_jobs = (f', and {len(failed_jobs) - MAX_JOBS_TO_DISPLAY} more jobs'
                    if len(failed_jobs) > MAX_JOBS_TO_DISPLAY else '')

      report.add_failed(
          project,
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -