def execute()

in assets/mwaa_dags/dags/custom/glue_trigger_crawler_operator.py [0:0]

• 24 lines of code
• 6 McCabe index (conditional complexity)

    def execute(self, context):
        hook = AwsHook(self._aws_conn_id)
        glue_client = hook.get_client_type(client_type="glue", region_name=self._region_name)

        self.log.info("Triggering crawler")
        response = glue_client.start_crawler(Name=self._crawler_name)

        if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
            raise RuntimeError(
                "An error occurred while triggering the crawler: %r" % response
            )

        self.log.info("Waiting for crawler to finish")

        current_retries = 0

        while True:

            crawler = glue_client.get_crawler(Name=self._crawler_name)
            crawler_state = crawler["Crawler"]["State"]

            if crawler_state == "READY":
                self.log.info("Crawler finished running")
                return crawler_state
                break

            if self._max_wait_time and current_retries >= self._max_wait_time:
                raise RuntimeError(
                    "An error occurred while triggering the crawler: %r" % response
                )

            current_retries += 1
            time.sleep(1)