def getStreamingSource()

in awsglue/context.py [0:0]

• 17 lines of code
• 8 McCabe index (conditional complexity)

    def getStreamingSource(self, connection_type, format = None, transformation_ctx = "", push_down_predicate= "", **options):
        """Creates a Streaming Data Source object.

        This can be used to read Dataframes from external sources.
        """
        options["callSite"] = callsite()
        if(format and format.lower() in self.Spark_SQL_Formats):
            connection_type = format

        j_source = self._ssql_ctx.getSource(connection_type,
                                            makeOptions(self._sc, options), transformation_ctx, push_down_predicate)

        prefix = None
        if 'paths' in options and options['paths'] != None:
            paths = options['paths']
            prefix = os.path.commonprefix(paths)
            if prefix != None:
                prefix = prefix.split(':')[-1]
                prefix = re.sub('[:/.]', '', prefix)

        # in case paths is not in options or no common prefix
        if prefix == None:
            prefix = str(uuid.uuid1())
            prefix = re.sub('[-]', '_', prefix)

        return StreamingDataSource(j_source, self, prefix)