def scan()

in src/plugins/scanners/travis.py [0:0]

• 111 lines of code
• 12 McCabe index (conditional complexity)

def scan(KibbleBit, source):
    # Simple URL check
    travis = re.match(r"https?://travis-ci\.(org|com)", source['sourceURL'])
    if travis:
        # Is this travs-ci.org or travis-ci.com - we need to know!
        TLD = travis.group(1)
        source['steps']['travis'] = {
            'time': time.time(),
            'status': 'Parsing Travis job changes...',
            'running': True,
            'good': True
        }
        KibbleBit.updateSource(source)
        
        badOnes = 0
        pendingJobs = []
        KibbleBit.pprint("Parsing Travis activity at %s" % source['sourceURL'])
        source['steps']['travis'] = {
            'time': time.time(),
            'status': 'Downloading changeset',
            'running': True,
            'good': True
        }
        KibbleBit.updateSource(source)
        
        # Travis needs a token
        token = None
        if source['creds'] and 'token' in source['creds'] and source['creds']['token'] and len(source['creds']['token']) > 0:
            token = source['creds']['token']
        else:
            KibbleBit.pprint("Travis CI requires a token to work!")
            return False
            
        # Get the job list, paginated
        sURL = source['sourceURL']
        
        # Used for pagination
        jobs = 100
        offset = 0
        
        # Counters; builds queued, running and total jobs
        queued = 0 # We don't know how to count this yet
        building = 0
        total = 0
        blocked = 0 # Dunno how to count yet
        stuck = 0 # Ditto
        avgqueuetime = 0 # Ditto, fake it
        
        maybeQueued = []
        while jobs == 100:
            URL = "https://api.travis-ci.%s/repos?repository.active=true&sort_by=current_build:desc&offset=%u&limit=100&include=repository.last_started_build" % (TLD, offset)
            offset += 100
            r = requests.get(URL, headers = {'Travis-API-Version': '3', 'Authorization': "token %s" % token})
            
            if r.status_code != 200:
                KibbleBit.pprint("Travis did not return a 200 Okay, bad token?!")
                
                source['steps']['travis'] = {
                    'time': time.time(),
                    'status': 'Travis CI scan failed at ' + time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(time.time()) + ". Bad token??!"),
                    'running': False,
                    'good': False
                }
                KibbleBit.updateSource(source)
                return
            
            
            # For each build job
            js = r.json()
            for repo in js['repositories']:
                total += 1
                cb = repo.get('last_started_build')
                if cb:
                    # Is the build currently running?
                    if cb['state'] in ['started','created', 'queued', 'pending']:
                        for job in cb.get('jobs', []):
                            maybeQueued.append(job['id'])
                        
                
                # Queue up build jobs for the threaded scanner
                bid = repo['id']
                pendingJobs.append(bid)
            
            jobs = len(js['repositories'])
            KibbleBit.pprint("Scanned %u jobs..." % total)
            
        # Find out how many building and pending jobs
        for jobID in maybeQueued:
            URL = "https://api.travis-ci.%s/job/%u" % (TLD, jobID)
            r = requests.get(URL, headers = {'Travis-API-Version': '3', 'Authorization': "token %s" % token})
            if r.status_code == 200:
                jobjs = r.json()
                if jobjs['state'] == 'started':
                    building += 1
                    KibbleBit.pprint("Job %u is building" % jobID)
                elif jobjs['state'] in ['created', 'queued', 'pending']:
                    queued += 1
                    blocked += 1 # Queued in Travis generally means a job can't find an executor, and thus is blocked.
                    KibbleBit.pprint("Job %u is pending" % jobID)
        KibbleBit.pprint("%u building, %u queued..." % (building, queued))
        
        # Save queue snapshot
        NOW = int(datetime.datetime.utcnow().timestamp())
        queuehash = hashlib.sha224( ("%s-%s-queue-%s" % (source['organisation'], source['sourceURL'], int(time.time())) ).encode('ascii', errors='replace')).hexdigest()
        
        # Write up a queue doc
        queuedoc = {
            'id': queuehash,
            'date': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(NOW)),
            'time': NOW,
            'building': building,
            'size': queued,
            'blocked': blocked,
            'stuck': stuck,
            'avgwait': avgqueuetime,
            'ci': 'travis',
            
            # Standard docs values
            'sourceID': source['sourceID'],
            'organisation': source['organisation'],
            'upsert': True,
        }
        KibbleBit.append('ci_queue', queuedoc)
        
        
        KibbleBit.pprint("Found %u jobs in Travis" % len(pendingJobs))
        
        threads = []
        block = threading.Lock()
        KibbleBit.pprint("Scanning jobs using 4 sub-threads")
        for i in range(0,4):
            t = travisThread(block, KibbleBit, source, token, pendingJobs, TLD)
            threads.append(t)
            t.start()
        
        for t in threads:
            t.join()

        # We're all done, yaay        
        KibbleBit.pprint("Done scanning %s" % source['sourceURL'])

        source['steps']['travis'] = {
            'time': time.time(),
            'status': 'Travis successfully scanned at ' + time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(time.time())),
            'running': False,
            'good': True
        }
        KibbleBit.updateSource(source)