def _update_datasets()

in data-access/nexustiles/nexustiles.py [0:0]
48 lines of code
12 McCabe index (conditional complexity)

    def _update_datasets():
        update_logger = logging.getLogger("nexus-tile-svc.backends")
        solrcon = NexusTileService._get_datasets_store()

        update_logger.info('Executing Solr query to check for new datasets')

        present_datasets = {None, '__nexusproto__'}
        next_cursor_mark = '*'

        added_datasets = 0

        while True:
            response = solrcon.search('*:*', cursorMark=next_cursor_mark, sort='id asc')

            try:
                response_cursor_mark = response.nextCursorMark
            except AttributeError:
                break

            if response_cursor_mark == next_cursor_mark:
                break
            else:
                next_cursor_mark = response_cursor_mark

            for dataset in response.docs:
                d_id = dataset['dataset_s']
                store_type = dataset.get('store_type_s', 'nexusproto')

                present_datasets.add(d_id)

                if d_id in NexusTileService.backends:
                    continue

                added_datasets += 1

                if store_type == 'nexus_proto' or store_type == 'nexusproto':
                    update_logger.info(f"Detected new nexusproto dataset {d_id}, using default nexusproto backend")
                    NexusTileService.backends[d_id] = NexusTileService.backends[None]
                elif store_type == 'zarr':
                    update_logger.info(f"Detected new zarr dataset {d_id}, opening new zarr backend")

                    ds_config = json.loads(dataset['config'][0])
                    try:
                        NexusTileService.backends[d_id] = {
                            'backend': ZarrBackend(dataset_name=dataset['dataset_s'], **ds_config),
                            'up': True
                        }
                    except NexusTileServiceException:
                        added_datasets -= 1
                else:
                    update_logger.warning(f'Unsupported backend {store_type} for dataset {d_id}')
                    added_datasets -= 1

        removed_datasets = set(NexusTileService.backends.keys()).difference(present_datasets)

        if len(removed_datasets) > 0:
            update_logger.info(f'{len(removed_datasets)} old datasets marked for removal')

        for dataset in removed_datasets:
            update_logger.info(f"Removing dataset {dataset}")
            del NexusTileService.backends[dataset]

        update_logger.info(f'Finished dataset update: {added_datasets} added, {len(removed_datasets)} removed, '
                           f'{len(NexusTileService.backends) - 2} total')