def __init__()

in fairnr/data/shape_dataset.py [0:0]

• 40 lines of code
• 14 McCabe index (conditional complexity)

    def __init__(self, 
                paths, 
                preload=True,
                repeat=1,
                subsample_valid=-1,
                ids=None):
        
        if os.path.isdir(paths):
            self.paths = [paths]
        else:
            self.paths = [line.strip() for line in open(paths)]

        self.subsample_valid = subsample_valid
        self.total_num_shape = len(self.paths)
        self.cache = None
        self.repeat = repeat

        # -- load per-shape data
        _data_per_shape = {}
        _data_per_shape['shape'] = list(range(len(self.paths)))
        _ixts = self.find_intrinsics()
        _glbs = self.find_global()
        if len(_ixts) > 0:
            _data_per_shape['ixt'] = _ixts
        if len(_glbs) > 0:
            _data_per_shape['glb'] = _glbs

        if self.subsample_valid > -1:
            for key in _data_per_shape:
                _data_per_shape[key] = _data_per_shape[key][::self.subsample_valid]
            self.paths = self.paths[::self.subsample_valid]
            self.total_num_shape = len(self.paths)
        
        # group the data..
        data_list = []
        for r in range(repeat):
            # HACK: making several copies to enable multi-GPU usage.
            if r == 0 and preload:
                self.cache = []
                logger.info('pre-load the dataset into memory.')

            for id in range(self.total_num_shape): 
                element = {}
                for key in _data_per_shape:
                    element[key] = _data_per_shape[key][id]
                data_list.append(element)

                if r == 0 and preload:
                    self.cache += [self._load_batch(data_list, id)]

        # group the data together
        self.data = data_list