def _construct_loader()

in datasets/GDTPretrainDataset.py [0:0]

• 53 lines of code
• 9 McCabe index (conditional complexity)

    def _construct_loader(self):
        """
        Construct the video loader.
        """
        # Get list of paths
        os.makedirs(self.path_to_data_dir, exist_ok=True)
        path_to_file = os.path.join(
            self.path_to_data_dir, f"{self.ds_name}_{self.mode}.txt"
        )
        if not os.path.exists(path_to_file) and self.ds_name != 'audioset':
            files = list(sorted(glob.glob(os.path.join(self.data_prefix, '*', '*')))) 
            with open(path_to_file, 'w') as f:
                for item in files:
                    f.write("%s\n" % item)

        self._path_to_videos = []
        self._labels = []
        self._spatial_temporal_idx = []
        self._vid_indices = []
        with open(path_to_file, "r") as f:
            for clip_idx, path in enumerate(f.read().splitlines()):
                for idx in range(self._num_clips):
                    self._path_to_videos.append(
                        os.path.join(self.data_prefix, path)
                    )
                    if self.ds_name != 'audioset':
                        class_name = path.split('/')[-2]
                        label = self.class_to_idx[class_name]
                    self._labels.append(int(label))
                    self._spatial_temporal_idx.append(idx)
                    self._vid_indices.append(clip_idx)
                    self._video_meta[clip_idx * self._num_clips + idx] = {}
        assert (
            len(self._path_to_videos) > 0
        ), "Failed to load {} split {} from {}".format(
            self.ds_name, self._split_idx, path_to_file
        )
        print(
            "Constructing {} dataloader (size: {}) from {}".format(
                self.ds_name, len(self._path_to_videos), path_to_file
            )
        )

        # Create / Load valid indices (has audio)
        vid_valid_file = f'{self.path_to_data_dir}/{self.ds_name}_valid.pkl'
        if os.path.exists(vid_valid_file):
            with open(vid_valid_file, 'rb') as handle:
                self.valid_indices = pickle.load(handle)
        else:
            self.valid_indices = filter_videos(self._path_to_videos)
            with open(vid_valid_file, 'wb') as handle:
                pickle.dump(
                    self.valid_indices, 
                    handle, 
                    protocol=pickle.HIGHEST_PROTOCOL
                )
        if self.num_data_samples is not None:
            self.valid_indices = self.valid_indices[:self.num_data_samples]
        print(f"Total number of videos: {len(self._path_to_videos)}, Valid videos: {len(self.valid_indices)}", flush=True)

        # Make lists a Manager objects
        #self._path_to_videos = self.manager.list(self._path_to_videos)
        self.valid_indices = list(self.valid_indices)