def load_data()

in datasets.py [0:0]

• 21 lines of code
• 4 McCabe index (conditional complexity)

    def load_data(self, file_name):
        """
        Load training data from training file name into datasets class
        :param file_name:
        :return:
        """
        count = 0

        with open(file_name, 'r', encoding='utf-8') as file:
            header = file.readline()
            for line in file:
                count += 1
                try:
                    if count % 100 == 0:
                        logger.info('Processing image id # : %s' % count)

                    image_id, \
                        image_feature, \
                        cleaned_image_questions, \
                        image_url, keyword = self.get_processed_fields(line, file_name, build_image_feature=True)
                    self.train_image_id_imagefeat_dict[image_id] = image_feature
                    self.train_image_id_questions_dict[image_id] = cleaned_image_questions
                    self.train_image_id_keyword_dict[image_id] = keyword

                    if count >= self.max_samples:
                        break
                except:
                    logger.error('Image url has an issue: %s' % image_url)

            logger.info("Training data loaded")