def load_test_data()

in datasets.py [0:0]

• 29 lines of code
• 7 McCabe index (conditional complexity)

    def load_test_data(self):
        """
        Function loads test data from test file name into datasets class
        :return:
        """
        for id, train_questions in self.train_image_id_questions_dict.items():
            for question in train_questions:
                question = question.split()[1:-1]
                question = ' '.join(question)
                if question in ['None', 'none']:
                    continue
                self.unique_train_questions.add(question)
        logger.info('Done creating unique training question set. Size: %s' % str(len(self.unique_train_questions)))

        with open(self.test_file, 'r', encoding='utf-8') as file:
            count = 0
            header = file.readline()
            for line in file:
                count += 1
                if count % 100 == 0:
                    logger.info('Processing image id # : %s' % count)

                image_id, \
                    image_feature, \
                    cleaned_image_questions, \
                    image_url, keyword = self.get_processed_fields(line,
                                                                   self.test_file,
                                                                   build_image_feature=True)
                self.test_image_id_questions_dict[image_id] = cleaned_image_questions
                self.test_image_id_url_dict[image_id] = image_url
                self.test_image_id_imagefeat_dict[image_id] = image_feature
                self.test_image_id_keyword_dict[image_id] = keyword

                if count >= self.max_samples:
                    break
        logger.info("Test data loaded")