def __init__()

in data.py [0:0]

• 52 lines of code
• 12 McCabe index (conditional complexity)

    def __init__(self, split, batchsize, idx, num_workers, rescale=1):
        IMAGENET_NUM_TRAIN_IMAGES = 1281167
        IMAGENET_NUM_VAL_IMAGES = 50000

        self.rescale = rescale

        if split == "train":
            im_length = IMAGENET_NUM_TRAIN_IMAGES
            records_to_skip = im_length * idx // num_workers
            records_to_read = im_length * (idx + 1) // num_workers - records_to_skip
        else:
            im_length = IMAGENET_NUM_VAL_IMAGES

        self.curr_sample = 0

        index_path = osp.join(FLAGS.imagenet_datadir, 'index.json')
        with open(index_path) as f:
            metadata = json.load(f)
            counts = metadata['record_counts']

        if split == 'train':
            file_names = list(sorted([x for x in counts.keys() if x.startswith('train')]))

            result_records_to_skip = None
            files = []
            for filename in file_names:
                records_in_file = counts[filename]
                if records_to_skip >= records_in_file:
                    records_to_skip -= records_in_file
                    continue
                elif records_to_read > 0:
                    if result_records_to_skip is None:
                        # Record the number to skip in the first file
                        result_records_to_skip = records_to_skip
                    files.append(filename)
                    records_to_read -= (records_in_file - records_to_skip)
                    records_to_skip = 0
                else:
                    break
        else:
            files = list(sorted([x for x in counts.keys() if x.startswith('validation')]))

        files = [osp.join(FLAGS.imagenet_datadir, x) for x in files]
        preprocess_function = ImagenetPreprocessor(128, dtype=tf.float32, train=False).parse_and_preprocess

        ds = tf.data.TFRecordDataset.from_generator(lambda: files, output_types=tf.string)
        ds = ds.apply(tf.data.TFRecordDataset)
        ds = ds.take(im_length)
        ds = ds.prefetch(buffer_size=FLAGS.batch_size)
        ds = ds.apply(tf.contrib.data.shuffle_and_repeat(buffer_size=10000))
        ds = ds.apply(batching.map_and_batch(map_func=preprocess_function, batch_size=FLAGS.batch_size, num_parallel_batches=4))
        ds = ds.prefetch(buffer_size=2)

        ds_iterator = ds.make_initializable_iterator()
        labels, images = ds_iterator.get_next()
        self.images = tf.clip_by_value(images / 256 + tf.random_uniform(tf.shape(images), 0, 1. / 256), 0.0, 1.0)
        self.labels = labels

        config = tf.ConfigProto(device_count = {'GPU': 0})
        sess = tf.Session(config=config)
        sess.run(ds_iterator.initializer)

        self.im_length = im_length // batchsize

        self.sess = sess