loader_mnist/data_reader.py [284:326]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    self.prefetch_queue = queue.Queue(maxsize=self.prefetch_num)
    self.prefetch_thread = threading.Thread(target=_run_prefetch,
      args=(self.prefetch_queue, self.batch_loader, self.imdb,
         self.shuffle, self.one_pass, self.params))
    self.prefetch_thread.daemon = True
    self.prefetch_thread.start()

  def batches(self):
    while True:
      # Get a batch from the prefetching queue
      if self.prefetch_queue.empty(): pass
        #print('data reader: waiting for data loading (IO is slow)...')
      batch = self.prefetch_queue.get(block=True)
      if batch is None:
        assert(self.one_pass)
        print('data reader: one pass finished')
        raise StopIteration()
      yield batch

def _run_prefetch(prefetch_queue, batch_loader, imdb, shuffle,
                  one_pass, params):
  num_samples = len(imdb['data'])
  batch_size = params['batch_size']

  n_sample = 0
  fetch_order = np.arange(num_samples)
  while True:
    # Shuffle the sample order for every epoch
    if n_sample == 0 and shuffle:
      fetch_order = np.random.permutation(num_samples)

    # Load batch from file
    # note that len(sample_ids) <= batch_size, not necessarily equal
    sample_ids = fetch_order[n_sample:n_sample+batch_size]
    batch = batch_loader.load_one_batch(sample_ids)
    prefetch_queue.put(batch, block=True)

    n_sample += len(sample_ids)
    if n_sample >= num_samples:
      # Put in a None batch to indicate a whole pass is over
      if one_pass:
        prefetch_queue.put(None, block=True)
      n_sample = 0
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



loader_vd/data_reader.py [466:508]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    self.prefetch_queue = queue.Queue(maxsize=self.prefetch_num)
    self.prefetch_thread = threading.Thread(target=_run_prefetch,
      args=(self.prefetch_queue, self.batch_loader, self.imdb,
         self.shuffle, self.one_pass, self.params))
    self.prefetch_thread.daemon = True
    self.prefetch_thread.start()

  def batches(self):
    while True:
      # Get a batch from the prefetching queue
      if self.prefetch_queue.empty(): pass
        #print('data reader: waiting for data loading (IO is slow)...')
      batch = self.prefetch_queue.get(block=True)
      if batch is None:
        assert(self.one_pass)
        print('data reader: one pass finished')
        raise StopIteration()
      yield batch

def _run_prefetch(prefetch_queue, batch_loader, imdb, shuffle,
                  one_pass, params):
  num_samples = len(imdb['data'])
  batch_size = params['batch_size']

  n_sample = 0
  fetch_order = np.arange(num_samples)
  while True:
    # Shuffle the sample order for every epoch
    if n_sample == 0 and shuffle:
      fetch_order = np.random.permutation(num_samples)

    # Load batch from file
    # note that len(sample_ids) <= batch_size, not necessarily equal
    sample_ids = fetch_order[n_sample:n_sample+batch_size]
    batch = batch_loader.load_one_batch(sample_ids)
    prefetch_queue.put(batch, block=True)

    n_sample += len(sample_ids)
    if n_sample >= num_samples:
      # Put in a None batch to indicate a whole pass is over
      if one_pass:
        prefetch_queue.put(None, block=True)
      n_sample = 0
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -