def generate_random_folders()

in tcav/tcav_examples/image_models/imagenet/imagenet_and_broden_fetcher.py [0:0]

• 24 lines of code
• 7 McCabe index (conditional complexity)

def generate_random_folders(working_directory, random_folder_prefix,
                            number_of_random_folders,
                            number_of_examples_per_folder, imagenet_dataframe):
  imagenet_concepts = imagenet_dataframe["class_name"].values.tolist()
  for partition_number in range(number_of_random_folders):
    partition_name = random_folder_prefix + "_" + str(partition_number)
    partition_folder_path = os.path.join(working_directory, partition_name)
    tf.io.gfile.makedirs(partition_folder_path)

    # Select a random concept
    examples_selected = 0
    while examples_selected < number_of_examples_per_folder:
      random_concept = random.choice(imagenet_concepts)
      urls = fetch_all_urls_for_concept(imagenet_dataframe, random_concept)
      for url in urls:
        # We are filtering out images from Flickr urls, since several of those were removed
        if "flickr" not in url:
          try:
            download_image(partition_folder_path, url)
            examples_selected += 1
            if (examples_selected) % 10 == 0:
              tf.compat.v1.logging.info("Downloaded " + str(examples_selected) + "/" +
                              str(number_of_examples_per_folder) + " for " +
                              partition_name)
            break  # Break if we successfully downloaded an image
          except:
              pass # try new url