example_zoo/tensorflow/probability/latent_dirichlet_allocation_distributions/trainer/latent_dirichlet_allocation_distributions.py [359:463]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "kl": tf.compat.v1.metrics.mean(kl), "perplexity": (perplexity_tensor, log_perplexity_update), "topics": (topics, tf.no_op()), }, ) def get_topics_strings(topics_words, alpha, vocabulary, topics_to_print=10, words_per_topic=10): """Returns the summary of the learned topics. Arguments: topics_words: KxV tensor with topics as rows and words as columns. alpha: 1xK tensor of prior Dirichlet concentrations for the topics. vocabulary: A mapping of word's integer index to the corresponding string. topics_to_print: The number of topics with highest prior weight to summarize. words_per_topic: Number of wodrs per topic to return. Returns: summary: A np.array with strings. """ alpha = np.squeeze(alpha, axis=0) # Use a stable sorting algorithm so that when alpha is fixed # we always get the same topics. highest_weight_topics = np.argsort(-alpha, kind="mergesort") top_words = np.argsort(-topics_words, axis=1) res = [] for topic_idx in highest_weight_topics[:topics_to_print]: l = ["index={} alpha={:.2f}".format(topic_idx, alpha[topic_idx])] l += [vocabulary[word] for word in top_words[topic_idx, :words_per_topic]] res.append(" ".join(l)) return np.array(res) ROOT_PATH = "https://github.com/akashgit/autoencoding_vi_for_topic_models/raw/9db556361409ecb3a732f99b4ef207aeb8516f83/data/20news_clean" FILE_TEMPLATE = "{split}.txt.npy" def download(directory, filename): """Download a file.""" filepath = os.path.join(directory, filename) if tf.io.gfile.exists(filepath): return filepath if not tf.io.gfile.exists(directory): tf.io.gfile.makedirs(directory) url = os.path.join(ROOT_PATH, filename) print("Downloading %s to %s" % (url, filepath)) urllib.request.urlretrieve(url, filepath) return filepath def newsgroups_dataset(directory, split_name, num_words, shuffle_and_repeat): """Return 20 newsgroups tf.data.Dataset.""" data = np.load(download(directory, FILE_TEMPLATE.format(split=split_name))) # The last row is empty in both train and test. data = data[:-1] # Each row is a list of word ids in the document. We first convert this to # sparse COO matrix (which automatically sums the repeating words). Then, # we convert this COO matrix to CSR format which allows for fast querying of # documents. num_documents = data.shape[0] indices = np.array([(row_idx, column_idx) for row_idx, row in enumerate(data) for column_idx in row]) sparse_matrix = scipy.sparse.coo_matrix( (np.ones(indices.shape[0]), (indices[:, 0], indices[:, 1])), shape=(num_documents, num_words), dtype=np.float32) sparse_matrix = sparse_matrix.tocsr() dataset = tf.data.Dataset.range(num_documents) # For training, we shuffle each epoch and repeat the epochs. if shuffle_and_repeat: dataset = dataset.shuffle(num_documents).repeat() # Returns a single document as a dense TensorFlow tensor. The dataset is # stored as a sparse matrix outside of the graph. def get_row_py_func(idx): def get_row_python(idx_py): return np.squeeze(np.array(sparse_matrix[idx_py].todense()), axis=0) py_func = tf.compat.v1.py_func( get_row_python, [idx], tf.float32, stateful=False) py_func.set_shape((num_words,)) return py_func dataset = dataset.map(get_row_py_func) return dataset def build_fake_input_fns(batch_size): """Build fake data for unit testing.""" num_words = 1000 vocabulary = [str(i) for i in range(num_words)] random_sample = np.random.randint( 10, size=(batch_size, num_words)).astype(np.float32) def train_input_fn(): dataset = tf.data.Dataset.from_tensor_slices(random_sample) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - example_zoo/tensorflow/probability/latent_dirichlet_allocation_edward2/trainer/latent_dirichlet_allocation_edward2.py [365:470]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "kl": tf.compat.v1.metrics.mean(kl), "perplexity": (perplexity_tensor, log_perplexity_update), "topics": (topics, tf.no_op()), }, ) def get_topics_strings(topics_words, alpha, vocabulary, topics_to_print=10, words_per_topic=10): """Returns the summary of the learned topics. Arguments: topics_words: KxV tensor with topics as rows and words as columns. alpha: 1xK tensor of prior Dirichlet concentrations for the topics. vocabulary: A mapping of word's integer index to the corresponding string. topics_to_print: The number of topics with highest prior weight to summarize. words_per_topic: Number of wodrs per topic to return. Returns: summary: A np.array with strings. """ alpha = np.squeeze(alpha, axis=0) # Use a stable sorting algorithm so that when alpha is fixed # we always get the same topics. highest_weight_topics = np.argsort(-alpha, kind="mergesort") top_words = np.argsort(-topics_words, axis=1) res = [] for topic_idx in highest_weight_topics[:topics_to_print]: l = ["index={} alpha={:.2f}".format(topic_idx, alpha[topic_idx])] l += [vocabulary[word] for word in top_words[topic_idx, :words_per_topic]] res.append(" ".join(l)) return np.array(res) ROOT_PATH = "https://github.com/akashgit/autoencoding_vi_for_topic_models/raw/9db556361409ecb3a732f99b4ef207aeb8516f83/data/20news_clean" FILE_TEMPLATE = "{split}.txt.npy" def download(directory, filename): """Download a file.""" filepath = os.path.join(directory, filename) if tf.io.gfile.exists(filepath): return filepath if not tf.io.gfile.exists(directory): tf.io.gfile.makedirs(directory) url = os.path.join(ROOT_PATH, filename) print("Downloading %s to %s" % (url, filepath)) urllib.request.urlretrieve(url, filepath) return filepath def newsgroups_dataset(directory, split_name, num_words, shuffle_and_repeat): """20 newsgroups as a tf.data.Dataset.""" data = np.load(download(directory, FILE_TEMPLATE.format(split=split_name))) # The last row is empty in both train and test. data = data[:-1] # Each row is a list of word ids in the document. We first convert this to # sparse COO matrix (which automatically sums the repeating words). Then, # we convert this COO matrix to CSR format which allows for fast querying of # documents. num_documents = data.shape[0] indices = np.array([(row_idx, column_idx) for row_idx, row in enumerate(data) for column_idx in row]) sparse_matrix = scipy.sparse.coo_matrix( (np.ones(indices.shape[0]), (indices[:, 0], indices[:, 1])), shape=(num_documents, num_words), dtype=np.float32) sparse_matrix = sparse_matrix.tocsr() dataset = tf.data.Dataset.range(num_documents) # For training, we shuffle each epoch and repeat the epochs. if shuffle_and_repeat: dataset = dataset.shuffle(num_documents).repeat() # Returns a single document as a dense TensorFlow tensor. The dataset is # stored as a sparse matrix outside of the graph. def get_row_py_func(idx): def get_row_python(idx_py): return np.squeeze(np.array(sparse_matrix[idx_py].todense()), axis=0) py_func = tf.compat.v1.py_func( get_row_python, [idx], tf.float32, stateful=False) py_func.set_shape((num_words,)) return py_func dataset = dataset.map(get_row_py_func) return dataset def build_fake_input_fns(batch_size): """Builds fake data for unit testing.""" num_words = 1000 vocabulary = [str(i) for i in range(num_words)] random_sample = np.random.randint( 10, size=(batch_size, num_words)).astype(np.float32) def train_input_fn(): dataset = tf.data.Dataset.from_tensor_slices(random_sample) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -