opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java [277:311]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	public static List<String> removeDuplicatesFromQueries(List<String> hits) {
		StringDistanceMeasurer meas = new StringDistanceMeasurer();
		double dupeThresh = 0.8; // if more similar, then considered dupes was
		// 0.7
		List<Integer> idsToRemove = new ArrayList<>();
		List<String> hitsDedup = new ArrayList<>();
		try {
			for (int i = 0; i < hits.size(); i++)
				for (int j = i + 1; j < hits.size(); j++) {
					String title1 = hits.get(i);
					String title2 = hits.get(j);
					if (StringUtils.isEmpty(title1) || StringUtils.isEmpty(title2))
						continue;
					if (meas.measureStringDistance(title1, title2) > dupeThresh) {
						idsToRemove.add(j); // dupes found, later list member to
						// be deleted

					}
				}

			for (int i = 0; i < hits.size(); i++)
				if (!idsToRemove.contains(i))
					hitsDedup.add(hits.get(i));

			if (hitsDedup.size() < hits.size()) {
				LOG.info("Removed duplicates from formed query, including {}", hits.get(idsToRemove.get(0)));
			}

		} catch (Exception e) {
			LOG.error("Problem removing duplicates from query list", e);
		}

		return hitsDedup;

	}
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java [158:190]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	public static List<String> removeDuplicatesFromQueries(List<String> hits) {
		StringDistanceMeasurer meas = new StringDistanceMeasurer();
		double dupeThresh = 0.8; // if more similar, then considered dupes was
		// 0.7
		List<Integer> idsToRemove = new ArrayList<>();
		List<String> hitsDedup = new ArrayList<>();
		try {
			for (int i = 0; i < hits.size(); i++)
				for (int j = i + 1; j < hits.size(); j++) {
					String title1 = hits.get(i);
					String title2 = hits.get(j);
					if (StringUtils.isEmpty(title1) || StringUtils.isEmpty(title2))
						continue;
					if (meas.measureStringDistance(title1, title2) > dupeThresh) {
						idsToRemove.add(j); // dupes found, later list member to be deleted
					}
				}

			for (int i = 0; i < hits.size(); i++)
				if (!idsToRemove.contains(i))
					hitsDedup.add(hits.get(i));

			if (hitsDedup.size() < hits.size()) {
				LOG.info("Removed duplicates from formed query, including {}", hits.get(idsToRemove.get(0)));
			}

		} catch (Exception e) {
			LOG.error("Problem removing duplicates from query list", e);
		}

		return hitsDedup;

	}
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -