opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java [158:212]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	public static List<String> removeDuplicatesFromQueries(List<String> hits) {
		StringDistanceMeasurer meas = new StringDistanceMeasurer();
		double dupeThresh = 0.8; // if more similar, then considered dupes was
		// 0.7
		List<Integer> idsToRemove = new ArrayList<>();
		List<String> hitsDedup = new ArrayList<>();
		try {
			for (int i = 0; i < hits.size(); i++)
				for (int j = i + 1; j < hits.size(); j++) {
					String title1 = hits.get(i);
					String title2 = hits.get(j);
					if (StringUtils.isEmpty(title1) || StringUtils.isEmpty(title2))
						continue;
					if (meas.measureStringDistance(title1, title2) > dupeThresh) {
						idsToRemove.add(j); // dupes found, later list member to be deleted
					}
				}

			for (int i = 0; i < hits.size(); i++)
				if (!idsToRemove.contains(i))
					hitsDedup.add(hits.get(i));

			if (hitsDedup.size() < hits.size()) {
				LOG.info("Removed duplicates from formed query, including {}", hits.get(idsToRemove.get(0)));
			}

		} catch (Exception e) {
			LOG.error("Problem removing duplicates from query list", e);
		}

		return hitsDedup;

	}

	/**
	 * remove dupes from search results
	 * 
	 * @param hits  A List<HitBase> of search results objects
	 * @return List<String> of search results objects where dupes are removed
	 */
	public static List<HitBase> removeDuplicatesFromResultantHits(List<HitBase> hits) {
		StringDistanceMeasurer meas = new StringDistanceMeasurer();
		double dupeThresh = // 0.8; // if more similar, then considered dupes was
				0.7;
		try {
			for (int i = 0; i < hits.size(); i++)
				for (int j = i + 1; j < hits.size(); j++) {
					HitBase hit2 = hits.get(j);
					List<Fragment> fragmList1 = hits.get(i).getFragments();
					List<Fragment> fragmList2 = hits.get(j).getFragments();
					List<Fragment> fragmList2Results = new ArrayList<>(fragmList2);
					for (Fragment f1 : fragmList1)
						for (Fragment f2 : fragmList2) {
							String sf1 = f1.getResultText();
							String sf2 = f2.getResultText();
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java [277:333]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	public static List<String> removeDuplicatesFromQueries(List<String> hits) {
		StringDistanceMeasurer meas = new StringDistanceMeasurer();
		double dupeThresh = 0.8; // if more similar, then considered dupes was
		// 0.7
		List<Integer> idsToRemove = new ArrayList<>();
		List<String> hitsDedup = new ArrayList<>();
		try {
			for (int i = 0; i < hits.size(); i++)
				for (int j = i + 1; j < hits.size(); j++) {
					String title1 = hits.get(i);
					String title2 = hits.get(j);
					if (StringUtils.isEmpty(title1) || StringUtils.isEmpty(title2))
						continue;
					if (meas.measureStringDistance(title1, title2) > dupeThresh) {
						idsToRemove.add(j); // dupes found, later list member to
						// be deleted

					}
				}

			for (int i = 0; i < hits.size(); i++)
				if (!idsToRemove.contains(i))
					hitsDedup.add(hits.get(i));

			if (hitsDedup.size() < hits.size()) {
				LOG.info("Removed duplicates from formed query, including {}", hits.get(idsToRemove.get(0)));
			}

		} catch (Exception e) {
			LOG.error("Problem removing duplicates from query list", e);
		}

		return hitsDedup;

	}

	/**
	 * remove dupes from search results
	 *
	 * @param hits List<HitBase> of search results objects
	 * @return List<String> of search results objects where dupes are removed
	 */
	public static List<HitBase> removeDuplicatesFromResultantHits(List<HitBase> hits) {
		StringDistanceMeasurer meas = new StringDistanceMeasurer();
		double dupeThresh = // 0.8; // if more similar, then considered dupes was
						0.7;
		try {
			for (int i = 0; i < hits.size(); i++)
				for (int j = i + 1; j < hits.size(); j++) {
					HitBase hit2 = hits.get(j);
					List<Fragment> fragmList1 = hits.get(i).getFragments();
					List<Fragment> fragmList2 = hits.get(j).getFragments();
					List<Fragment> fragmList2Results = new ArrayList<>(fragmList2);
					for (Fragment f1 : fragmList1)
						for (Fragment f2 : fragmList2) {
							String sf1 = f1.getResultText();
							String sf2 = f2.getResultText();
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -