apache / nutch

Duplication

Places in code with 6 or more lines that are exactly the same.

Longest Duplicates

The list of 50 longest duplicates.

See data for all 3,467 duplicates...

Size	#	Folders	Files	Lines	Code
140	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	DOMContentUtils.java DOMContentUtils.java	134:417 (59%) 132:415 (59%)	view
136	x 2	src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	68:274 (44%) 63:269 (50%)	view
74	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	DOMBuilder.java DOMBuilder.java	350:774 (51%) 359:776 (47%)	view
73	x 2	src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	416:530 (23%) 366:480 (26%)	view
58	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-inte...col/interactiveselenium	HttpResponse.java HttpResponse.java	86:170 (17%) 81:164 (19%)	view
58	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	86:170 (17%) 76:159 (21%)	view
36	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	537:595 (10%) 422:480 (13%)	view
36	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-inte...col/interactiveselenium	HttpResponse.java HttpResponse.java	537:595 (10%) 472:530 (11%)	view
34	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	478:524 (10%) 366:412 (12%)	view
34	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-inte...col/interactiveselenium	HttpResponse.java HttpResponse.java	478:524 (10%) 416:462 (11%)	view
33	x 2	src/java/org/apache/nutch/crawl src/java/org/apache/nutch/crawl	AdaptiveFetchSchedule.java MimeAdaptiveFetchSchedule.java	322:360 (17%) 189:233 (31%)	view
31	x 2	src/java/org/apache/nutch/tools/warc src/java/org/apache/nutch/tools/warc	WARCExporter.java WARCExporter.java	306:341 (9%) 366:401 (9%)	view
29	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-inte...col/interactiveselenium	HttpResponse.java HttpResponse.java	179:218 (7%) 181:219 (9%)	view
29	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	179:218 (7%) 176:214 (10%)	view
28	x 2	src/plugin/index-geoip/s...che/nutch/indexer/geoip src/plugin/index-geoip/s...che/nutch/indexer/geoip	GeoIPDocumentCreator.java GeoIPDocumentCreator.java	84:118 (21%) 199:234 (21%)	view
28	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	DOMContentUtils.java DOMContentUtils.java	450:489 (11%) 417:455 (11%)	view
28	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	DOMBuilder.java DOMBuilder.java	125:266 (19%) 132:271 (17%)	view
27	x 2	src/plugin/indexer-elast...tch/indexwriter/elastic src/plugin/indexer-opens...ndexwriter/opensearch1x	ElasticIndexWriter.java OpenSearch1xIndexWriter.java	282:332 (13%) 347:397 (10%)	view
25	x 2	src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium	DummyX509TrustManager.java DummyX509TrustManager.java	32:95 (100%) 32:95 (100%)	view
25	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-http...tch/protocol/httpclient	DummyX509TrustManager.java DummyX509TrustManager.java	32:95 (100%) 32:95 (100%)	view
25	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-inte...col/interactiveselenium	DummyX509TrustManager.java DummyX509TrustManager.java	32:95 (100%) 32:95 (100%)	view
25	x 2	src/plugin/protocol-http...tch/protocol/httpclient src/plugin/protocol-sele...nutch/protocol/selenium	DummyX509TrustManager.java DummyX509TrustManager.java	32:95 (100%) 32:95 (100%)	view
25	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-sele...nutch/protocol/selenium	DummyX509TrustManager.java DummyX509TrustManager.java	32:95 (100%) 32:95 (100%)	view
25	x 2	src/java/org/apache/nutch/crawl src/java/org/apache/nutch/crawl	CrawlDbReader.java CrawlDbReader.java	220:250 (2%) 303:333 (2%)	view
25	x 2	src/plugin/protocol-http...tch/protocol/httpclient src/plugin/protocol-inte...col/interactiveselenium	DummyX509TrustManager.java DummyX509TrustManager.java	32:95 (100%) 32:95 (100%)	view
24	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-http...che/nutch/protocol/http	HttpResponse.java HttpResponse.java	360:402 (7%) 431:474 (6%)	view
23	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-sele...nutch/protocol/selenium	DummyX509TrustManager.java DummyX509TrustManager.java	51:102 (85%) 44:95 (92%)	view
23	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	174:206 (7%) 162:193 (8%)	view
23	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-http...che/nutch/protocol/http	DummyX509TrustManager.java DummyX509TrustManager.java	44:95 (92%) 51:102 (85%)	view
23	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-http...tch/protocol/httpclient	DummyX509TrustManager.java DummyX509TrustManager.java	51:102 (85%) 44:95 (92%)	view
23	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-inte...col/interactiveselenium	DummyX509TrustManager.java DummyX509TrustManager.java	51:102 (85%) 44:95 (92%)	view
23	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	DOMBuilder.java DOMBuilder.java	271:349 (15%) 279:357 (14%)	view
23	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-inte...col/interactiveselenium	HttpResponse.java HttpResponse.java	174:206 (7%) 167:198 (7%)	view
21	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-inte...col/interactiveselenium	HttpResponse.java HttpResponse.java	105:131 (5%) 105:131 (6%)	view
21	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	DOMContentUtils.java DOMContentUtils.java	73:101 (8%) 75:103 (8%)	view
21	x 2	src/plugin/indexer-elast...tch/indexwriter/elastic src/plugin/indexer-opens...ndexwriter/opensearch1x	ElasticIndexWriter.java OpenSearch1xIndexWriter.java	251:278 (10%) 316:343 (7%)	view
21	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-http...che/nutch/protocol/http	HttpResponse.java HttpResponse.java	111:137 (6%) 105:131 (5%)	view
21	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	105:131 (5%) 100:126 (7%)	view
20	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-inte...col/interactiveselenium	HttpResponse.java HttpResponse.java	294:330 (6%) 323:358 (6%)	view
20	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	HTMLMetaProcessor.java HTMLMetaProcessor.java	183:212 (18%) 164:193 (15%)	view
19	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	295:330 (5%) 318:352 (7%)	view
19	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-http...che/nutch/protocol/http	HttpResponse.java HttpResponse.java	405:436 (5%) 477:508 (4%)	view
19	x 2	src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	324:358 (6%) 318:352 (7%)	view
19	x 2	src/java/org/apache/nutch/service/impl src/java/org/apache/nutch/service/impl	LinkReader.java NodeReader.java	52:79 (19%) 52:80 (18%)	view
18	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	338:372 (4%) 319:352 (6%)	view
18	x 2	src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-inte...col/interactiveselenium	HttpResponse.java HttpResponse.java	338:372 (4%) 325:358 (5%)	view
18	x 2	src/plugin/protocol-html...nutch/protocol/htmlunit src/plugin/protocol-http...che/nutch/protocol/http	HttpResponse.java HttpResponse.java	296:330 (5%) 338:372 (4%)	view
17	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	HTMLMetaProcessor.java HTMLMetaProcessor.java	33:72 (15%) 35:74 (13%)	view
17	x 2	src/plugin/parse-html/sr...apache/nutch/parse/html src/plugin/parse-tika/sr...apache/nutch/parse/tika	HTMLMetaProcessor.java HTMLMetaProcessor.java	147:177 (15%) 216:246 (13%)	view
17	x 2	src/plugin/urlfilter-dom.../nutch/urlfilter/domain src/plugin/urlfilter-dom...rlfilter/domaindenylist	DomainURLFilter.java DomainDenylistURLFilter.java	134:158 (28%) 135:159 (28%)	view

Duplicated Units

The list of top 23 duplicated units.

See data for all 23 unit duplicates...

Size	#	Folders	Files	Lines	Code
43	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMContentUtils.java DOMContentUtils.java	152:201  154:203	view
36	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMContentUtils.java DOMContentUtils.java	328:377  330:379	view
24	x 3	src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium src/plugin/protocol-html...nutch/protocol/htmlunit	HttpResponse.java HttpResponse.java HttpResponse.java	498:525  448:475  563:590	view
24	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMBuilder.java DOMBuilder.java	141:174  136:169	view
21	x 3	src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium src/plugin/protocol-html...nutch/protocol/htmlunit	HttpResponse.java HttpResponse.java HttpResponse.java	439:463  389:413  501:525	view
21	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMContentUtils.java DOMContentUtils.java	284:315  286:317	view
17	x 2	src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium	HttpResponse.java HttpResponse.java	466:496  416:446	view
17	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMBuilder.java DOMBuilder.java	408:429  398:419	view
16	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMContentUtils.java DOMContentUtils.java	254:277  256:279	view
14	x 3	src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium src/plugin/protocol-html...nutch/protocol/htmlunit	HttpResponse.java HttpResponse.java HttpResponse.java	416:437  366:387  478:499	view
13	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMContentUtils.java DOMContentUtils.java	211:226  213:228	view
10	x 5	src/plugin/protocol-http...tch/protocol/httpclient src/plugin/protocol-inte...col/interactiveselenium src/plugin/protocol-sele...nutch/protocol/selenium src/plugin/protocol-http...che/nutch/protocol/http src/plugin/protocol-html...nutch/protocol/htmlunit	DummyX509TrustManager.java DummyX509TrustManager.java DummyX509TrustManager.java DummyX509TrustManager.java DummyX509TrustManager.java	44:55  44:55  44:55  51:62  44:55	view
10	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMBuilder.java DOMBuilder.java	645:659  643:657	view
8	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMBuilder.java DOMBuilder.java	443:454  435:446	view
8	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	XMLCharacterRecognizer.java XMLCharacterRecognizer.java	98:110  98:110	view
7	x 2	src/plugin/indexer-opens...ndexwriter/opensearch1x src/plugin/indexer-elast...tch/indexwriter/elastic	OpenSearch1xIndexWriter.java ElasticIndexWriter.java	376:386  311:321	view
7	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMContentUtils.java DOMContentUtils.java	235:243  237:245	view
7	x 2	src/plugin/urlfilter-dom.../nutch/urlfilter/domain src/plugin/urlfilter-dom...rlfilter/domaindenylist	DomainURLFilter.java DomainDenylistURLFilter.java	87:98  87:98	view
6	x 3	src/java/org/apache/nutch/util src/java/org/apache/nutch/util src/java/org/apache/nutch/util	DomainStatistics.java CrawlCompletionStats.java ProtocolStatusStatistics.java	219:228  233:242  154:163	view
6	x 3	src/java/org/apache/nutch/util src/java/org/apache/nutch/util src/java/org/apache/nutch/util	DomainStatistics.java CrawlCompletionStats.java ProtocolStatusStatistics.java	234:242  248:256  169:177	view
6	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	DOMContentUtils.java DOMContentUtils.java	317:324  319:326	view
6	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	XMLCharacterRecognizer.java XMLCharacterRecognizer.java	60:70  60:70	view
6	x 2	src/plugin/parse-tika/sr...apache/nutch/parse/tika src/plugin/parse-html/sr...apache/nutch/parse/html	XMLCharacterRecognizer.java XMLCharacterRecognizer.java	79:89  79:89	view