huggingface / picotron_tutorial

Duplication

Places in code with 6 or more lines that are exactly the same.

Longest Duplicates

The list of 50 longest duplicates.

See data for all 245 duplicates...

Size	#	Folders	Files	Lines	Code
207	x 2	step4_tensor_parallel step7_pipeline_parallel_afab	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step5_data_parallel_naive step8_pipeline_parallel_1f1b	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step5_data_parallel_naive step7_pipeline_parallel_afab	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step5_data_parallel_naive step6_data_parallel_bucket	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step4_tensor_parallel step6_data_parallel_bucket	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step6_data_parallel_bucket step8_pipeline_parallel_1f1b	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step6_data_parallel_bucket step7_pipeline_parallel_afab	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step4_tensor_parallel step5_data_parallel_naive	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step4_tensor_parallel step8_pipeline_parallel_1f1b	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
207	x 2	step7_pipeline_parallel_afab step8_pipeline_parallel_1f1b	tensor_parallel.py tensor_parallel.py	10:299 (100%) 10:299 (100%)	view
139	x 2	step6_data_parallel_bucket step8_pipeline_parallel_1f1b	data_parallel.py data_parallel.py	10:257 (100%) 10:255 (100%)	view
139	x 2	step7_pipeline_parallel_afab step8_pipeline_parallel_1f1b	data_parallel.py data_parallel.py	10:255 (100%) 10:255 (100%)	view
139	x 2	step6_data_parallel_bucket step7_pipeline_parallel_afab	data_parallel.py data_parallel.py	10:257 (100%) 10:255 (100%)	view
116	x 2	step4_tensor_parallel step5_data_parallel_naive	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step5_data_parallel_naive step7_pipeline_parallel_afab	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step5_data_parallel_naive step8_pipeline_parallel_1f1b	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step4_tensor_parallel step6_data_parallel_bucket	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step6_data_parallel_bucket step8_pipeline_parallel_1f1b	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step4_tensor_parallel step8_pipeline_parallel_1f1b	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step7_pipeline_parallel_afab step8_pipeline_parallel_1f1b	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step5_data_parallel_naive step6_data_parallel_bucket	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step6_data_parallel_bucket step7_pipeline_parallel_afab	model.py model.py	9:152 (100%) 9:152 (100%)	view
116	x 2	step4_tensor_parallel step7_pipeline_parallel_afab	model.py model.py	9:152 (100%) 9:152 (100%)	view
114	x 2	step1_modeling step2_process_group_manager	model.py model.py	8:148 (100%) 8:148 (100%)	view
114	x 2	step2_process_group_manager step3_dataloader	model.py model.py	8:148 (100%) 8:148 (100%)	view
114	x 2	step1_modeling step3_dataloader	model.py model.py	8:148 (100%) 8:148 (100%)	view
113	x 2	step7_pipeline_parallel_afab step8_pipeline_parallel_1f1b	train.py train.py	27:195 (81%) 27:195 (80%)	view
92	x 2	step5_data_parallel_naive step7_pipeline_parallel_afab	dataloader.py dataloader.py	10:121 (100%) 10:121 (100%)	view
92	x 2	step5_data_parallel_naive step8_pipeline_parallel_1f1b	dataloader.py dataloader.py	10:121 (100%) 10:121 (100%)	view
92	x 2	step6_data_parallel_bucket step8_pipeline_parallel_1f1b	dataloader.py dataloader.py	10:121 (100%) 10:121 (100%)	view
92	x 2	step7_pipeline_parallel_afab step8_pipeline_parallel_1f1b	dataloader.py dataloader.py	10:121 (100%) 10:121 (100%)	view
92	x 2	step5_data_parallel_naive step6_data_parallel_bucket	dataloader.py dataloader.py	10:121 (100%) 10:121 (100%)	view
92	x 2	step6_data_parallel_bucket step7_pipeline_parallel_afab	dataloader.py dataloader.py	10:121 (100%) 10:121 (100%)	view
84	x 2	step3_dataloader step4_tensor_parallel	dataloader.py dataloader.py	10:112 (100%) 10:112 (100%)	view
78	x 2	step5_data_parallel_naive step6_data_parallel_bucket	train.py train.py	26:141 (63%) 26:141 (63%)	view
76	x 2	step5_data_parallel_naive step7_pipeline_parallel_afab	train.py train.py	26:135 (61%) 35:144 (55%)	view
76	x 2	step5_data_parallel_naive step8_pipeline_parallel_1f1b	train.py train.py	26:135 (61%) 35:144 (54%)	view
76	x 2	step6_data_parallel_bucket step7_pipeline_parallel_afab	train.py train.py	26:135 (61%) 35:144 (55%)	view
76	x 2	step6_data_parallel_bucket step8_pipeline_parallel_1f1b	train.py train.py	26:135 (61%) 35:144 (54%)	view
71	x 2	step1_modeling step5_data_parallel_naive	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step1_modeling step7_pipeline_parallel_afab	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step1_modeling step8_pipeline_parallel_1f1b	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step2_process_group_manager step7_pipeline_parallel_afab	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step3_dataloader step5_data_parallel_naive	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step2_process_group_manager step8_pipeline_parallel_1f1b	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step2_process_group_manager step5_data_parallel_naive	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step1_modeling step4_tensor_parallel	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step3_dataloader step7_pipeline_parallel_afab	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step3_dataloader step6_data_parallel_bucket	model.py model.py	58:148 (62%) 62:152 (61%)	view
71	x 2	step2_process_group_manager step4_tensor_parallel	model.py model.py	58:148 (62%) 62:152 (61%)	view

Duplicated Units

The list of top 9 duplicated units.

See data for all 9 unit duplicates...

Size	#	Folders	Files	Lines	Code
36	x 5	step7_pipeline_parallel_afab step5_data_parallel_naive step8_pipeline_parallel_1f1b step4_tensor_parallel step6_data_parallel_bucket	tensor_parallel.py tensor_parallel.py tensor_parallel.py tensor_parallel.py tensor_parallel.py	0:0  0:0  0:0  0:0  0:0	view
36	x 3	step7_pipeline_parallel_afab step8_pipeline_parallel_1f1b step6_data_parallel_bucket	data_parallel.py data_parallel.py data_parallel.py	0:0  0:0  0:0	view
32	x 7	step7_pipeline_parallel_afab step5_data_parallel_naive step2_process_group_manager step8_pipeline_parallel_1f1b step3_dataloader step4_tensor_parallel step6_data_parallel_bucket	process_group_manager.py process_group_manager.py process_group_manager.py process_group_manager.py process_group_manager.py process_group_manager.py process_group_manager.py	0:0  0:0  0:0  0:0  0:0  0:0  0:0	view
25	x 4	step7_pipeline_parallel_afab step5_data_parallel_naive step8_pipeline_parallel_1f1b step6_data_parallel_bucket	dataloader.py dataloader.py dataloader.py dataloader.py	0:0  0:0  0:0  0:0	view
17	x 8	step7_pipeline_parallel_afab step5_data_parallel_naive step2_process_group_manager step8_pipeline_parallel_1f1b step1_modeling step3_dataloader step4_tensor_parallel step6_data_parallel_bucket	model.py model.py model.py model.py model.py model.py model.py model.py	0:0  0:0  0:0  0:0  0:0  0:0  0:0  0:0	view
18	x 2	step3_dataloader step4_tensor_parallel	dataloader.py dataloader.py	0:0  0:0	view
18	x 4	step7_pipeline_parallel_afab step5_data_parallel_naive step8_pipeline_parallel_1f1b step6_data_parallel_bucket	train.py train.py train.py train.py	0:0  0:0  0:0  0:0	view
15	x 8	step7_pipeline_parallel_afab step5_data_parallel_naive step2_process_group_manager step8_pipeline_parallel_1f1b step1_modeling step3_dataloader step4_tensor_parallel step6_data_parallel_bucket	utils.py utils.py utils.py utils.py utils.py utils.py utils.py utils.py	0:0  0:0  0:0  0:0  0:0  0:0  0:0  0:0	view
14	x 2	step3_dataloader step4_tensor_parallel	train.py train.py	0:0  0:0	view