s3_diver/s3_diver_03-test.py [172:232]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - recv_buf_size = recv_buf.tell() #print ('1. recv_buf_pos: %s' % recv_buf.tell()) if recv_buf_size > max_part_size: print('multi part uploading: %s / %s , size: %s' % (parts_index, max_part_count, recv_buf_size)) chunk_count = int(recv_buf_size / max_part_size) tar_file_size = tar_file_size + recv_buf_size print('%s is accumulating, size: %s' % (key_name, tar_file_size)) #print('chunk_count: %s ' % chunk_count) for buf_index in range(chunk_count): start_pos = buf_index * max_part_size recv_buf.seek(start_pos,0) mpu_parts = upload_mpu(key_name, mpu_id, recv_buf.read(max_part_size), parts_index) parts_index += 1 #################### buf_fifo(recv_buf) recv_buf_size = recv_buf.tell() #print('3.after fifo, recv_buf_pos : %s' % recv_buf.tell()) #print ('3. after fifo, recv_buf_size: %s' % len(recv_buf.getvalue())) else: pass #print('accumulating files...') else: log_error(e_log, org_file," does not exist\n") print (org_file + ' is not exist...............................................\n') recv_buf.seek(0,0) mpu_parts = upload_mpu(key_name, mpu_id, recv_buf.read(), parts_index) parts_index += 1 mpu_parts = adjusting_parts_order(mpu_parts) complete_mpu(key_name, mpu_id, mpu_parts) ### print metadata meta_out = s3.head_object(Bucket=bucket_name, Key=key_name) print ('\n metadata info: %s' % str(meta_out)) log_success(s_log, str(meta_out), '!!\n') print ("\n tar file: %s \n" % key_name) log_success(s_log, key_name, ' is uploaded successfully\n') def snowball_uploader_help(**args): print ("Usage: %s 'genlist | cp_snowball | help'" % sys.argv[0]) print ("use python3, not compatible with python2!!!") #print ('\n') print ('genlist: ') print ('this option will generate files which are containing target files list in %s'% (filelist_dir)) #print ('\n') print ('cp_snowball: ') print ('cp_snowball option will copy the files on server to snowball efficiently') print ('the mechanism is here:') print ('1. reads the target file name from the one filelist file in filelist directory') print ('2. accumulates files to max_part_size in memory') print ('3. if it reachs max_part_size, send it to snowball using MultiPartUpload') print ('4. tar files uploading to Snowball concurrently according to max_process') print ('5. after complete to send, tar file is generated in snowball') print ('6. then, moves to the next filelist file recursively') if __name__ == "__main__": if len(sys.argv) != 2: print ("Usage: %s genlist | cp_snowball | help" % sys.argv[0]) print ("use python3, not compatible with python2!!!") sys.exit() elif sys.argv[1] == "genlist": gen_filelist() elif sys.argv[1] == "cp_snowball": - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - snowball_uploader_26-success.py [206:266]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - recv_buf_size = recv_buf.tell() #print ('1. recv_buf_pos: %s' % recv_buf.tell()) if recv_buf_size > max_part_size: print('multi part uploading: %s / %s , size: %s' % (parts_index, max_part_count, recv_buf_size)) chunk_count = int(recv_buf_size / max_part_size) tar_file_size = tar_file_size + recv_buf_size print('%s is accumulating, size: %s' % (key_name, tar_file_size)) #print('chunk_count: %s ' % chunk_count) for buf_index in range(chunk_count): start_pos = buf_index * max_part_size recv_buf.seek(start_pos,0) mpu_parts = upload_mpu(key_name, mpu_id, recv_buf.read(max_part_size), parts_index) parts_index += 1 #################### buf_fifo(recv_buf) recv_buf_size = recv_buf.tell() #print('3.after fifo, recv_buf_pos : %s' % recv_buf.tell()) #print ('3. after fifo, recv_buf_size: %s' % len(recv_buf.getvalue())) else: pass #print('accumulating files...') else: log_error(e_log, org_file," does not exist\n") print (org_file + ' is not exist...............................................\n') recv_buf.seek(0,0) mpu_parts = upload_mpu(key_name, mpu_id, recv_buf.read(), parts_index) parts_index += 1 mpu_parts = adjusting_parts_order(mpu_parts) complete_mpu(key_name, mpu_id, mpu_parts) ### print metadata meta_out = s3.head_object(Bucket=bucket_name, Key=key_name) print ('\n metadata info: %s' % str(meta_out)) log_success(s_log, str(meta_out), '!!\n') print ("\n tar file: %s \n" % key_name) log_success(s_log, key_name, ' is uploaded successfully\n') def snowball_uploader_help(**args): print ("Usage: %s 'genlist | cp_snowball | help'" % sys.argv[0]) print ("use python3, not compatible with python2!!!") #print ('\n') print ('genlist: ') print ('this option will generate files which are containing target files list in %s'% (filelist_dir)) #print ('\n') print ('cp_snowball: ') print ('cp_snowball option will copy the files on server to snowball efficiently') print ('the mechanism is here:') print ('1. reads the target file name from the one filelist file in filelist directory') print ('2. accumulates files to max_part_size in memory') print ('3. if it reachs max_part_size, send it to snowball using MultiPartUpload') print ('4. tar files uploading to Snowball concurrently according to max_process') print ('5. after complete to send, tar file is generated in snowball') print ('6. then, moves to the next filelist file recursively') if __name__ == "__main__": if len(sys.argv) != 2: print ("Usage: %s genlist | cp_snowball | help" % sys.argv[0]) print ("use python3, not compatible with python2!!!") sys.exit() elif sys.argv[1] == "genlist": gen_filelist() elif sys.argv[1] == "cp_snowball": - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -