int FdEntity::RowFlushMixMultipart()

in src/fdcache_entity.cpp [1609:1732]


int FdEntity::RowFlushMixMultipart(PseudoFdInfo* pseudo_obj, const char* tpath)
{
    S3FS_PRN_INFO3("[tpath=%s][path=%s][pseudo_fd=%d][physical_fd=%d]", SAFESTRPTR(tpath), path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd);

    if(-1 == physical_fd || !pseudo_obj){
        return -EBADF;
    }

    int result = 0;

    if(!pseudo_obj->IsUploading()){
        // Start uploading

        // If there is no loading all of the area, loading all area.
        off_t restsize = pagelist.GetTotalUnloadedPageSize(/* start */ 0, /* size = all */ 0, MIN_MULTIPART_SIZE);

        // Check rest size and free disk space
        if(0 < restsize && !ReserveDiskSpace(restsize)){
           // no enough disk space
           if(0 != (result = NoCachePreMultipartPost(pseudo_obj))){
               S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result);
               return result;
           }
           // upload all by multipart uploading
           if(0 != (result = NoCacheLoadAndPost(pseudo_obj))){
               S3FS_PRN_ERR("failed to upload all area by multipart uploading(errno=%d)", result);
               return result;
           }

        }else{
            // enough disk space or no rest size
            std::string tmppath    = path;
            headers_t   tmporgmeta = orgmeta;

            FdManager::FreeReservedDiskSpace(restsize);

            // backup upload file size
            struct stat st;
            memset(&st, 0, sizeof(struct stat));
            if(-1 == fstat(physical_fd, &st)){
                S3FS_PRN_ERR("fstat is failed by errno(%d), but continue...", errno);
            }

            if(pagelist.Size() > MAX_MULTIPART_CNT * S3fsCurl::GetMultipartSize()){
                S3FS_PRN_ERR("Part count exceeds %d.  Increase multipart size and try again.", MAX_MULTIPART_CNT);
                return -EFBIG;

            }else if(pagelist.Size() >= S3fsCurl::GetMultipartSize()){
                // mix multipart uploading

                // This is to ensure that each part is 5MB or more.
                // If the part is less than 5MB, download it.
                fdpage_list_t dlpages;
                fdpage_list_t mixuppages;
                if(!pagelist.GetPageListsForMultipartUpload(dlpages, mixuppages, S3fsCurl::GetMultipartSize())){
                    S3FS_PRN_ERR("something error occurred during getting download pagelist.");
                    return -1;
                }

                // [TODO] should use parallel downloading
                //
                for(fdpage_list_t::const_iterator iter = dlpages.begin(); iter != dlpages.end(); ++iter){
                    if(0 != (result = Load(iter->offset, iter->bytes, AutoLock::ALREADY_LOCKED, /*is_modified_flag=*/ true))){  // set loaded and modified flag
                        S3FS_PRN_ERR("failed to get parts(start=%lld, size=%lld) before uploading.", static_cast<long long int>(iter->offset), static_cast<long long int>(iter->bytes));
                        return result;
                    }
                }

                // multipart uploading with copy api
                result = S3fsCurl::ParallelMixMultipartUploadRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd, mixuppages);

            }else{
                // normal uploading (too small part size)

                // If there are unloaded pages, they are loaded at here.
                if(0 != (result = Load(/*start=*/ 0, /*size=*/ 0, AutoLock::ALREADY_LOCKED))){
                    S3FS_PRN_ERR("failed to load parts before uploading object(%d)", result);
                    return result;
                }

                S3fsCurl s3fscurl(true);
                result = s3fscurl.PutRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd);
            }

            // reset uploaded file size
            size_orgmeta = st.st_size;
        }
        pseudo_obj->ClearUntreated();

    }else{
        // Already start uploading

        // upload rest data
        off_t untreated_start = 0;
        off_t untreated_size  = 0;
        if(pseudo_obj->GetLastUntreated(untreated_start, untreated_size, S3fsCurl::GetMultipartSize(), 0) && 0 < untreated_size){
            if(0 != (result = NoCacheMultipartPost(pseudo_obj, physical_fd, untreated_start, untreated_size))){
                S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(physical_fd=%d).", static_cast<long long int>(untreated_start), static_cast<long long int>(untreated_size), physical_fd);
                return result;
            }
            pseudo_obj->ClearUntreated(untreated_start, untreated_size);
	    }
        // complete multipart uploading.
        if(0 != (result = NoCacheCompleteMultipartPost(pseudo_obj))){
            S3FS_PRN_ERR("failed to complete(finish) multipart post for file(physical_fd=%d).", physical_fd);
            return result;
        }
        // truncate file to zero
        if(-1 == ftruncate(physical_fd, 0)){
            // So the file has already been removed, skip error.
            S3FS_PRN_ERR("failed to truncate file(physical_fd=%d) to zero, but continue...", physical_fd);
        }
        // put pending headers
        if(0 != (result = UploadPendingMeta())){
            return result;
        }
    }

    if(0 == result){
        pagelist.ClearAllModified();
        is_meta_pending = false;
    }
    return result;
}