tools/integration_tests/util/operations/file_operations.go (586 lines of code) (raw):

// Copyright 2023 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Provide helper functions related to file. package operations import ( "bufio" "bytes" "compress/gzip" "fmt" "hash/crc32" "io" "io/fs" "log" "os" "path" "strconv" "strings" "syscall" "testing" "time" "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" ) const ( OneKiB = 1024 OneMiB = OneKiB * OneKiB // ChunkSizeForContentComparison is currently set to 1 MiB. ChunkSizeForContentComparison int = OneMiB // TimeSlop The radius we use for "expect mtime is within"-style assertions as kernel // time can be slightly out of sync of time.Now(). // Ref: https://github.com/golang/go/issues/33510 TimeSlop = 25 * time.Millisecond // TmpDirectory specifies the directory where temporary files will be created. // In this case, we are using the system's default temporary directory. TmpDirectory = "/tmp" ) func copyFile(srcFileName, dstFileName string, allowOverwrite bool) (err error) { if !allowOverwrite { if _, err = os.Stat(dstFileName); err == nil { err = fmt.Errorf("destination file %s already present", dstFileName) return } } source, err := os.OpenFile(srcFileName, syscall.O_DIRECT, FilePermission_0600) if err != nil { err = fmt.Errorf("file %s opening error: %v", srcFileName, err) return } // Closing file at the end. defer CloseFile(source) var destination *os.File if allowOverwrite { destination, err = os.OpenFile(dstFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, FilePermission_0600) } else { destination, err = os.OpenFile(dstFileName, os.O_WRONLY|os.O_CREATE, FilePermission_0600) } if err != nil { err = fmt.Errorf("copied file creation error: %v", err) return } // Closing file at the end. defer CloseFile(destination) // File copying with io.Copy() utility. _, err = io.Copy(destination, source) if err != nil { err = fmt.Errorf("error in file copying: %v", err) return } return } func CopyFile(srcFileName, newFileName string) (err error) { return copyFile(srcFileName, newFileName, false) } func CopyFileAllowOverwrite(srcFileName, newFileName string) (err error) { return copyFile(srcFileName, newFileName, true) } func ReadFile(filePath string) (content []byte, err error) { file, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, FilePermission_0600) if err != nil { err = fmt.Errorf("error in the opening the file %v", err) return } // Closing file at the end. defer CloseFile(file) content, err = os.ReadFile(file.Name()) if err != nil { err = fmt.Errorf("ReadAll: %v", err) return } return } func RenameFile(fileName string, newFileName string) (err error) { if _, err = os.Stat(newFileName); err == nil { err = fmt.Errorf("renamed file %s already present", newFileName) return } if err = os.Rename(fileName, newFileName); err != nil { err = fmt.Errorf("rename unsuccessful: %v", err) return } if _, err = os.Stat(fileName); err == nil { err = fmt.Errorf("original file %s still exists", fileName) return } if _, err = os.Stat(newFileName); err != nil { err = fmt.Errorf("renamed file %s not found", newFileName) return } return } func WriteFileInAppendMode(fileName string, content string) (err error) { f, err := os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY|syscall.O_DIRECT, FilePermission_0600) if err != nil { err = fmt.Errorf("open file for append: %v", err) return } // Closing file at the end. defer CloseFile(f) _, err = f.WriteString(content) return } func WriteFile(fileName string, content string) (err error) { f, err := os.OpenFile(fileName, os.O_RDWR|syscall.O_DIRECT, FilePermission_0600) if err != nil { err = fmt.Errorf("open file for write at start: %v", err) return } // Closing file at the end. defer CloseFile(f) _, err = f.WriteAt([]byte(content), 0) return } func CloseFiles(t *testing.T, files []*os.File) { t.Helper() for _, file := range files { err := file.Close() assert.NoError(t, err) } } // Deprecated: please use CloseFileShouldNotThrowError instead. func CloseFile(file *os.File) { if err := file.Close(); err != nil { log.Fatalf("error in closing: %v", err) } } func RemoveFile(filePath string) { err := os.Remove(filePath) if err != nil { log.Printf("os.Remove(%s): %v", filePath, err) } } func ReadFileSequentially(filePath string, chunkSize int64) (content []byte, err error) { chunk := make([]byte, chunkSize) var offset int64 = 0 file, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, FilePermission_0600) if err != nil { log.Printf("Error in opening file: %v", err) } // Closing the file at the end. defer CloseFile(file) for err != io.EOF { var numberOfBytes int // Reading 200 MB chunk sequentially from the file. numberOfBytes, err = file.ReadAt(chunk, offset) // If the file reaches the end, write the remaining content in the buffer and return. if err == io.EOF { for i := offset; i < offset+int64(numberOfBytes); i++ { // Adding remaining bytes. content = append(content, chunk[i-offset]) } err = nil return } if err != nil { return } // Write bytes in the buffer to compare with original content. content = append(content, chunk...) // The number of bytes read is not equal to 200MB. if int64(numberOfBytes) != chunkSize { log.Printf("Incorrect number of bytes read from file.") } // The offset will shift to read the next chunk. offset = offset + chunkSize } return } func WriteChunkOfRandomBytesToFiles(files []*os.File, chunkSize int, offset int64) error { // Generate random data of chunk size. chunk, err := GenerateRandomData(int64(chunkSize)) if err != nil { return fmt.Errorf("error in generating random data: %v", err) } for _, file := range files { // Write data in the file. n, err := file.WriteAt(chunk, offset) if err != nil { return fmt.Errorf("error in writing randomly in file: %s, %v", file.Name(), err) } if n != chunkSize { return fmt.Errorf("incorrect number of bytes written in the file %s actual %d, expected %d", file.Name(), n, chunkSize) } err = file.Sync() if err != nil { return fmt.Errorf("error in syncing file: %v", err) } } return nil } func WriteFilesSequentially(t *testing.T, filePaths []string, fileSize int64, chunkSize int64) { t.Helper() files := OpenFiles(t, filePaths) defer CloseFiles(t, files) var offset int64 = 0 for offset < fileSize { // Reduce chunk size to remaining file size in case chunk size is larger. chunkSize = min(chunkSize, fileSize-offset) err := WriteChunkOfRandomBytesToFiles(files, int(chunkSize), offset) assert.NoError(t, err) offset = offset + chunkSize } return } func ReadChunkFromFile(filePath string, chunkSize int64, offset int64, flag int) (chunk []byte, err error) { chunk = make([]byte, chunkSize) file, err := os.OpenFile(filePath, flag, FilePermission_0600) if err != nil { log.Printf("Error in opening file: %v", err) return } f, err := os.Stat(filePath) if err != nil { log.Printf("Error in stating file: %v", err) return } // Closing the file at the end. defer CloseFile(file) var numberOfBytes int // Reading chunk size randomly from the file. numberOfBytes, err = file.ReadAt(chunk, offset) if err == io.EOF { err = nil } if err != nil { return } // The number of bytes read is not equal to 200MB. if int64(numberOfBytes) != chunkSize && int64(numberOfBytes) != f.Size()-offset { log.Printf("Incorrect number of bytes read from file.") } return } func ReadFileBetweenOffset(t *testing.T, file *os.File, startOffset, endOffset, chunkSize int64) string { t.Helper() chunk := make([]byte, chunkSize) var readData []byte for startOffset < endOffset { readSize := min(chunkSize, endOffset-startOffset) n, err := file.ReadAt(chunk[:readSize], startOffset) if err == io.EOF { readData = append(readData, chunk[:n]...) break } else if err != nil { t.Errorf("Failed to read file chunk at offset %d: %v", startOffset, err) return "" } readData = append(readData, chunk[:n]...) startOffset += int64(n) } return string(readData) } // Returns the stats of a file. // Fails if the passed input is a directory. func StatFile(file string) (*fs.FileInfo, error) { fstat, err := os.Stat(file) if err != nil { return nil, fmt.Errorf("failed to stat input file %s: %v", file, err) } else if fstat.IsDir() { return nil, fmt.Errorf("input file %s is a directory", file) } return &fstat, nil } func OpenFileAsReadonly(filepath string) (*os.File, error) { f, err := os.OpenFile(filepath, os.O_RDONLY|syscall.O_DIRECT, FilePermission_0400) if err != nil { return nil, fmt.Errorf("failed to open file %s as readonly: %v", filepath, err) } return f, nil } func readBytesFromFile(f *os.File, numBytesToRead int, b []byte) error { numBytesRead, err := f.Read(b) if err != nil { return fmt.Errorf("failed to read file %s: %v", f.Name(), err) } if numBytesRead != numBytesToRead { return fmt.Errorf("failed to read file %s, expected read bytes = %d, actual read bytes = %d", f.Name(), numBytesToRead, numBytesRead) } return nil } // Finds if two local files have identical content (equivalnt to binary diff). // Needs (a) both files to exist, (b)read permission on both the files, (c) both // inputs to be proper files, i.e. directories not supported. // Compares file names first. If different, compares sizes next. // If sizes match, then compares the contents of both the files. // Returns true if no error and files match. // Returns false if files don't match (captures reason for mismatch in err) or if any other error. func AreFilesIdentical(filepath1, filepath2 string) (bool, error) { if filepath1 == "" || filepath2 == "" { return false, fmt.Errorf("one or both files being diff'ed have empty path") } else if filepath1 == filepath2 { return true, nil } fstat1, err := StatFile(filepath1) if err != nil { return false, err } fstat2, err := StatFile(filepath2) if err != nil { return false, err } file1size := (*fstat1).Size() file2size := (*fstat2).Size() if file1size != file2size { return false, fmt.Errorf("files don't match in size: %s (%d bytes), %s (%d bytes)", filepath1, file1size, filepath2, file2size) } if file1size == 0 { return true, nil } f1, err := OpenFileAsReadonly(filepath1) if err != nil { return false, err } defer CloseFile(f1) f2, err := OpenFileAsReadonly(filepath2) if err != nil { return false, err } defer CloseFile(f2) sizeRemaining := int(file1size) b1 := make([]byte, ChunkSizeForContentComparison) b2 := make([]byte, ChunkSizeForContentComparison) numBytesBeingRead := ChunkSizeForContentComparison for sizeRemaining > 0 { if sizeRemaining < ChunkSizeForContentComparison { numBytesBeingRead = sizeRemaining } err := readBytesFromFile(f1, numBytesBeingRead, b1) if err != nil { return false, err } err = readBytesFromFile(f2, numBytesBeingRead, b2) if err != nil { return false, err } if !bytes.Equal(b1[:numBytesBeingRead], b2[:numBytesBeingRead]) { return false, fmt.Errorf("files don't match in content: %s, %s", filepath1, filepath2) } sizeRemaining -= numBytesBeingRead } return true, nil } // Returns size of a give GCS object with path (without 'gs://'). // Fails if the object doesn't exist or permission to read object's metadata is not // available. func GetGcsObjectSize(gcsObjPath string) (int, error) { stdout, err := ExecuteGcloudCommandf("storage du -s gs://%s", gcsObjPath) if err != nil { return 0, err } // The above gcloud command returns output in the following format: // <size> <gcs-object-path> // So, we need to pick out only the first string before ' '. gcsObjectSize, err := strconv.Atoi(strings.TrimSpace(strings.Split(string(stdout), " ")[0])) if err != nil { return gcsObjectSize, err } return gcsObjectSize, nil } // Deletes a given GCS object (with path without 'gs://'). // Fails if the object doesn't exist or permission to delete object is not // available. func DeleteGcsObject(gcsObjPath string) error { _, err := ExecuteGcloudCommandf("rm gs://%s", gcsObjPath) return err } // Clears cache-control attributes on given GCS object (with path without 'gs://'). // Fails if the file doesn't exist or permission to modify object's metadata is not // available. func ClearCacheControlOnGcsObject(gcsObjPath string) error { _, err := ExecuteGcloudCommandf("storage objects update --cache-control='' gs://%s", gcsObjPath) return err } func CreateFile(filePath string, filePerms os.FileMode, t testing.TB) (f *os.File) { // Creating a file shouldn't create file on GCS. f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, filePerms) if err != nil { t.Fatalf("CreateFile(%s): %v", filePath, err) } return } func OpenFiles(t *testing.T, filePaths []string) []*os.File { t.Helper() var files []*os.File // Open all files. for _, filePath := range filePaths { file, err := os.OpenFile(filePath, os.O_RDWR|syscall.O_DIRECT|os.O_CREATE, FilePermission_0600) require.NoError(t, err) files = append(files, file) } return files } func OpenFile(filePath string, t *testing.T) (f *os.File) { f, err := os.OpenFile(filePath, os.O_RDWR, FilePermission_0777) if err != nil { t.Fatalf("OpenFile(%s): %v", filePath, err) } return } func OpenFileWithODirect(t *testing.T, filePath string) (f *os.File) { t.Helper() f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC|syscall.O_DIRECT, FilePermission_0600) if err != nil { require.NoError(t, err) } return } func CreateSymLink(filePath, symlink string, t *testing.T) { err := os.Symlink(filePath, symlink) // Verify os.Symlink operation succeeds. if err != nil { t.Fatalf("os.Symlink(%s, %s): %v", filePath, symlink, err) } } func VerifyStatFile(filePath string, fileSize int64, filePerms os.FileMode, t *testing.T) { fi, err := os.Stat(filePath) if err != nil { t.Fatalf("os.Stat err: %v", err) } if fi.Name() != path.Base(filePath) { t.Fatalf("File name mismatch in stat call. Expected: %s, Got: %s", path.Base(filePath), fi.Name()) } if fi.Size() != fileSize { t.Fatalf("File size mismatch in stat call. Expected: %d, Got: %d", fileSize, fi.Size()) } if fi.Mode() != filePerms { t.Fatalf("File permissions mismatch in stat call. Expected: %v, Got: %v", filePerms, fi.Mode()) } } func VerifyReadFile(filePath, expectedContent string, t *testing.T) { gotContent, err := os.ReadFile(filePath) // Verify os.ReadFile operation succeeds. if err != nil { t.Fatalf("os.ReadFile(%s): %v", filePath, err) } if expectedContent != string(gotContent) { t.Fatalf("Content mismatch. Expected: %s, Got: %s", expectedContent, gotContent) } } func VerifyFileEntry(entry os.DirEntry, fileName string, size int64, t *testing.T) { if entry.IsDir() { t.Fatalf("Expected: file entry, Got: directory entry.") } if entry.Name() != fileName { t.Fatalf("File name, Expected: %s, Got: %s", fileName, entry.Name()) } fileInfo, err := entry.Info() if err != nil { t.Fatalf("%s.Info() err: %v", fileName, err) } if fileInfo.Size() != size { t.Fatalf("Local file %s size, Expected: %d, Got: %d", fileName, size, fileInfo.Size()) } } func VerifyReadLink(expectedTarget, symlinkName string, t *testing.T) { gotTarget, err := os.Readlink(symlinkName) // Verify os.Readlink operation succeeds. if err != nil { t.Fatalf("os.Readlink(%s): %v", symlinkName, err) } if expectedTarget != gotTarget { t.Fatalf("Symlink target mismatch. Expected: %s, Got: %s", expectedTarget, gotTarget) } } func WriteWithoutClose(fh *os.File, content string, t *testing.T) { _, err := fh.Write([]byte(content)) if err != nil { t.Fatalf("Error while writing to local file. err: %v", err) } } func WriteAt(content string, offset int64, fh *os.File, t testing.TB) { _, err := fh.WriteAt([]byte(content), offset) if err != nil { t.Fatalf("%s.WriteAt(%s, %d): %v", fh.Name(), content, offset, err) } } func CloseFileShouldNotThrowError(t testing.TB, file *os.File) { err := file.Close() assert.NoError(t, err) } func CloseFileShouldThrowError(t *testing.T, file *os.File) { t.Helper() if err := file.Close(); err == nil { t.Fatalf("file.Close() for file %s should throw an error: %v", file.Name(), err) } } func SyncFile(fh *os.File, t *testing.T) { err := fh.Sync() // Verify fh.Sync operation succeeds. if err != nil { t.Fatalf("%s.Sync(): %v", fh.Name(), err) } } func SyncFileShouldThrowError(t *testing.T, file *os.File) { t.Helper() if err := file.Sync(); err == nil { t.Fatalf("file.Close() for file %s should throw an error: %v", file.Name(), err) } } func CreateFileWithContent(filePath string, filePerms os.FileMode, content string, t testing.TB) { fh := CreateFile(filePath, filePerms, t) WriteAt(content, 0, fh, t) CloseFileShouldNotThrowError(t, fh) } // CreateFileOfSize creates a file of given size with random data. func CreateFileOfSize(fileSize int64, filePath string, t testing.TB) { randomData, err := GenerateRandomData(fileSize) if err != nil { t.Errorf("operations.GenerateRandomData: %v", err) } CreateFileWithContent(filePath, FilePermission_0600, string(randomData), t) } // CalculateCRC32 calculates and returns the CRC-32 checksum of the data from the provided Reader. func CalculateCRC32(src io.Reader) (uint32, error) { crc32Table := crc32.MakeTable(crc32.Castagnoli) // Pre-calculate the table hasher := crc32.New(crc32Table) if _, err := io.Copy(hasher, src); err != nil { return 0, fmt.Errorf("error calculating CRC-32: %w", err) // Wrap error } return hasher.Sum32(), nil // Return checksum and nil error on success } // CalculateFileCRC32 calculates and returns the CRC-32 checksum of a file. func CalculateFileCRC32(filePath string) (uint32, error) { // Open file with simplified flags and permissions file, err := os.Open(filePath) if err != nil { return 0, fmt.Errorf("error opening file: %w", err) } defer file.Close() // Ensure file closure return CalculateCRC32(file) } // SizeOfFile returns the size of the given file by path. // by invoking a stat call on it. func SizeOfFile(filepath string) (size int64, err error) { fstat, err := StatFile(filepath) if err != nil { return 0, err } return (*fstat).Size(), nil } func writeGzipToFile(f *os.File, filepath, content string, contentSize int) (string, error) { w := gzip.NewWriter(f) if w == nil { return "", fmt.Errorf("failed to create gzip writer for file %s", filepath) } defer func() { if err := w.Close(); err != nil { log.Printf("failed to close gzip writer for file %s: %v", filepath, err) } }() n, err := w.Write([]byte(content)) if err != nil { return "", fmt.Errorf("failed to write content to %s using gzip-writer: %w", filepath, err) } if n != contentSize { return "", fmt.Errorf("failed to write to gzip file %s. Content-size: %d bytes, wrote = %d bytes", filepath, contentSize, n) } return filepath, nil } func writeTextToFile(f *os.File, filepath, content string, contentSize int) (string, error) { n, err := f.WriteString(content) if err != nil { return "", err } if n != contentSize { return "", fmt.Errorf("failed to write to text file %s. Content-size: %d bytes, wrote = %d bytes", filepath, contentSize, n) } return filepath, nil } // Creates a temporary file (name-collision-safe) in /tmp with given content. // If gzipCompress is true, output file is a gzip-compressed file. // Caller is responsible for deleting the created file when done using it. // Failure cases: // 1. os.CreateTemp() returned error or nil handle // 2. gzip.NewWriter() returned nil handle // 3. Failed to write the content to the created temp file func CreateLocalTempFile(content string, gzipCompress bool) (string, error) { // create appropriate name template for temp file filenameTemplate := "testfile-*.txt" if gzipCompress { filenameTemplate += ".gz" } f, err := os.CreateTemp(TmpDirectory, filenameTemplate) if err != nil { return "", fmt.Errorf("failed to create tempfile for template %s: %w", filenameTemplate, err) } if f == nil { return "", fmt.Errorf("nil file handle returned from os.CreateTemp") } defer CloseFile(f) if gzipCompress { return writeGzipToFile(f, f.Name(), content, len(content)) } return writeTextToFile(f, f.Name(), content, len(content)) } // ReadAndCompare reads content from the given file paths and compares them. func ReadAndCompare(t *testing.T, filePathInMntDir string, filePathInLocalDisk string, offset int64, chunkSize int64) { t.Helper() mountContents, err := ReadChunkFromFile(filePathInMntDir, chunkSize, offset, os.O_RDONLY|syscall.O_DIRECT) if err != nil { t.Fatalf("error in read file from mounted directory :%d", err) } diskContents, err := ReadChunkFromFile(filePathInLocalDisk, chunkSize, offset, os.O_RDONLY) if err != nil { t.Fatalf("error in read file from local directory :%d", err) } if !bytes.Equal(mountContents, diskContents) { t.Fatalf("data mismatch between mounted directory and local disk") } } func CreateLocalFile(ctx context.Context, t *testing.T, mntDir string, bucket gcs.Bucket, fileName string) (filePath string, f *os.File) { t.Helper() // Creating a file shouldn't create file on GCS. filePath = path.Join(mntDir, fileName) f, err := os.Create(filePath) assert.Equal(t, nil, err) ValidateObjectNotFoundErr(ctx, t, bucket, fileName) return } func CloseLocalFile(t *testing.T, f **os.File) error { t.Helper() err := (*f).Close() *f = nil return err } func CheckLogFileForMessage(t *testing.T, expectedLog, logFile string) bool { file, err := os.Open(logFile) require.NoError(t, err, "Failed to open log file") defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { if strings.Contains(scanner.Text(), expectedLog) { return true } } return false }