library/base/file_utilities.cpp (609 lines of code) (raw):
/*
* Copyright (c) 2010, 2022, Oracle and/or its affiliates. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2.0,
* as published by the Free Software Foundation.
*
* This program is designed to work with certain software (including
* but not limited to OpenSSL) that is licensed under separate terms, as
* designated in a particular file or component or in included license
* documentation. The authors of MySQL hereby grant you an additional
* permission to link the program and your derivative works with the
* separately licensed software that they have either included with
* the program or referenced in the documentation.
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
* the GNU General Public License, version 2.0, for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "base/string_utilities.h"
#include "base/file_utilities.h"
#include "base/file_functions.h"
#include <stdexcept>
#include <glib.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <glib/gstdio.h>
#ifdef _MSC_VER
#include <windows.h>
#else
#include <errno.h>
#include <fcntl.h>
#include <sys/file.h>
#endif
#include <algorithm>
namespace base {
std::string format_file_error(const std::string &text, int err) {
#ifdef _MSC_VER
return strfmt("%s: error code %i", text.c_str(), err);
#else
return strfmt("%s: %s", text.c_str(), strerror(err));
#endif
}
file_error::file_error(const std::string &text, int err)
: std::runtime_error(format_file_error(text, err)), sys_error_code(err) {
}
error_code file_error::code() {
#ifdef _MSC_VER
switch (sys_error_code) {
case 0:
return success;
case ERROR_FILE_NOT_FOUND:
case ERROR_PATH_NOT_FOUND:
return file_not_found;
case ERROR_ALREADY_EXISTS:
return already_exists;
case ERROR_ACCESS_DENIED:
return access_denied;
default:
return other_error;
}
#else
switch (sys_error_code) {
case 0:
return success;
case ENOENT:
return file_not_found;
case EEXIST:
return already_exists;
case EACCES:
return access_denied;
default:
return other_error;
}
#endif
}
//--------------------------------------------------------------------------------------------------
std::list<std::string> scan_for_files_matching(const std::string &pattern, bool recursive) {
std::list<std::string> matches;
std::string path = dirname(pattern);
if (!g_file_test(path.c_str(), G_FILE_TEST_EXISTS)) {
return matches;
}
std::string pure_pattern = pattern.substr(path.size() + 1);
std::string bname = basename(pattern);
GPatternSpec *pat = g_pattern_spec_new(bname.c_str());
GDir *dir;
{
GError *err = NULL;
dir = g_dir_open(path.empty() ? "." : path.c_str(), 0, &err);
if (!dir) {
std::string msg = strfmt("can't open %s: %s", !path.empty() ? path.c_str() : ".", err->message);
g_error_free(err);
g_pattern_spec_free(pat);
throw std::runtime_error(msg);
}
}
const gchar *filename;
while ((filename = g_dir_read_name(dir))) {
std::string full_path = strfmt("%s%s%s", path.c_str(), G_DIR_SEPARATOR_S, filename);
// #ifdef GLIB_VERSION_2_70 won't work in RHEL9/OL9 because the glib-2.68 package already contains this definition
#if defined(GLIB_VERSION_CUR_STABLE) && defined(GLIB_VERSION_2_70) && GLIB_VERSION_CUR_STABLE >= GLIB_VERSION_2_70
bool match_string = g_pattern_spec_match_string(pat, filename);
#else
bool match_string = g_pattern_match_string(pat, filename);
#endif
if (match_string)
matches.push_back(full_path);
if (recursive && g_file_test(full_path.c_str(), G_FILE_TEST_IS_DIR)) {
std::string subpattern = strfmt("%s%s%s", full_path.c_str(), G_DIR_SEPARATOR_S, pure_pattern.c_str());
std::list<std::string> submatches = scan_for_files_matching(subpattern, true);
if (submatches.size() > 0)
matches.insert(matches.end(), submatches.begin(), submatches.end());
}
}
g_dir_close(dir);
g_pattern_spec_free(pat);
return matches;
}
//--------------------------------------------------------------------------------------------------
#ifdef _MSC_VER
LockFile::LockFile(const std::string &path)
: path(path), handle(0) {
std::wstring wpath(string_to_wstring(path));
if (path.empty())
throw std::invalid_argument("invalid path");
handle = CreateFileW(wpath.c_str(), GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (handle == INVALID_HANDLE_VALUE) {
if (GetLastError() == ERROR_SHARING_VIOLATION)
throw file_locked_error("File already locked");
throw std::runtime_error(strfmt("Error creating lock file (%i)", GetLastError()));
}
char buffer[32];
sprintf_s(buffer, "%i", GetCurrentProcessId());
DWORD bytes_written;
if (!WriteFile(handle, buffer, (DWORD)strlen(buffer), &bytes_written, NULL) || bytes_written != strlen(buffer)) {
CloseHandle(handle);
DeleteFileW(wpath.c_str());
throw std::runtime_error("Could not write to lock file");
}
}
LockFile::~LockFile() {
if (handle)
CloseHandle(handle);
DeleteFileW(string_to_wstring(path).c_str());
}
LockFile::LockStatus LockFile::check(const std::string &path) {
// Can we open the file in exclusive mode?
std::wstring wpath = string_to_wstring(path);
HANDLE h = CreateFile(wpath.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, 0, NULL);
if (h != INVALID_HANDLE_VALUE) {
CloseHandle(h);
return NotLocked;
}
switch (GetLastError()) {
case ERROR_SHARING_VIOLATION:
// If file cannot be opened for writing it's open somewhere else.
// Try to open it and read the first bytes to see if that corresponds to our process id.
h = CreateFileW(wpath.c_str(), GENERIC_READ, FILE_SHARE_WRITE | FILE_SHARE_READ, NULL, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL, NULL);
if (h != INVALID_HANDLE_VALUE) {
char buffer[32];
DWORD bytes_read;
if (ReadFile(h, buffer, sizeof(buffer), &bytes_read, NULL)) {
CloseHandle(h);
buffer[bytes_read] = 0;
if (base::atoi<int>(buffer, -1) != GetCurrentProcessId())
return LockedOther; // TODO: this unreliable. Any non-lock file would qualify here (which is wrong).
return LockedSelf;
}
CloseHandle(h);
return LockedOther;
}
// If the file cannot be read assume its locked by another process.
return LockedOther;
case ERROR_FILE_NOT_FOUND:
return NotLocked;
case ERROR_PATH_NOT_FOUND:
throw std::invalid_argument("Invalid path");
default:
throw std::runtime_error(
strfmt("Unknown error while checking file %s for locks (%i)", path.c_str(), GetLastError()));
}
}
#else
LockFile::LockFile(const std::string &apath) : path(apath) {
if (path.empty())
throw std::invalid_argument("invalid path");
fd = open(path.c_str(), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
if (fd < 0) {
// this could mean lock exists, that it's a dangling file or that it's currently being locked by some other
// process/thread
// we just go on and try to lock the file if the file already exists
if (errno == ENOENT || errno == ENOTDIR)
throw std::invalid_argument("invalid path");
throw std::runtime_error(strfmt("%s creating lock file", g_strerror(errno)));
}
if (flock(fd, LOCK_EX | LOCK_NB) < 0) {
close(fd);
fd = -1;
if (errno == EWOULDBLOCK)
throw file_locked_error("file already locked");
throw std::runtime_error(strfmt("%s while locking file", g_strerror(errno)));
}
if (ftruncate(fd, 0)) {
close(fd);
fd = -1;
throw std::runtime_error(strfmt("%s while truncating file", g_strerror(errno)));
}
char pid[32];
snprintf(pid, sizeof(pid), "%i", getpid());
if (write(fd, pid, strlen(pid) + 1) < 0) {
close(fd);
throw std::runtime_error(strfmt("%s while locking file", g_strerror(errno)));
}
}
LockFile::~LockFile() {
if (fd >= 0)
close(fd);
unlink(path.c_str());
}
LockFile::LockStatus LockFile::check(const std::string &path) {
int fd = open(path.c_str(), O_RDONLY);
if (fd < 0)
return NotLocked;
if (flock(fd, LOCK_EX | LOCK_NB) < 0) {
char pid[32];
// couldn't lock file, check if we own it ourselves
ssize_t c = read(fd, pid, sizeof(pid) - 1);
close(fd);
if (c < 0)
return LockedOther;
pid[c] = 0;
if (base::atoi<int>(pid, -1) != getpid())
return LockedOther;
return LockedSelf;
} else // nobody holds a lock on the file, so this is a leftover
{
flock(fd, LOCK_UN);
close(fd);
return NotLocked;
}
}
#endif
bool create_directory(const std::string &path, int mode, bool with_parents) {
#ifdef _MSC_VER
SetLastError(0);
if (!CreateDirectoryW(path_from_utf8(path).c_str(), NULL)) {
DWORD error = GetLastError();
if (error == ERROR_ALREADY_EXISTS)
return false;
if (with_parents) {
if (error == ERROR_PATH_NOT_FOUND) {
// create parents too
std::string tmp = path;
std::list<std::string> stack;
while (!tmp.empty() && !file_exists(tmp)) {
stack.push_back(tmp);
tmp = dirname(tmp);
}
if (tmp.empty()) // path is invalid
throw file_error(strfmt("Could not create directory %s", path.c_str()), ERROR_PATH_NOT_FOUND);
while (!stack.empty()) {
if (!CreateDirectoryW(path_from_utf8(stack.back()).c_str(), NULL))
throw file_error(strfmt("Could not create directory %s", path.c_str()), GetLastError());
stack.pop_back();
}
return true;
}
}
throw file_error(strfmt("Could not create directory %s", path.c_str()), GetLastError());
}
#else
if (with_parents) {
if (g_mkdir_with_parents(path_from_utf8(path).c_str(), mode) < 0)
throw file_error(strfmt("Could not create directory %s", path.c_str()), errno);
} else {
// do not use g_mkdir_with_parents here, as we rely on the specific behaviour of g_mkdir (g_mkdir_with_parents
// doesnt fail if dir exists)
if (g_mkdir(path_from_utf8(path).c_str(), mode) < 0) {
if (errno == EEXIST)
return false;
throw file_error(strfmt("Could not create directory %s", path.c_str()), errno);
}
}
#endif
return true;
}
//--------------------------------------------------------------------------------------------------------------------
bool copyDirectoryRecursive(const std::string &src, const std::string &dst, bool includeFiles) {
GError *error = NULL;
GDir *srcDir, *dstDir;
const char *dirEntry;
gchar *entryPathSrc, *entryPathDst;
srcDir = g_dir_open(src.c_str(), 0, &error);
if (!srcDir && error) {
g_error_free(error);
return false;
}
dstDir = g_dir_open(dst.c_str(), 0, &error);
if (!dstDir && error) {
g_error_free(error);
create_directory(dst, 0700);
} else
g_dir_close(dstDir);
while ((dirEntry = g_dir_read_name(srcDir))) {
entryPathDst = g_build_filename(dst.c_str(), dirEntry, NULL);
entryPathSrc = g_build_filename(src.c_str(), dirEntry, NULL);
try {
if (g_file_test(entryPathSrc, G_FILE_TEST_IS_DIR))
copyDirectoryRecursive(entryPathSrc, entryPathDst, includeFiles);
if (g_file_test(entryPathSrc, G_FILE_TEST_IS_REGULAR) && includeFiles) {
std::ifstream src(entryPathSrc, std::ios::binary);
std::ofstream dst(entryPathDst, std::ios::binary);
dst << src.rdbuf();
}
} catch (...) {
g_free(entryPathSrc);
g_free(entryPathDst);
throw;
}
g_free(entryPathSrc);
g_free(entryPathDst);
}
g_dir_close(srcDir);
return true;
}
//--------------------------------------------------------------------------------------------------------------------
std::wifstream openTextInputStream(const std::string &fileName) {
std::wifstream result;
#ifdef _MSC_VER
result.open(base::string_to_wstring(fileName));
#else
result.open(fileName);
#endif
return result;
}
//--------------------------------------------------------------------------------------------------------------------
std::wofstream openTextOutputStream(const std::string &fileName) {
std::wofstream result;
#ifdef _MSC_VER
result.open(base::string_to_wstring(fileName));
#else
result.open(fileName);
#endif
return result;
}
//--------------------------------------------------------------------------------------------------------------------
std::ifstream openBinaryInputStream(const std::string &fileName) {
std::ifstream result;
#ifdef _MSC_VER
result.open(base::string_to_wstring(fileName), std::ios_base::in | std::ios_base::binary);
#else
result.open(fileName, std::ios_base::in | std::ios_base::binary);
#endif
return result;
}
//--------------------------------------------------------------------------------------------------------------------
std::ofstream openBinaryOutputStream(const std::string &fileName) {
std::ofstream result;
#ifdef _MSC_VER
result.open(base::string_to_wstring(fileName), std::ios_base::out | std::ios_base::binary);
#else
result.open(fileName, std::ios_base::out | std::ios_base::binary);
#endif
return result;
}
//--------------------------------------------------------------------------------------------------------------------
bool copyFile(const std::string &source, const std::string &target) {
std::ifstream src = openBinaryInputStream(source);
if (src.bad())
return false;
std::ofstream dst = openBinaryOutputStream(target);
if (dst.bad())
return false;
dst << src.rdbuf();
return true;
}
//--------------------------------------------------------------------------------------------------------------------
void rename(const std::string &from, const std::string &to) {
#ifdef _MSC_VER
if (!MoveFile(path_from_utf8(from).c_str(), path_from_utf8(to).c_str()))
throw file_error(strfmt("Could not rename file %s to %s", from.c_str(), to.c_str()), GetLastError());
#else
if (::g_rename(path_from_utf8(from).c_str(), path_from_utf8(to).c_str()) < 0)
throw file_error(strfmt("Could not rename file %s to %s", from.c_str(), to.c_str()), errno);
#endif
}
//--------------------------------------------------------------------------------------------------------------------
bool remove_recursive(const std::string &path) {
GError *error = NULL;
GDir *dir;
const char *dir_entry;
gchar *entry_path;
dir = g_dir_open(path.c_str(), 0, &error);
if (!dir && error) {
g_error_free(error);
return false;
}
while ((dir_entry = g_dir_read_name(dir))) {
entry_path = g_build_filename(path.c_str(), dir_entry, NULL);
if (g_file_test(entry_path, G_FILE_TEST_IS_DIR))
(void)remove_recursive(entry_path);
else
(void)::g_remove(entry_path);
g_free(entry_path);
}
(void)g_rmdir(path.c_str());
g_dir_close(dir);
return true;
}
//--------------------------------------------------------------------------------------------------------------------
/**
* Deletes file or folder.
* Returns false if the object doesn't exist and throws an exception on error.
*/
bool remove(const std::string &path) {
#ifdef _MSC_VER
if (is_directory(path)) {
if (!RemoveDirectoryW(path_from_utf8(path).c_str())) {
if (GetLastError() == ERROR_FILE_NOT_FOUND || GetLastError() == ERROR_PATH_NOT_FOUND)
return false;
throw file_error(strfmt("Could not delete file %s", path.c_str()), GetLastError());
}
} else {
if (!DeleteFileW(path_from_utf8(path).c_str())) {
if (GetLastError() == ERROR_FILE_NOT_FOUND)
return false;
throw file_error(strfmt("Could not delete file %s", path.c_str()), GetLastError());
}
}
#else
if (::g_remove(path_from_utf8(path).c_str()) < 0) {
if (errno == ENOENT)
return false;
throw file_error(strfmt("Could not delete file %s", path.c_str()), errno);
}
#endif
return true;
}
//--------------------------------------------------------------------------------------------------
/**
* Tries to delete a file or folder.
* No exception is thrown if that fails. Returns true on success, otherwise false.
*/
bool tryRemove(const std::string &path) {
#ifdef _MSC_VER
if (is_directory(path))
return RemoveDirectory(path_from_utf8(path).c_str()) == TRUE;
else
return DeleteFile(path_from_utf8(path).c_str()) == TRUE;
#else
return ::g_remove(path_from_utf8(path).c_str()) == 0;
#endif
}
//--------------------------------------------------------------------------------------------------
bool file_exists(const std::string &path) {
char *f = g_filename_from_utf8(path.c_str(), -1, NULL, NULL, NULL);
if (g_file_test(f, G_FILE_TEST_EXISTS)) {
g_free(f);
return true;
}
g_free(f);
return false;
}
//--------------------------------------------------------------------------------------------------
bool is_directory(const std::string &path) {
char *f = g_filename_from_utf8(path.c_str(), -1, NULL, NULL, NULL);
if (g_file_test(f, G_FILE_TEST_IS_DIR)) {
g_free(f);
return true;
}
g_free(f);
return false;
}
//--------------------------------------------------------------------------------------------------
std::string extension(const std::string &path) {
std::string::size_type p = path.rfind('.');
if (p != std::string::npos) {
std::string ext(path.substr(p));
if (ext.find('/') != std::string::npos || ext.find('\\') != std::string::npos)
return "";
return ext;
}
return "";
}
//--------------------------------------------------------------------------------------------------
std::string appendExtensionIfNeeded(const std::string &path, const std::string &ext) {
if (!base::hasSuffix(path, ext))
return path + ext;
return path;
}
//--------------------------------------------------------------------------------------------------
std::string dirname(const std::string &path) {
char *dn = g_path_get_dirname(path.c_str());
std::string tmp(dn);
g_free(dn);
return tmp;
}
std::string basename(const std::string &path) {
char *dn = g_path_get_basename(path.c_str());
std::string tmp(dn);
g_free(dn);
return tmp;
}
std::string strip_extension(const std::string &path) {
std::string ext;
if (!(ext = extension(path)).empty()) {
return path.substr(0, path.size() - ext.size());
}
return path;
}
FileHandle::FileHandle(const std::string &filename, const char* mode, bool throwOnFail) : _file(nullptr) {
_file = base_fopen(filename.c_str(), mode);
if (!_file && throwOnFail)
throw file_error(std::string("Failed to open file \"").append(filename).append("\""), errno);
_path = filename;
}
FileHandle &FileHandle::operator=(FileHandle &fh) {
dispose();
swap(fh);
return *this;
}
FileHandle &FileHandle::operator=(FileHandle &&fh) {
dispose();
swap(fh);
return *this;
}
std::string FileHandle::getPath() const {
return _path;
}
void FileHandle::swap(FileHandle &fh) {
std::swap(_file, fh._file);
_path = std::move(fh._path);
}
void FileHandle::dispose() {
if (_file) {
::fclose(_file);
_file = NULL;
_path = "";
}
}
/**
* Returns the last modification time of the given file.
*/
bool file_mtime(const std::string &path, time_t &mtime) {
#ifdef _MSC_VER
struct _stat stbuf;
#else
struct stat stbuf;
#endif
if (base_stat(path.c_str(), &stbuf) == 0) {
#ifdef __APPLE__
mtime = (time_t)stbuf.st_mtimespec.tv_sec;
#else
mtime = stbuf.st_mtime;
#endif
return true;
}
return false;
}
std::string makePath(const std::string &prefix, const std::string &file) {
if (prefix.empty())
return file;
if (prefix[prefix.size() - 1] == '/' || prefix[prefix.size() - 1] == '\\')
return prefix + file;
return prefix + G_DIR_SEPARATOR + file;
}
std::string joinPath(const char *prefix, ...) {
std::string path = prefix;
#ifdef _MSC_VER
char wrong_path_separator = '\\';
#else
char wrong_path_separator = '/';
#endif
std::replace(path.begin(), path.end(), wrong_path_separator, G_DIR_SEPARATOR);
std::string arg = const_cast<char *>(prefix);
va_list ap;
va_start(ap, prefix);
while (!arg.empty()) {
arg = va_arg(ap, char *);
if (!arg.empty()) {
if (path[path.size() - 1] == G_DIR_SEPARATOR)
path += arg;
else
path += G_DIR_SEPARATOR + arg;
}
}
va_end(ap);
return path;
}
//----------------------------------------------------------------------------------------------------
/**
* Returns the second path relative to the given base path, provided both have a common ancestor.
* If not then the second path is return unchanged.
* Paths can contain both forward and backward slash separators. The result only uses backslashes.
* Folder names are compared case insensitively on Windows, otherwise case matters.
*/
std::string relativePath(const std::string &basePath, const std::string &pathToMakeRelative) {
std::vector<std::string> basePathList = split_by_set(basePath, "/\\");
std::vector<std::string> otherPathList = split_by_set(pathToMakeRelative, "/\\");
#ifdef _MSC_VER
bool caseSensitive = false;
#else
bool caseSensitive = true;
#endif
size_t totalDepth = std::min(basePathList.size(), otherPathList.size());
size_t commonDepth = 0;
for (size_t i = 0; i < totalDepth; ++i, ++commonDepth) {
if (!same_string(basePathList[i], otherPathList[i], caseSensitive))
break;
}
if (commonDepth == 0)
return pathToMakeRelative;
std::string result;
for (size_t i = 0; i < basePathList.size() - commonDepth; ++i)
result += "../";
for (size_t i = commonDepth; i < otherPathList.size(); ++i) {
result += otherPathList[i];
if (i < otherPathList.size() - 1)
result += "/";
}
return result;
}
//--------------------------------------------------------------------------------------------------------------------
/**
* Returns temporary file with the given prefix.
*/
FileHandle makeTmpFile(const std::string &prefix) {
std::string tmp(prefix);
#if _MSC_VER
wchar_t tempPathBuffer[MAX_PATH] = { 0 };
DWORD dwRetVal = GetTempPath(MAX_PATH, tempPathBuffer);
if (dwRetVal > MAX_PATH || (dwRetVal == 0)) {
throw std::runtime_error("GetTempPath failed.");
}
TCHAR tempFileName[MAX_PATH] = { 0 };
UINT uRetVal = GetTempFileName(tempPathBuffer, TEXT("wb_"), 0, tempFileName);
if (uRetVal == 0) {
throw std::runtime_error("GetTempFileName failed.");
}
tmp = base::wstring_to_string(tempFileName);
#else
tmp.append("XXXXXX");
int fd = mkstemp(&tmp[0]);
if (fd == -1) {
throw std::runtime_error("Unable to create temporary file.");
}
close(fd);
#endif
FileHandle fh(tmp, "w+");
return fh;
}
std::string pathlistAppend(const std::string &l, const std::string &s) {
if (l.empty())
return s;
return l + G_SEARCHPATH_SEPARATOR + s;
}
std::string pathlistPrepend(const std::string &l, const std::string &s) {
if (l.empty())
return s;
return s + G_SEARCHPATH_SEPARATOR + l;
}
std::string cwd() {
#ifdef _MSC_VER
wchar_t widePath[FILENAME_MAX + 1];
::_wgetcwd(widePath, FILENAME_MAX);
return normalize_path(wstring_to_string(widePath));
#else
char currentPath[FILENAME_MAX + 1];
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
::getcwd(currentPath, FILENAME_MAX);
return currentPath;
#pragma GCC diagnostic pop
#endif
}
//--------------------------------------------------------------------------------------------------------------------
};