in python/packages/mysql_gadgets/command/sandbox.py [0:0]
def create_sandbox(**kwargs):
"""Create a new MySQL sandbox.
:param kwargs: Keyword arguments:
port: The port where the sandbox will listen for
MySQL connections.
passwd: password to be used for the root account in the
MySQL sandbox.
basedir: the directory that will be used as the basedir
of the MySQL sandbox instance.
mysqlx_port: the port where the sandbox will listen
for the X-Protocol connections. Default
value is <port>*10.
sandbox_base_dir: base path for the created MySQL
sandbox instances. Default is
DEFAULT_SANDBOX_DIR.
mysqld_path: Path to the mysqld executable. By default
it will search the PATH of the system.
mysqladmin_path: Path to the mysqladmin executable. By
default it will search the PATH of the
system.
mysql_ssl_rsa_setup_path: Path to the mysql_ssl_rsa_setup
executable. By default it will
search the PATH of the system.
server_id: Server-id value of the MySQL sandbox
instance. By default a random id is used.
opt: list of additional values to save under the
[mysqld] section of the option file.
timeout: timeout in seconds to wait for the sandbox
instance to start listening for connections.
ignore_ssl_error: If false (default) the sandbox must be
created with support for SSL throwing an error
if SSL support cannot be added. If true no error
will be issued if SSL support cannot be provided
and SSL support will be skipped.
start: if true leave the sandbox running after its creation
:type kwargs: dict
"""
# get mandatory values
try:
port = int(kwargs["port"])
except KeyError:
raise exceptions.GadgetError("It is mandatory to specify a port.")
password = kwargs.get("passwd")
ignore_ssl_error = kwargs.get("ignore_ssl_error", False)
start = kwargs.get("start", False)
# Get default values for optional variables
timeout = kwargs.get("timeout", SANDBOX_TIMEOUT)
mysqlx_port = int(kwargs.get("mysqlx_port", port * 10))
# Verify if mysqlx port is valid.
if (mysqlx_port < 1024 or mysqlx_port > 65535) and \
"mysqlx_port" not in kwargs.keys():
raise exceptions.GadgetError(
"Invalid X port '{0}', it must be >= 1024 and <= 65535. "
"Use a lower value for 'port' to generate a valid X port "
"(by default, portx = port * 10), or use the 'portx' "
"option to specify a custom value.".format(mysqlx_port))
_, sandbox_dir = _get_sandbox_dirs(**kwargs)
enc_sandbox_dir = tools.fs_encode(sandbox_dir)
# Check if sandbox_dir is empty
if os.path.isdir(enc_sandbox_dir) and os.listdir(enc_sandbox_dir):
raise exceptions.GadgetError(u"The sandbox dir '{0}' is not empty."
u"".format(sandbox_dir))
# If no value is provided for mysqld, search value on PATH and default
# mysqld paths.
try:
mysqld_path = kwargs.get("mysqld_path",
tools.get_tool_path(
None, "mysqld", search_path=True,
required=True,
check_tool_func=server.is_valid_mysqld))
except exceptions.GadgetError as err:
if err.errno == 1:
raise exceptions.GadgetError(_ERROR_CANNOT_FIND_TOOL.format(
exec_name="mysqld", path_var_name=PATH_ENV_VAR))
elif err.errno == 2:
raise exceptions.GadgetError(_ERROR_CANNOT_FIND_VALID_TOOL.format(
exec_name="mysqld",
min_ver='.'.join(str(i) for i in MIN_MYSQL_VERSION),
max_ver='.'.join(str(i) for i in MAX_MYSQL_VERSION),
path_var_name=PATH_ENV_VAR))
else:
raise exceptions.GadgetError(_ERROR_CHECK_VALID_TOOL.format(
exec_name="mysqld", error=err.errmsg))
# If no value is provided for mysqladmin, by default search value on PATH
mysqladmin_path = kwargs.get("mysqladmin_path",
tools.get_tool_path(None, "mysqladmin",
search_path=True,
required=False))
if not mysqladmin_path:
raise exceptions.GadgetError(_ERROR_CANNOT_FIND_TOOL.format(
exec_name="mysqladmin", path_var_name=PATH_ENV_VAR))
# If no value is provided for mysql_ssl_rsa_setup, by default search value
# on PATH
mysql_ssl_rsa_setup_path = kwargs.get(
"mysql_ssl_rsa_setup_path", tools.get_tool_path(
None, "mysql_ssl_rsa_setup", search_path=True, required=False))
if not mysql_ssl_rsa_setup_path and not ignore_ssl_error:
raise exceptions.GadgetError(
_ERROR_CANNOT_FIND_TOOL.format(exec_name="mysql_ssl_rsa_setup",
path_var_name=PATH_ENV_VAR))
# Checking if mysql, mysqladmin and mysql_ssl_rsa_setup meet requirements
if not tools.is_executable(mysqld_path):
raise exceptions.GadgetError(
"Provided mysqld '{0}' is not a valid executable."
"".format(mysqld_path))
if not tools.is_executable(mysqladmin_path):
raise exceptions.GadgetError(
"Provided mysqladmin '{0}' is not a valid executable."
"".format(mysqladmin_path))
if not ignore_ssl_error and not \
tools.is_executable(mysql_ssl_rsa_setup_path):
raise exceptions.GadgetError(
"Provided mysql_ssl_rsa_setup '{0}' is not a valid executable."
"".format(mysql_ssl_rsa_setup_path))
mysqld_ver, version_str = server.get_mysqld_version(mysqld_path)
if not MIN_MYSQL_VERSION <= mysqld_ver < MAX_MYSQL_VERSION:
raise exceptions.GadgetError(
_ERROR_VERSION_NOT_SUPPORTED.format(
mysqld_path, version_str,
'.'.join(str(i) for i in MIN_MYSQL_VERSION),
'.'.join(str(i) for i in MAX_MYSQL_VERSION)))
basedir = kwargs.get("basedir", None)
if basedir is None:
# If no value was provided, try to guess it from mysqld
try:
basedir = _find_basedir(mysqld_path)
except exceptions.GadgetError:
raise exceptions.GadgetError(
"Unable to find the basedir for mysqld executable '{0}'. "
"Please use the --basedir option to specify it."
"".format(mysqld_path))
# By default a random ID value.
server_id = kwargs.get("server_id", server.generate_server_id())
# Get list of options to override
mysqld_opts = kwargs.get("opt", [])
opt_override_dict = option_list_to_dictionary(mysqld_opts)
# Datadir
datadir = os.path.join(sandbox_dir, "sandboxdata")
# Binary dir
sandbox_bin_dir = os.path.join(sandbox_dir, "bin")
enc_sandbox_bin_dir = tools.fs_encode(sandbox_bin_dir)
# pid file_path
pidf_path = os.path.join(sandbox_dir, "{0}.pid".format(port))
# Initialize new mysql sandbox
# pylint: disable=E1101
_LOGGER.step("Initializing new MySQL sandbox on '%s'.", sandbox_dir)
# Check if sandbox_dir exists:
if not os.path.isdir(enc_sandbox_dir):
# Try to create it if it does not exist
try:
os.makedirs(enc_sandbox_dir)
os.makedirs(enc_sandbox_bin_dir)
except OSError as err:
raise exceptions.GadgetError(
_ERROR_CREATE_DIR.format(dir="sandbox", dir_path=sandbox_dir,
error=unicode(err)))
# Update opt_override_dict with value used for secure_file_priv.
_set_secure_file_priv(opt_override_dict, sandbox_dir)
_LOGGER.debug("Option secure_file_priv will be set with value: %s",
opt_override_dict['secure_file_priv'])
# Create option file dictionary
# MySQL prefers Unix stile paths, so convert all paths to unix style
opt_dict = {"mysqld": {
"port": port,
"loose_mysqlx_port": mysqlx_port,
"server_id": server_id,
"socket": "mysqld.sock",
"loose_mysqlx_socket": "mysqlx.sock",
"basedir": basedir.replace("\\", "/"),
"datadir": datadir.replace("\\", "/"),
"report_port": port,
"report_host": "127.0.0.1",
"log_error": os.path.join(datadir, "error.log").replace("\\", "/"),
"binlog_checksum": "NONE",
"gtid_mode": "ON",
"transaction_write_set_extraction": "XXHASH64",
"binlog_format": "ROW",
"log_bin": None,
"enforce_gtid_consistency": "ON",
"pid_file": pidf_path.replace("\\", '/'),
}, "client": {
"port": port,
"user": "root",
"protocol": "TCP",
}}
# master_info_repository and relay_log_info_repository were deprecated in
# 8.0.23 and the setting TABLE is the default since then
if mysqld_ver < (8, 0, 23):
opt_dict["mysqld"]["master_info_repository"] = "TABLE"
opt_dict["mysqld"]["relay_log_info_repository"] = "TABLE"
# log_slave_updates is ON by default since 8.0.3
if mysqld_ver < (8, 0, 3):
opt_dict["mysqld"]["log_slave_updates"] = "ON"
if mysqld_ver < (8, 0, 13):
# Disable syslog to avoid issue on Windows.
opt_dict["mysqld"]["loose_log_syslog"] = "OFF"
# Enable mysql_cache_cleaner plugin on server versions = 8.0.4.
# This plugin is required for the hash based authentication to work
# (caching_sha2_password) to allow the shell to connect using the X
# protocol if SSL is disabled.
if mysqld_ver == (8, 0, 4):
opt_dict["mysqld"]["mysqlx_cache_cleaner"] = "ON"
# Starting on mysql 8.0.21, group replication supports binlog_checksum
# so we can remove the NONE requirement from opt dict an use the
# server default (CRC32)
if mysqld_ver >= (8, 0, 21):
del opt_dict["mysqld"]["binlog_checksum"]
# Starting with MySQL 8.0.23, having parallel-appliers enabled is a
# requirement for InnoDB cluster/ReplicaSet usage.
# So when deploying sandboxes, we already enable those settings
if mysqld_ver >= (8, 0, 23):
opt_dict["mysqld"]["binlog_transaction_dependency_tracking"] = "WRITESET"
if mysqld_ver >= (8, 0, 26):
opt_dict["mysqld"]["replica_preserve_commit_order"] = "ON"
opt_dict["mysqld"]["replica_parallel_type"] = "LOGICAL_CLOCK"
opt_dict["mysqld"]["replica_parallel_workers"] = 4
else:
opt_dict["mysqld"]["slave_preserve_commit_order"] = "ON"
opt_dict["mysqld"]["slave_parallel_type"] = "LOGICAL_CLOCK"
opt_dict["mysqld"]["slave_parallel_workers"] = 4
# MySQLx plugin is automatically loaded starting from versions 8.0.11.
if mysqld_ver < (8, 0, 11):
opt_dict["mysqld"]["plugin_load"] = \
"mysqlx.so" if os.name == "posix" else "mysqlx.dll"
if opt_override_dict:
# If port is one of the options to override raise exception
_LOGGER.debug("Adding/Overriding option file values.")
if "port" in opt_override_dict:
raise exceptions.GadgetError(_ERROR_OVERRIDE_PORT)
# override mysqld dict with options received from cmd line
opt_dict["mysqld"].update(opt_override_dict)
# Create option file
optf_path = create_option_file(opt_dict, "my.cnf", sandbox_dir)
# If on Linux, create a temporary copy of the mysqld binary to avoid
# possible AppArmor or SELinux issues.
# Note: Creating a symbolic link will not solve the problem.
if os.name != "nt" and sys.platform != "darwin":
local_mysqld_path = os.path.join(sandbox_dir, "bin", "mysqld")
try:
_LOGGER.debug(u"Copying mysqld binary '%s' to '%s'", mysqld_path,
local_mysqld_path)
shutil.copy(tools.fs_encode(mysqld_path),
tools.fs_encode(local_mysqld_path))
mysql_bindir = os.path.dirname(mysqld_path)
# Symlink possibly bundled OpenSSL shared libs
for name in os.listdir(tools.fs_encode(mysql_bindir)):
if name.startswith("lib") and ".so" in name:
path = os.path.join(mysql_bindir, name)
new_path = os.path.join(sandbox_dir, "bin", name)
_LOGGER.debug(u"Symlinking '%s' to '%s'", path,
new_path)
os.symlink(tools.fs_encode(path),
tools.fs_encode(new_path))
except (IOError, shutil.Error) as err:
raise exceptions.GadgetError(
u"Unable to copy mysqld binary '{0}' to '{1}': '{2}'."
u"".format(mysqld_path, sandbox_dir, unicode(err)))
# Copies the protobuf libraries when they are bundled in the package
# it is assumed that id not bundled they are system wide and the mysqld
# binary will be able to find them.
# TODO(rennox): This should be turned into a function like get_tool_path
# As right now it is handling the cases of working with a package using
# the variants used by PB2 on the tests, but it does not guarantee it
# will work with a system installed MySQL.
if mysqld_ver >= (8, 0, 18):
library_path = os.path.join(basedir, "lib", "mysql", "private")
sandbox_lib_dir = os.path.join(sandbox_dir, "lib", "mysql",
"private")
if not os.path.exists(library_path):
library_path = os.path.join(basedir, "lib64", "mysql",
"private")
sandbox_lib_dir = os.path.join(sandbox_dir, "lib64", "mysql",
"private")
if not os.path.exists(library_path):
library_path = os.path.join(basedir, "lib", "private")
sandbox_lib_dir = os.path.join(sandbox_dir, "lib", "private")
path = ""
if os.path.exists(library_path):
enc_sandbox_lib_dir = tools.fs_encode(sandbox_lib_dir)
try:
os.makedirs(enc_sandbox_lib_dir)
except OSError as err:
raise exceptions.GadgetError(
_ERROR_CREATE_DIR.format(dir="protobuf library",
dir_path=sandbox_lib_dir,
error=str(err)))
try:
for name in os.listdir(tools.fs_encode(library_path)):
if name.startswith("lib") and ".so" in name:
path = os.path.join(library_path, name)
target_path = os.path.join(sandbox_lib_dir, name)
_LOGGER.debug(u"Copying library '%s' to '%s'", path,
sandbox_lib_dir)
shutil.copy(tools.fs_encode(path),
tools.fs_encode(target_path))
except (IOError, shutil.Error) as err:
raise exceptions.GadgetError(
u"Unable to copy mysqld library '{0}' to '{1}': '{2}'."
u"".format(path, sandbox_lib_dir, unicode(err)))
else:
local_mysqld_path = mysqld_path
# Get the command string
create_cmd = _CREATE_SANDBOX_CMD.format(
mysqld_path=tools.shell_quote(local_mysqld_path),
config_file=tools.shell_quote(os.path.normpath(optf_path)))
# If we are running the script as root , the --user=root option is needed
if os.name == "posix" and getpass.getuser() == "root":
_LOGGER.warning("Creating a sandbox as root is not recommended.")
create_cmd = "{0} --user=root".format(create_cmd)
# Fake PID to avoid the server starting the monitoring process
if os.name == "nt":
os.environ['MYSQLD_PARENT_PID'] = "{0}".format(port)
init_proc = tools.run_subprocess(create_cmd, shell=False,
stderr=subprocess.PIPE)
_, stderr = init_proc.communicate()
if init_proc.returncode != 0:
raise exceptions.GadgetError(
f"Error initializing MySQL sandbox '{port}'. '{create_cmd}' failed\